diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d93c1695..b042547a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,14 +71,19 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.11" + - name: Set up QEMU for Linux aarch64 wheel tests + uses: docker/setup-qemu-action@v4 + with: + platforms: arm64 - name: Build and test repaired Linux wheels uses: pypa/cibuildwheel@v4.1.0 env: CIBW_PLATFORM: linux + CIBW_ARCHS_LINUX: "x86_64 aarch64" with: output-dir: dist-linux - name: Verify repaired Linux wheels - run: python scripts/check_python_wheel.py --wheel-only --out-dir dist-linux + run: python scripts/check_python_wheel.py --wheel-only --out-dir dist-linux --require-repaired-linux-architectures x86_64 aarch64 - uses: actions/upload-artifact@v7 with: name: dotmatch-linux-repaired-wheels @@ -108,6 +113,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - name: Set up QEMU for Linux arm64 container tests + uses: docker/setup-qemu-action@v4 + with: + platforms: arm64 - uses: docker/setup-buildx-action@v4 - name: Build local smoke-test image run: docker build -t dotmatch:ci . @@ -118,6 +127,14 @@ jobs: docker run --rm dotmatch:ci dist ACGT AGGT | grep '^1$' docker run --rm dotmatch:ci leq 1 ACGT AGGT | grep '^true$' docker image inspect dotmatch:ci --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' | grep "^${VERSION}$" + - name: Build Linux arm64 smoke-test image + run: docker buildx build --platform linux/arm64 --load -t dotmatch:ci-arm64 . + - name: Smoke test Linux arm64 container + run: | + VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") + docker run --rm --platform linux/arm64 dotmatch:ci-arm64 --version | grep "^dotmatch ${VERSION}$" + docker run --rm --platform linux/arm64 dotmatch:ci-arm64 dist ACGT AGGT | grep '^1$' + docker run --rm --platform linux/arm64 dotmatch:ci-arm64 leq 1 ACGT AGGT | grep '^true$' - uses: docker/metadata-action@v6 id: meta with: @@ -143,8 +160,18 @@ jobs: with: context: . push: ${{ startsWith(github.ref, 'refs/tags/') }} + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + - name: Verify published multi-architecture manifest + if: startsWith(github.ref, 'refs/tags/') + env: + VERSION: ${{ github.ref_name }} + run: | + docker buildx imagetools inspect "ghcr.io/dnncha/dotmatch:${VERSION}" --raw > ghcr-manifest.json + python scripts/check_oci_manifest.py ghcr-manifest.json \ + --require-platform linux/amd64 \ + --require-platform linux/arm64 pypi-sdist: name: Publish PyPI sdist, macOS wheel, and repaired Linux wheels diff --git a/.zenodo.json b/.zenodo.json index 306dd9a0..e4552f26 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,7 +1,7 @@ { "title": "DotMatch: deterministic known-target short-DNA assignment for sequencing workflows", "upload_type": "software", - "version": "0.2.2", + "version": "0.3.0", "conceptdoi": "10.5281/zenodo.20541628", "creators": [ { @@ -25,7 +25,7 @@ ], "related_identifiers": [ { - "identifier": "10.5281/zenodo.20541629", + "identifier": "10.5281/zenodo.21511337", "relation": "isNewVersionOf", "scheme": "doi", "resource_type": "software" diff --git a/CHANGELOG.md b/CHANGELOG.md index 928e6346..17c43d83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable user-facing changes are tracked here. Public statements in release notes must stay aligned with `docs/scientific-claims.md`. +## 0.3.0 - 2026-07-23 + +### Added + +- Added `dotmatch feature matrix` for deterministic cell-by-feature matrices + from pre-extracted observation tables. The command retains unique, + ambiguous, unmatched, and invalid observations in explicit output artifacts. +- Added strict paired R1/R2 support to `dotmatch pair-count`, including + canonical read-name and record-count checks before counting. +- Added a Galaxy XML wrapper and Planemo fixtures for CRISPR guide counting. + The wrapper is a local integration asset and does not indicate IUC + acceptance. + +### Changed + +- Configured tagged releases to build repaired Linux `x86_64` and `aarch64` + wheels and publish multi-architecture `linux/amd64` and `linux/arm64` + container manifests after their checks pass. + ## 0.2.2 - 2026-07-23 ### Changed diff --git a/CITATION.cff b/CITATION.cff index 6ce2c67b..9e63c19f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,8 +8,7 @@ authors: orcid: "https://orcid.org/0009-0003-5012-7229" repository-code: "https://github.com/dnncha/dotmatch" license: Apache-2.0 -version: "0.2.2" -doi: 10.5281/zenodo.21511337 +version: "0.3.0" abstract: "DotMatch is a deterministic known-target short-DNA assignment engine for CRISPR guide counting, barcode demultiplexing, and fixed-target FASTQ workflows." keywords: - bioinformatics diff --git a/DESCRIPTION b/DESCRIPTION index 24230ca9..25514d63 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: dotmatch Title: R Interface to DotMatch (Deterministic Short-DNA Assignment) -Version: 0.2.2 +Version: 0.3.0 Authors@R: person("DotMatch Contributors", email = "donncha@example.com", role = c("aut", "cre")) Description: Provides R wrappers around the Python dotmatch package via diff --git a/Dockerfile b/Dockerfile index 5febb373..972100b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ LABEL org.opencontainers.image.title="DotMatch" \ org.opencontainers.image.source="https://github.com/dnncha/dotmatch" \ org.opencontainers.image.url="https://dotmatch.readthedocs.io/" \ org.opencontainers.image.documentation="https://dotmatch.readthedocs.io/" \ - org.opencontainers.image.version="0.2.2" \ + org.opencontainers.image.version="0.3.0" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.authors="Donncha O'Toole" diff --git a/README.md b/README.md index 50927705..b5a7e962 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ match, an ambiguous match, unmatched, or invalid. [![Documentation](https://readthedocs.org/projects/dotmatch/badge/?version=latest)](https://dotmatch.readthedocs.io/en/latest/) [![Bioconda](https://img.shields.io/conda/vn/bioconda/dotmatch?label=Bioconda)](https://anaconda.org/bioconda/dotmatch) [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://github.com/dnncha/dotmatch/blob/main/LICENSE) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.21511337.svg)](https://doi.org/10.5281/zenodo.21511337) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.20541628.svg)](https://doi.org/10.5281/zenodo.20541628) [Documentation](https://dotmatch.readthedocs.io/en/latest/) · [Getting started](https://dotmatch.readthedocs.io/en/latest/getting-started.html) · @@ -148,6 +148,53 @@ dotmatch barcode autopsy \ Open `autopsy/report.html` first. The tables beside it record offset scans, near-neighbour barcodes, correction safety, and frequent unmatched windows. +### Build a cell-by-feature matrix from extracted observations + +When an upstream workflow has already extracted feature windows and attached an +explicit cell identifier, DotMatch can write a sparse cells × features matrix: + +```bash +dotmatch feature matrix \ + --observations feature_observations.tsv \ + --targets feature_library.tsv \ + --id-column observation_id \ + --cell-column cell_barcode \ + --sequence-column feature_seq \ + --metric hamming --k 1 \ + --out-dir feature_matrix/ +``` + +The output directory contains `matrix.mtx`, cell and feature axes, long-form +counts, per-observation assignments, per-cell QC, and a JSON summary. Only +unique assignments add a matrix count. This command does not perform FASTQ +pairing, cell-barcode correction, UMI deduplication, or cell calling; those +upstream steps should remain documented with the observation table. + +See the [scverse and feature-barcode tutorial](https://dotmatch.readthedocs.io/en/latest/tutorials/scverse-perturb-seq.html) +for the file contract and AnnData handoff. + +### Count target pairs across R1 and R2 + +Use `pair-count` when a left target and a right target are sequenced in +synchronized FASTQ mates: + +```bash +dotmatch pair-count \ + --left-targets r1_targets.tsv \ + --right-targets r2_targets.tsv \ + --left-reads sample_R1.fastq.gz \ + --right-reads sample_R2.fastq.gz \ + --left-start 0 --left-length 20 \ + --right-start 0 --right-length 20 \ + --k 1 --metric hamming \ + --out pair_counts.tsv \ + --summary pair_summary.json +``` + +R1 and R2 must contain the same records in the same order. DotMatch checks the +canonical read identifier before assignment and records input synchronization, +side-specific unmatched totals, and side-specific invalid totals in the summary. + ### Check a target library Before allowing mismatch correction, check whether neighbouring targets can diff --git a/app/page.tsx b/app/page.tsx index e4864b4b..4b53ce27 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -14,7 +14,7 @@ const biocondaUrl = "https://anaconda.org/bioconda/dotmatch"; const releaseUrl = `${repoUrl}/releases`; const containerUrl = `${repoUrl}/pkgs/container/dotmatch`; const workflowExamplesUrl = `${repoUrl}/tree/main/examples/workflows`; -const doiUrl = "https://doi.org/10.5281/zenodo.21511337"; +const doiUrl = "https://doi.org/10.5281/zenodo.20541628"; const basePath = process.env.NEXT_PUBLIC_BASE_PATH ?? ""; const assignmentWorkflowImage = `${basePath}/dotmatch-read-assignment-v2.webp`; @@ -37,7 +37,7 @@ const structuredData = { name: "DotMatch", applicationCategory: "Bioinformatics software", operatingSystem: "Linux, macOS", - softwareVersion: "0.2.2", + softwareVersion: "0.3.0", softwareHelp: docsUrl, codeRepository: repoUrl, downloadUrl: pypiUrl, diff --git a/codemeta.json b/codemeta.json index f8b6dcd5..1fb3ca32 100644 --- a/codemeta.json +++ b/codemeta.json @@ -9,8 +9,8 @@ "issueTracker": "https://github.com/dnncha/dotmatch/issues", "license": "https://spdx.org/licenses/Apache-2.0", "identifier": "https://doi.org/10.5281/zenodo.20541628", - "version": "0.2.2", - "softwareVersion": "0.2.2", + "version": "0.3.0", + "softwareVersion": "0.3.0", "programmingLanguage": [ "C", "Python", diff --git a/docs/assayspec.md b/docs/assayspec.md index 9708045d..390bbded 100644 --- a/docs/assayspec.md +++ b/docs/assayspec.md @@ -165,9 +165,25 @@ Demux mode uses `mode = "demux"`, `barcodes`, `reads`, `[extract]`, and writes `demuxed/`, `summary.json`, optional `assignments.tsv`, `ambiguous.fastq`, and `unmatched.fastq`. -Pair mode uses `mode = "pair-count"`, `left_targets`, `right_targets`, `reads`, -`[left]`, and `[right]`. It writes `pair_counts.tsv`, `pair_summary.json`, and -optional `pair_assignments.tsv`. +Pair mode uses `mode = "pair-count"`, `left_targets`, `right_targets`, `[left]`, and `[right]`. +It accepts one of two input layouts: + +```toml +# Both target windows occur in one read. +reads = "reads.fastq.gz" + +# Or, remove reads and use synchronized mates. +left_reads = "sample_R1.fastq.gz" +right_reads = "sample_R2.fastq.gz" +``` + +For paired inputs, left extraction coordinates apply to `left_reads` and right +coordinates apply to `right_reads`. The two files must have the same number of +complete records in matching order. DotMatch compares canonical read IDs and +stops on the first mismatch. It writes `pair_counts.tsv`, `pair_summary.json`, and +optional `pair_assignments.tsv`. The summary records the input layout and +side-specific unmatched and invalid outcomes; methods and assay reports list +both FASTQ inputs. ## Safety Policy diff --git a/docs/command-reference.md b/docs/command-reference.md index b1c26121..7a7c6ba8 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -46,6 +46,54 @@ dotmatch audit --targets guides.tsv --k 3 --audit-mode exact --out-dir audit/ Proceed only when `safe_at_hamming_k2` or `safe_at_hamming_k3` is true for the radius you plan to use. +## Pre-extracted Feature Matrices + +```bash +dotmatch feature matrix \ + --observations feature_observations.tsv \ + --targets feature_library.tsv \ + --id-column observation_id \ + --cell-column cell_barcode \ + --sequence-column feature_seq \ + --metric hamming --k 1 --ambiguity-policy radius \ + --out-dir feature_matrix/ +``` + +Use `feature matrix` when an upstream workflow has already made a headered +observation table with an explicit cell identifier and a feature sequence +window. It writes a deterministic sparse Matrix Market matrix with cells on +rows and features on columns, plus feature/cell axes, long-form counts, +per-observation assignments, per-cell QC, and a JSON summary. + +Only unique assignments add matrix counts. The command does not pair FASTQ +reads, correct cell barcodes, deduplicate UMIs, or call cells; retain those +upstream decisions and provenance with the input table. + +## Pair Counting Across Paired FASTQs + +`pair-count` assigns a left target and a right target for each record pair. +Use `--reads` when both windows are present in one read, or give synchronized +R1 and R2 files: + +```bash +dotmatch pair-count \ + --left-targets r1_targets.tsv \ + --right-targets r2_targets.tsv \ + --left-reads sample_R1.fastq.gz \ + --right-reads sample_R2.fastq.gz \ + --left-start 0 --left-length 20 \ + --right-start 0 --right-length 20 \ + --k 1 --metric hamming \ + --out pair_counts.tsv \ + --summary pair_summary.json \ + --assignments pair_assignments.tsv +``` + +Paired inputs must contain the same number of complete FASTQ records in the +same order. DotMatch compares the first header token after removing a terminal +`/1` or `/2`; a mismatch stops the command before counts are written. The +summary records the input mode and side-specific unmatched and invalid totals. + ## AssaySpec Workflows ```bash diff --git a/docs/conf.py b/docs/conf.py index ebaff4c4..913ec293 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = "DotMatch" author = "Donncha O'Toole" copyright = "2026, Donncha O'Toole" -release = "0.2.2" +release = "0.3.0" version = release extensions = [ diff --git a/docs/distribution-release.json b/docs/distribution-release.json index d48c678f..5acfaa50 100644 --- a/docs/distribution-release.json +++ b/docs/distribution-release.json @@ -1,70 +1,70 @@ { "schema_version": 1, - "status": "partially_verified", - "release_version": "0.2.2", + "status": "not_released", + "release_version": "0.3.0", "post_release_gate": "make distribution-channels", "channels": [ { "id": "pypi", - "status": "verified", - "expected_url": "https://pypi.org/project/dotmatch/0.2.2/", - "public_url": "https://pypi.org/project/dotmatch/0.2.2/", - "evidence_url": "https://github.com/dnncha/dotmatch/actions/runs/30012092792", - "verified_date": "2026-07-23", + "status": "prepared", + "expected_url": "https://pypi.org/project/dotmatch/0.3.0/", "verification_command": "make distribution-channels", - "files": 4, - "readme_relative_links": 0 + "linux_wheel_architectures": [ + "x86_64", + "aarch64" + ], + "blocker": "The 0.3.0 source distribution and repaired manylinux/musllinux wheels are not public yet.", + "next_action": "Publish the source distribution and repaired wheels, then run make distribution-channels." }, { "id": "bioconda", - "status": "blocked", + "status": "prepared", "expected_url": "https://anaconda.org/bioconda/dotmatch", "verification_command": "make distribution-channels", - "blocker": "A DotMatch 0.2.2 recipe has not been accepted and built by Bioconda.", + "blocker": "A DotMatch 0.3.0 recipe has not been accepted and built by Bioconda.", "next_action": "Submit the rendered recipe from the immutable source archive, then verify channel propagation." }, { "id": "bioconda-assaycode", - "status": "blocked", + "status": "prepared", "expected_url": "https://anaconda.org/bioconda/assaycode", "verification_command": "make distribution-channels", - "blocker": "The optional AssayCode compatibility metapackage has not been published for 0.2.2.", - "next_action": "Update the metapackage after the matching DotMatch 0.2.2 Bioconda build is available." + "blocker": "The optional AssayCode compatibility metapackage has not been published for 0.3.0.", + "next_action": "Update the metapackage after the matching DotMatch 0.3.0 Bioconda build is available." }, { "id": "ghcr", - "status": "verified", + "status": "prepared", "expected_url": "https://github.com/dnncha/dotmatch/pkgs/container/dotmatch", - "public_url": "https://github.com/dnncha/dotmatch/pkgs/container/dotmatch", - "evidence_url": "https://github.com/dnncha/dotmatch/actions/runs/30012092792", - "verified_date": "2026-07-23", "verification_command": "make distribution-channels", - "tag": "v0.2.2", - "digest": "sha256:dcf957f3b0c42215e714983315070a3043a56dd8b3f8fed42fe6ea0219394435" + "platforms": [ + "linux/amd64", + "linux/arm64" + ], + "blocker": "The multi-architecture 0.3.0 image manifest has not been published.", + "next_action": "Publish the tag, inspect the manifest, and run the OCI runtime checks." }, { "id": "biocontainers", - "status": "blocked", + "status": "prepared", "expected_url": "https://quay.io/repository/biocontainers/dotmatch", "verification_command": "make distribution-channels", - "blocker": "The BioContainers image depends on the matching Bioconda package and has not been generated for 0.2.2.", + "blocker": "The BioContainers image depends on the matching Bioconda package and has not been generated for 0.3.0.", "next_action": "Verify the generated image after Bioconda propagation, including a runtime check on an OCI host." }, { "id": "zenodo", - "status": "verified", - "expected_url": "https://doi.org/10.5281/zenodo.21511337", - "public_url": "https://doi.org/10.5281/zenodo.21511337", - "evidence_url": "https://zenodo.org/api/records/21511337", - "verified_date": "2026-07-23", + "status": "prepared", + "expected_url": "https://doi.org/10.5281/zenodo.20541628", "verification_command": "make distribution-channels", "concept_doi": "10.5281/zenodo.20541628", - "release_doi": "10.5281/zenodo.21511337" + "blocker": "A version-specific Zenodo record for 0.3.0 has not been minted.", + "next_action": "Publish the archived release, record its minted DOI, and run make distribution-channels." } ], "blockers": [ - "The DotMatch and optional AssayCode Bioconda recipes are not published for 0.2.2.", - "BioContainers generation and an OCI runtime check remain pending the Bioconda build." + "The 0.3.0 tag has not been published to public package channels.", + "Bioconda, BioContainers, and the version-specific Zenodo record require external publication or archival steps." ], - "next_action": "Submit the DotMatch and AssayCode Bioconda recipes, then verify Bioconda and BioContainers before marking all release channels verified." + "next_action": "Publish v0.3.0, verify each public channel, and replace this prepared record only with observed URLs and digests." } diff --git a/docs/external-review-packet.md b/docs/external-review-packet.md index 1a55d383..3e1029ac 100644 --- a/docs/external-review-packet.md +++ b/docs/external-review-packet.md @@ -38,7 +38,7 @@ variant caller, UMI/cell quantifier, or screen-level statistics package. ## Minimum Review Commands ```bash -python3 -m pip install dotmatch==0.2.2 +python3 -m pip install dotmatch dotmatch --version dotmatch dist ACGT AGGT ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index d8a0c498..836e0b43 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -10,7 +10,7 @@ adapter prefixes. For the current PyPI release: ```bash -python3 -m pip install dotmatch==0.2.2 +python3 -m pip install dotmatch dotmatch --version ``` diff --git a/docs/index.md b/docs/index.md index 6e49b2ee..b2909da7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,6 +36,7 @@ the [command reference](command-reference.md). | Split reads by inline barcode | [Getting started: demultiplexing](getting-started.md#demultiplex-inline-barcodes) | | Diagnose barcode failures | [Barcode run diagnosis](getting-started.md#diagnose-a-barcode-run) | | Design or check a barcode panel | [Barcode panel design](barcode-panel-design.md) | +| Build a cell-by-feature matrix from extracted observations | [scverse and feature barcodes](tutorials/scverse-perturb-seq.md) | | Use DotMatch from Python | [Streaming Python API](streaming-api.md) | | Add DotMatch to a pipeline | [Output schemas](schemas.md) | | Evaluate DotMatch for a workflow | [Bioinformatics evaluation](bioinformatics-evaluation.md) | diff --git a/docs/methods-and-citation.md b/docs/methods-and-citation.md index 64b023ec..c06872d1 100644 --- a/docs/methods-and-citation.md +++ b/docs/methods-and-citation.md @@ -16,20 +16,20 @@ compatibility mode. If you use DotMatch, cite the software release through `CITATION.cff`. Installed packages also provide `dotmatch citation` for a copyable citation. Use the Zenodo concept DOI `10.5281/zenodo.20541628` for general software -citation. The version-specific DOI for v0.2.2 is -`10.5281/zenodo.21511337` and is recorded in `CITATION.cff`. +citation. A release-specific DOI is recorded in `CITATION.cff` after its +archive is minted. Suggested citation: -> O'Toole D. DotMatch: deterministic known-target short-DNA assignment for sequencing workflows. Software release v0.2.2. https://github.com/dnncha/dotmatch +> O'Toole D. DotMatch: deterministic known-target short-DNA assignment for sequencing workflows. Software release v0.3.0. https://github.com/dnncha/dotmatch -DOI: +DOI: ## Methods Sentence For CRISPR guide-counting workflows: -> Reads were assigned to the guide library using DotMatch v0.2.2 with known-target assignment, literal-byte sequence semantics, and the radius ambiguity policy. Count matrices retained only reads for which exactly one guide lay inside the configured edit-distance radius; ambiguous and unmatched reads were excluded from target counts and retained in diagnostic summaries. +> Reads were assigned to the guide library using DotMatch v0.3.0 with known-target assignment, literal-byte sequence semantics, and the radius ambiguity policy. Count matrices retained only reads for which exactly one guide lay inside the configured edit-distance radius; ambiguous and unmatched reads were excluded from target counts and retained in diagnostic summaries. For one-edit Levenshtein rescue: @@ -121,7 +121,7 @@ statements out until real-data comparator evidence is in the repository. ## Evidence Boundary -Describe DotMatch v0.2.2 as a known-target short-DNA assignment engine. It is +Describe DotMatch v0.3.0 as a known-target short-DNA assignment engine. It is not a genome aligner, general Edlib replacement, production Illumina demultiplexer, full Perturb-seq analysis pipeline, adapter trimmer, UMI grouper, read merger, or amplicon consensus/variant-calling workflow. Current public diff --git a/docs/packaging.md b/docs/packaging.md index cbb272bc..4adb33fc 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -20,11 +20,15 @@ for Linux and macOS. Python 3.9 or newer is required. To install an exact release: ```bash -python3 -m pip install dotmatch==0.2.2 +python3 -m pip install dotmatch== ``` The PyPI page is . +The release workflow is configured to build repaired `manylinux` and +`musllinux` wheels for `x86_64` and `aarch64`. Check the release record for +the architectures confirmed for a specific version before pinning it. + ## Bioconda When the required version is available in Bioconda: @@ -54,10 +58,14 @@ installing it outside a package manager. Release images are published to GitHub Container Registry: ```bash -docker pull ghcr.io/dnncha/dotmatch:0.2.2 -docker run --rm ghcr.io/dnncha/dotmatch:0.2.2 dist ACGT AGGT +docker pull ghcr.io/dnncha/dotmatch:v +docker run --rm ghcr.io/dnncha/dotmatch:v dist ACGT AGGT ``` +The release workflow is configured to publish `linux/amd64` and `linux/arm64` +image manifests and smoke-test both native CLI paths. Check the release record +before pinning a tag. + BioContainers images are generated after the corresponding Bioconda package is published. Their tags include the Bioconda build number, so use the tag shown on the package page rather than guessing it. @@ -117,16 +125,17 @@ make release-ready ``` The release workflow builds the source distribution and platform wheels, -repairs Linux wheels, checks their contents, publishes through PyPI trusted -publishing, and uploads the same artifacts to the GitHub release. See -[Release process](release-process.md) for the complete maintainer sequence. +repairs Linux wheels for `x86_64` and `aarch64`, checks their contents, +publishes through PyPI trusted publishing, and uploads the same artifacts to +the GitHub release. See [Release process](release-process.md) for the complete +maintainer sequence. ### Verify published artifacts PyPI trusted publishing uploads a source distribution plus a macOS wheel and -repaired manylinux/musllinux wheels. The channel check rejects raw -`linux_x86_64` wheels, creates a clean virtual environment, and runs an exact -install such as: +repaired manylinux/musllinux wheels. The channel check rejects raw Linux wheels +(`linux_x86_64` or `linux_aarch64`) for the recorded architectures, creates a +clean virtual environment, and runs an exact install such as: ```bash pip install dotmatch== @@ -138,6 +147,13 @@ Check the GitHub Container Registry image with: docker run --rm ghcr.io/dnncha/dotmatch:v --version ``` +For a multi-architecture release, inspect the image index and confirm both +recorded platforms are present: + +```bash +docker buildx imagetools inspect ghcr.io/dnncha/dotmatch:v +``` + Check the Bioconda package in a new prefix: ```bash diff --git a/docs/registries/biotools.yml b/docs/registries/biotools.yml index 2525a81e..2af9c465 100644 --- a/docs/registries/biotools.yml +++ b/docs/registries/biotools.yml @@ -8,7 +8,7 @@ description: > feature-barcode reads, primer or adapter-prefix checks, amplicon-panel starts, whitelist-style assays, and barcode panel design. license: Apache-2.0 -version: "0.2.2" +version: "0.3.0" topic: - Bioinformatics - Sequencing diff --git a/docs/release-process.md b/docs/release-process.md index c89bd7fe..c787271e 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -71,8 +71,8 @@ The workflow builds: - raw Linux wheel release artifact; - macOS wheel; - source distribution; -- repaired manylinux/musllinux Linux wheels for PyPI; -- GHCR container image; +- repaired manylinux/musllinux Linux wheels for `x86_64` and `aarch64` for PyPI; +- GHCR container image index for `linux/amd64` and `linux/arm64`; - `SHA256SUMS.txt`; - PyPI publication through trusted publishing for the sdist, macOS wheel, and repaired Linux wheels; - a draft GitHub release with generated notes. @@ -99,7 +99,7 @@ Avoid: - Confirm the Zenodo archive for the tagged release and add the release DOI to `CITATION.cff` when available. -- Publish the PyPI source distribution, native macOS wheel, and repaired manylinux/musllinux wheels through trusted publishing; do not upload raw `linux_x86_64` wheels. The PyPI project must have a trusted publisher matching repository `dnncha/dotmatch`, workflow `.github/workflows/release.yml`, and environment `pypi`. +- Publish the PyPI source distribution, native macOS wheel, and repaired manylinux/musllinux wheels for `x86_64` and `aarch64` through trusted publishing; do not upload raw Linux wheels. The PyPI project must have a trusted publisher matching repository `dnncha/dotmatch`, workflow `.github/workflows/release.yml`, and environment `pypi`. - For Bioconda updates, submit or update the `bioconda-recipes` recipe after `make bioconda-recipe-ready`. Keep the `osx-arm64` additional-platforms opt-in in that recipe copy so Bioconda CI validates the Apple Silicon build. Replace @@ -107,7 +107,7 @@ Avoid: propagation, verify with `make distribution-channels` before announcing conda install instructions or BioContainers availability. -- Confirm the GHCR image labels and tag after the source tag is immutable. +- Confirm the GHCR image labels, tag, and `linux/amd64` plus `linux/arm64` manifest descriptors after the source tag is immutable. - Run `make distribution-channels` after PyPI, Bioconda, GHCR, and Zenodo are public. -- Update `docs/distribution-release.json` with verified public and evidence links after public channels are live. +- Update `docs/distribution-release.json` with verified public and evidence links, exact PyPI Linux wheel architectures, and exact GHCR platforms after public channels are live. - Update `docs/scientific-claims.md` only when new evidence is committed and a corresponding gate passes. diff --git a/docs/schemas.md b/docs/schemas.md index 4c355922..a4124e02 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -73,6 +73,99 @@ Rules: - `k1_rescued_reads` is retained for compatibility and equals `assigned_corrected`, including in Levenshtein `k=2` runs. +## `feature_matrix/` + +Artifacts from `dotmatch feature matrix`. The input is a headered TSV/CSV of +pre-extracted observations with explicit cell identifiers and feature-sequence +windows. The command does not perform FASTQ pairing, barcode correction, UMI +deduplication, or cell calling. + +### `matrix.mtx` + +A Matrix Market coordinate matrix with **cells on rows** and **features on +columns**. Indices are one-based as required by Matrix Market. + +```text +%%MatrixMarket matrix coordinate integer general +rows columns nonzero_entries +row_index column_index count +``` + +The row order is `barcodes.tsv`; the column order is `features.tsv`. Counts +include only uniquely assigned observations. + +### `barcodes.tsv` + +```text +cell_barcode +``` + +One row per matrix row, sorted lexically by the exact input cell identifier. + +### `features.tsv` + +```text +target_id +target_seq +``` + +One row per matrix column, sorted lexically by `target_id`. + +### `cell_feature_counts.tsv` + +Long-form nonzero counts, sorted by `cell_barcode` and `target_id`. + +```text +cell_barcode +target_id +count +``` + +### `assignments.tsv` + +One row per input observation. + +```text +observation_id +cell_barcode +observed_seq +target_id +target_seq +distance +status +match_count +second_best_distance +``` + +`target_id` and `target_seq` are empty unless the observation was uniquely +assigned. `status` is one of `unique`, `ambiguous`, `none`, or `invalid`. + +### `cell_qc.tsv` + +One row per observed cell identifier. + +```text +cell_barcode +total_observations +valid_observations +assigned_unique +ambiguous +unmatched +invalid +unique_features +assignment_rate +``` + +`assignment_rate = assigned_unique / valid_observations`; it is `0.0` when a +cell has no valid feature sequence windows. + +### `summary.json` + +Records the assignment settings, the target-file hash, an observation-content +hash computed while the input is streamed, aggregate outcome counts, matrix +dimensions, artifact list, and explicit non-performed upstream steps. +`matrix_orientation` is always `cells_by_features` for this schema version. + ## `assay_manifest.summary.tsv` One row per `dotmatch assay run` execution, intended for workflow systems and @@ -117,7 +210,8 @@ right_id count ``` -Only reads with uniquely assigned left and right windows contribute to `count`. +Only record pairs with uniquely assigned left and right windows contribute to +`count`. ## `pair_assignments.tsv` @@ -139,8 +233,9 @@ pair_status ``` `pair_status` is `unique` only when both windows are uniquely assigned. If -either side is ambiguous, unmatched, or invalid, the read is excluded from -`pair_counts.tsv`. +either side is ambiguous, unmatched, or invalid, the record pair is excluded +from `pair_counts.tsv`. For paired FASTQ input, `read_id` is the shared +canonical identifier after a terminal `/1` or `/2` is removed. ## `pair_summary.json` @@ -148,6 +243,8 @@ Top-level fields: ```text workflow +input_mode +input_sync k metric alphabet_policy @@ -158,22 +255,34 @@ right_length n_left_targets n_right_targets total_reads +total_pairs assigned_pairs pair_ambiguous left_unmatched right_unmatched invalid +left_invalid +right_invalid candidates_considered candidates_verified ``` Rules: -- `assigned_pairs` counts reads where both fixed windows are uniquely assigned; -- `pair_ambiguous` counts reads where either side is ambiguous and the read is +- `input_mode` is `single-read` when both windows were read from `--reads` and + `paired-fastq` when `--left-reads` and `--right-reads` were used; +- `input_sync` is `canonical-read-id` for paired FASTQ input and `not-applicable` + for single-read input; +- `total_reads` remains the number of processed input records; `total_pairs` is the + number of synchronized R1/R2 records for paired input and equals `total_reads` for + single-read input; +- `assigned_pairs` counts record pairs where both fixed windows are uniquely assigned; +- `pair_ambiguous` counts record pairs where either side is ambiguous and the pair is excluded from pair counts; - `left_unmatched` and `right_unmatched` count side-specific no-match outcomes; -- `invalid` counts reads where either fixed window cannot be extracted. +- `invalid` counts record pairs where either fixed window cannot be extracted; +- `left_invalid` and `right_invalid` identify the side whose window could not be + extracted. A pair with two invalid windows contributes to both side totals. ## `audit_summary.tsv` diff --git a/docs/tutorials/scverse-perturb-seq.md b/docs/tutorials/scverse-perturb-seq.md index ad8c81e0..23089ebf 100644 --- a/docs/tutorials/scverse-perturb-seq.md +++ b/docs/tutorials/scverse-perturb-seq.md @@ -1,69 +1,101 @@ -# DotMatch to scverse for Perturb-seq and Feature Barcodes +# DotMatch and scverse for Perturb-seq and Feature Barcodes -This tutorial shows the intended handoff from DotMatch assignment artifacts to -AnnData/scverse objects. Use the CLI for FASTQ-scale assignment, then load the -small, stable TSV outputs into Python. +This tutorial covers the handoff from DotMatch feature assignments to +AnnData/scverse objects. Use `dotmatch feature matrix` when another workflow +has already produced one row per observation with an explicit cell identifier +and an extracted feature sequence. -## 1. Count guide or feature-barcode reads +## 1. Build a cell-by-feature matrix + +The input table must be headered TSV (or CSV) and include a cell identifier and +the feature sequence window. A minimal table looks like this: + +```text +observation_id cell_barcode feature_seq +read_001 AAACCTGAGAAACCAT ACGTACGTACGTACGTACG +read_002 AAACCTGAGAAACCAT ACGTACGTACGTACGTACG +read_003 AAACCTGAGCTAACAA TGCATGCATGCATGCATGC +``` + +Run the matrix command with the input column names explicitly: ```bash -dotmatch count \ - --targets guides.tsv \ - --reads guide_capture_R2.fastq.gz \ - --sample-label guide_capture \ - --target-start 63 \ - --target-length 19 \ - --k 1 \ +dotmatch feature matrix \ + --observations feature_observations.tsv \ + --targets feature_library.tsv \ + --id-column observation_id \ + --cell-column cell_barcode \ + --sequence-column feature_seq \ --metric hamming \ + --k 1 \ --ambiguity-policy radius \ - --ambiguous discard \ - --out guide_counts.tsv \ - --summary guide_summary.json \ - --sample-qc guide_sample_qc.tsv \ - --assignments guide_assignments.tsv + --out-dir feature_matrix/ ``` -For TotalSeq/CITE-seq-style feature barcodes, use the feature-barcode table as -`--targets` and set `--target-start` / `--target-length` to the antibody or -feature barcode window. +`feature_matrix/matrix.mtx` is a sparse **cells × features** Matrix Market +matrix. Its row order is recorded in `barcodes.tsv`; its column order and +target sequences are recorded in `features.tsv`. `cell_feature_counts.tsv` is +the same unique-assignment result in long TSV form. `assignments.tsv`, +`cell_qc.tsv`, and `summary.json` retain the full outcome and run settings. -## 2. Load counts into AnnData +Only `unique` assignments add counts. `ambiguous`, `none`, and `invalid` +observations remain in the diagnostic artifacts instead of being forced into a +feature. + +This command does not extract reads from FASTQ, pair read sides, correct cell +barcodes, deduplicate UMIs, or call cells. Perform those steps in the upstream +workflow before writing the observation table, and retain their provenance next +to the DotMatch output directory. + +## 2. Load the matrix into AnnData ```python -import dotmatch +from pathlib import Path -guide_adata = dotmatch.counts_tsv_to_anndata("guide_counts.tsv") -guide_adata.uns["dotmatch_summary_json"] = "guide_summary.json" -guide_adata.uns["dotmatch_sample_qc_tsv"] = "guide_sample_qc.tsv" -``` +import anndata as ad +import pandas as pd +from scipy.io import mmread -The count matrix contains uniquely assigned targets only. Ambiguous reads are -reported in `summary.json`, `sample_qc.tsv`, and `assignments.tsv`; they are not -silently assigned to a guide or feature. +run = Path("feature_matrix") +cells = pd.read_csv(run / "barcodes.tsv", sep="\t") +features = pd.read_csv(run / "features.tsv", sep="\t") -## 3. Attach per-read assignments when cell barcodes are available +feature_adata = ad.AnnData( + X=mmread(run / "matrix.mtx").tocsr(), + obs=cells.set_index("cell_barcode"), + var=features.set_index("target_id"), +) +feature_adata.uns["dotmatch"] = { + "summary": str(run / "summary.json"), + "assignments": str(run / "assignments.tsv"), + "cell_qc": str(run / "cell_qc.tsv"), + "ambiguity_policy": "radius", + "ambiguous_observations_counted": False, +} +``` + +## 3. Attach assignment rows when needed -If your assignment table includes a cell barcode column, convert it to an -AnnData observation-level table: +For a smaller assignment table, `assignments_to_anndata` can build the same +kind of count matrix. Keep the `status` column and request unique-only counts: ```python +import pandas as pd import dotmatch -assign_adata = dotmatch.assignments_to_anndata( - "guide_assignments.tsv", +assignments = pd.read_csv("feature_matrix/assignments.tsv", sep="\t") +feature_adata = dotmatch.assignments_to_anndata( + assignments, cell_col="cell_barcode", - target_col="target_id", + feature_col="target_id", + status_col="status", + count_unique_only=True, ) ``` -For custom pipelines, join DotMatch assignments to cell barcodes before this -step. Keep `assignment_status` so downstream filtering can distinguish unique, -ambiguous, unmatched, and invalid reads. - -## 4. Use scanpy-style helpers +## 4. Use scanpy-style helpers for notebook-scale work ```python -import scanpy as sc import dotmatch.tl as dm_tl library = [ @@ -82,31 +114,21 @@ dm_tl.assign_features( feature_adata = dm_tl.feature_counts( adata, seq_col="guide_sequence", + cell_col="cell_barcode", library=library, k=1, metric="hamming", ) ``` -Use this path for notebook-scale inspection and prototypes. For production -FASTQ processing, prefer `dotmatch count` so the exact command, assignment -engine, ambiguity policy, and QC summaries are written as reproducible files. - -## 5. Recommended scverse metadata +Use the command-line matrix writer for reproducible table-to-matrix runs. Use +the `dotmatch.tl` helpers for notebook-scale inspection where the observations +are already in AnnData. -Store DotMatch provenance in `.uns`: - -```python -adata.uns["dotmatch"] = { - "summary": "guide_summary.json", - "sample_qc": "guide_sample_qc.tsv", - "assignments": "guide_assignments.tsv", - "ambiguity_policy": "radius", - "ambiguous_reads_counted": False, -} -``` +## 5. Review per-cell assignment QC -For Perturb-seq analysis, keep guide assignment QC next to standard scRNA-seq -QC. A high ambiguous or unmatched rate usually means the guide window, barcode -library, or correction radius should be checked before interpreting guide-level -effects. +For Perturb-seq and feature-barcode analysis, review `cell_qc.tsv` alongside +standard scRNA-seq QC. A high ambiguous or unmatched rate can indicate an +incorrect feature window, target library, orientation, or correction radius. +The feature matrix alone does not establish cell identity or UMI-collapsed +molecule counts. diff --git a/examples/workflows/fixtures/README.md b/examples/workflows/fixtures/README.md index 087365a7..b817eb72 100644 --- a/examples/workflows/fixtures/README.md +++ b/examples/workflows/fixtures/README.md @@ -15,3 +15,8 @@ CI smoke checks. `sample_b.fastq` adds a second sample with one unique exact `guide_c` assignment and one unmatched read so MAGeCK-style multi-sample output is exercised. + +`feature_observations.tsv` and `feature_library.tsv` exercise the +pre-extracted cell-by-feature matrix command. They include unique exact and +corrected observations, an ambiguous observation, an unmatched observation, +and an empty feature window recorded as invalid. diff --git a/examples/workflows/fixtures/feature_library.tsv b/examples/workflows/fixtures/feature_library.tsv new file mode 100644 index 00000000..d1463ed4 --- /dev/null +++ b/examples/workflows/fixtures/feature_library.tsv @@ -0,0 +1,4 @@ +target_id target_seq +feature_a AAAA +feature_b AACC +feature_c TTTT diff --git a/examples/workflows/fixtures/feature_observations.tsv b/examples/workflows/fixtures/feature_observations.tsv new file mode 100644 index 00000000..65a0a550 --- /dev/null +++ b/examples/workflows/fixtures/feature_observations.tsv @@ -0,0 +1,7 @@ +observation_id cell_barcode feature_seq +obs_001 cell_z AAAA +obs_002 cell_z AAAT +obs_003 cell_a TTTT +obs_004 cell_a AAAC +obs_005 cell_a CCCC +obs_006 cell_a diff --git a/examples/workflows/galaxy/README.md b/examples/workflows/galaxy/README.md index ecf1c189..7c911169 100644 --- a/examples/workflows/galaxy/README.md +++ b/examples/workflows/galaxy/README.md @@ -1,25 +1,26 @@ # Galaxy CRISPR Counting Wrapper Example This directory contains local example wrappers for running DotMatch from Galaxy. -`dotmatch_crispr_count.xml` keeps the native command interface, and -`dotmatch_assay_run.xml` runs an AssaySpec and exposes the assay report and -manifest summary. +The scoped IUC candidate is `dotmatch_crispr_count.xml`, which accepts one or +more CRISPR FASTQ datasets and is pinned to the publicly available Bioconda +package `dotmatch=0.2.1`. `dotmatch_assay_run.xml` remains a local AssaySpec +example with separate review scope. Validate the XML with Planemo from the repository root: ```bash planemo lint examples/workflows/galaxy/dotmatch_crispr_count.xml -planemo test examples/workflows/galaxy/dotmatch_crispr_count.xml +planemo test --install_galaxy examples/workflows/galaxy/dotmatch_crispr_count.xml planemo lint examples/workflows/galaxy/dotmatch_assay_run.xml planemo test examples/workflows/galaxy/dotmatch_assay_run.xml ``` -The wrapper exposes a two-sample CRISPR guide-counting surface: guide library, -two FASTQ inputs, sample labels, guide offset, guide length, edit-distance -threshold, metric, ambiguity policy, and optional one-base Levenshtein indel -window. It writes a MAGeCK-compatible count table, DotMatch summary JSON, and a -`sample_qc.tsv` table suitable for MultiQC custom content. The embedded Planemo -test uses `test-data/` fixtures copied from `examples/workflows/fixtures/`. +The CRISPR wrapper exposes a guide library, one or more FASTQ inputs, guide +offset, guide length, edit-distance threshold, metric, ambiguity policy, and an +optional one-base Levenshtein indel window. It writes a MAGeCK-compatible count +table, DotMatch summary JSON, and a `sample_qc.tsv` table suitable for MultiQC +custom content. The embedded Planemo test covers two samples plus unique, +ambiguous, unmatched, and invalid fixture reads. The AssaySpec wrapper builds a reviewed `status = "ready"` TOML spec from Galaxy-staged library and FASTQ inputs, then writes `assay_report.html`, diff --git a/examples/workflows/galaxy/dotmatch_crispr_count.xml b/examples/workflows/galaxy/dotmatch_crispr_count.xml index 49b9ce7d..921ea9ea 100644 --- a/examples/workflows/galaxy/dotmatch_crispr_count.xml +++ b/examples/workflows/galaxy/dotmatch_crispr_count.xml @@ -1,47 +1,44 @@ - - assign CRISPR guide reads to a known guide library + + count guides from known-target FASTQ reads - dotmatch + dotmatch + dotmatch --version samples.tsv +#import re -dotmatch crispr-count \ - --library '$library' \ - --samples samples.tsv \ - --guide-start '$guide_start' \ - --guide-length '$guide_length' \ - --k '$k' \ - --metric '$metric' \ - --ambiguity-policy radius \ +#set $indel_args = "" #if str($metric) == 'levenshtein' and int($indel_window) > 0 - --indel-window '$indel_window' \ + #set $indel_args = "--indel-window '" + str($indel_window) + "'" #end if - --out '$counts' \ - --summary '$summary' \ - --sample-qc '$sample_qc' \ - --ambiguous '$ambiguous' + +printf 'sample_id\tfastq\n' > samples.tsv && +#for $sample in $reads: + #set $sample_id = re.sub(r'[^\w.-]', '_', str($sample.element_identifier)) + #if $sample.is_of_type('fastq.gz', 'fastqsanger.gz'): + #set $fastq_name = $sample_id + '.fastq.gz' + #else: + #set $fastq_name = $sample_id + '.fastq' + #end if +ln -s '$sample' '$fastq_name' && +printf '%s\t%s\n' '$sample_id' '$fastq_name' >> samples.tsv && +#end for +dotmatch crispr-count --library '$library' --samples samples.tsv --guide-start '$guide_start' --guide-length '$guide_length' --k '$k' --metric '$metric' --ambiguity-policy radius $indel_args --out '$counts' --summary '$summary' --sample-qc '$sample_qc' --ambiguous '$ambiguous' --no-progress ]]> - - - - + - - + + - + - - + + @@ -52,10 +49,7 @@ dotmatch crispr-count \ - - - - + @@ -72,29 +66,36 @@ dotmatch crispr-count \ - - - + + + - + 10.5281/zenodo.20541628 diff --git a/include/qdalign.h b/include/qdalign.h index 78c46357..5f08b3aa 100644 --- a/include/qdalign.h +++ b/include/qdalign.h @@ -7,7 +7,7 @@ extern "C" { #endif -#define QDALN_VERSION "0.2.2" +#define QDALN_VERSION "0.3.0" #define QDALN_ALPHABET_POLICY "literal-byte; A/C/G/T/N/IUPAC symbols are ordinary byte symbols; no wildcard expansion" enum qdaln_match_status { diff --git a/package.json b/package.json index 857e658f..0a3a55dd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dotmatch-site", - "version": "0.2.2", + "version": "0.3.0", "private": true, "scripts": { "dev": "next dev", diff --git a/packaging/bioconda/assaycode-meta.yaml b/packaging/bioconda/assaycode-meta.yaml index 581e54c5..3689c0c4 100644 --- a/packaging/bioconda/assaycode-meta.yaml +++ b/packaging/bioconda/assaycode-meta.yaml @@ -1,4 +1,4 @@ -{% set version = "0.2.2" %} +{% set version = "0.3.0" %} package: name: assaycode diff --git a/packaging/bioconda/meta.yaml b/packaging/bioconda/meta.yaml index aef12fe3..b0354e4f 100644 --- a/packaging/bioconda/meta.yaml +++ b/packaging/bioconda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "dotmatch" %} -{% set version = "0.2.2" %} +{% set version = "0.3.0" %} {% set sha256 = "REPLACE_WITH_RELEASE_TARBALL_SHA256" %} package: @@ -47,6 +47,7 @@ test: - dotmatch audit --help | grep 'safe_at_hamming_k3' - dotmatch assay --help | grep 'dotmatch assay' - dotmatch barcode --help | grep 'dotmatch barcode' + - dotmatch feature --help | grep 'cell-by-feature matrix' - dotmatch panel --help | grep 'dotmatch panel' - test -f "${PREFIX}/include/qdalign.h" - test -f "${PREFIX}/lib/libdotmatch.a" @@ -76,6 +77,9 @@ test: - printf '@r1\nNACGTAAAA\n+\nIIIIIIIII\n@r2\nNTTTTAAAA\n+\nIIIIIIIII\n' > barcode_reads.fastq - dotmatch barcode infer --barcodes barcodes.tsv --reads barcode_reads.fastq --scan-starts 0:2 --barcode-length 4 --sample-reads 10 --out offset_scan.tsv --summary barcode_summary.json - "grep '\"recommended_start\": 1' barcode_summary.json" + - printf 'observation_id\tcell_barcode\tfeature_seq\nfeature0\tcell0\tACGT\n' > feature_observations.tsv + - dotmatch feature matrix --observations feature_observations.tsv --targets targets.tsv --id-column observation_id --cell-column cell_barcode --sequence-column feature_seq --metric hamming --k 0 --out-dir feature_matrix_out + - test -f feature_matrix_out/matrix.mtx - dotmatch panel design --n 2 --length 4 --candidate-pool-size 100 --restarts 1 --min-hamming-distance 2 --min-levenshtein-distance 2 --out-dir panel_out - test -f panel_out/barcodes.tsv - test -f panel_out/design_report.json diff --git a/pyproject.toml b/pyproject.toml index 9bd282f0..847fa61c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dotmatch" -version = "0.2.2" +version = "0.3.0" description = "Known-target short-DNA assignment from FASTQ for CRISPR guides, barcodes, feature tags, primers, and other targets" readme = "README.md" requires-python = ">=3.9" @@ -71,6 +71,6 @@ dotmatch = "dotmatch.cli:main" quickdna = "dotmatch.cli:main" [tool.cibuildwheel] -build = "cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp39-musllinux_x86_64 cp310-musllinux_x86_64 cp311-musllinux_x86_64 cp312-musllinux_x86_64" +build = "cp39-manylinux_x86_64 cp39-manylinux_aarch64 cp310-manylinux_x86_64 cp310-manylinux_aarch64 cp311-manylinux_x86_64 cp311-manylinux_aarch64 cp312-manylinux_x86_64 cp312-manylinux_aarch64 cp39-musllinux_x86_64 cp39-musllinux_aarch64 cp310-musllinux_x86_64 cp310-musllinux_aarch64 cp311-musllinux_x86_64 cp311-musllinux_aarch64 cp312-musllinux_x86_64 cp312-musllinux_aarch64" skip = "pp*" -test-command = "python -c \"import dotmatch, quickdna; assert dotmatch.distance('ACGT', 'AGGT') == 1; assert quickdna.distance_leq('ACGT', 'AGGT', 1)\" && assaycode --version && dotmatch --version && dotmatch dist ACGT AGGT" +test-command = "python -c \"import dotmatch, quickdna; assert dotmatch.distance('ACGT', 'AGGT') == 1; assert quickdna.distance_leq('ACGT', 'AGGT', 1)\" && assaycode --version && dotmatch --version && test \"$(dotmatch dist ACGT AGGT)\" = 1 && test \"$(dotmatch leq 1 ACGT AGGT)\" = true" diff --git a/python/dotmatch/__init__.py b/python/dotmatch/__init__.py index bff71ff0..65a7a2cb 100644 --- a/python/dotmatch/__init__.py +++ b/python/dotmatch/__init__.py @@ -33,6 +33,7 @@ targets_from_dataframe, write_assignments_tsv, ) +from .feature_matrix import FeatureMatrixResult, build_feature_matrix # Advanced / optional integrations (import submodules to avoid heavy dep cost) # from . import anndata as anndata # if you have the extra @@ -52,7 +53,7 @@ def _source_tree_version() -> Optional[str]: try: __version__ = _source_tree_version() or _metadata_version("dotmatch") except PackageNotFoundError: - __version__ = "0.2.2" + __version__ = "0.3.0" __all__ = [ "__version__", @@ -84,5 +85,7 @@ def _source_tree_version() -> Optional[str]: "stream_assign", "targets_from_dataframe", "write_assignments_tsv", + "FeatureMatrixResult", + "build_feature_matrix", "tl", ] diff --git a/python/dotmatch/assayspec.py b/python/dotmatch/assayspec.py index d09d1de8..5d2361bc 100644 --- a/python/dotmatch/assayspec.py +++ b/python/dotmatch/assayspec.py @@ -315,7 +315,18 @@ def validate_assay_spec(assay: AssaySpec) -> None: else: _require_path(assay, "left_targets") _require_path(assay, "right_targets") - _require_path(assay, "reads") + has_single_reads = "reads" in data + has_left_reads = "left_reads" in data + has_right_reads = "right_reads" in data + if has_single_reads and (has_left_reads or has_right_reads): + raise AssaySpecError( + "pair-count must use reads or both left_reads and right_reads, not both input layouts" + ) + if has_single_reads: + _require_path(assay, "reads") + else: + _require_path(assay, "left_reads") + _require_path(assay, "right_reads") _require_extract(data, "left") _require_extract(data, "right") @@ -1649,10 +1660,11 @@ def _autopsy_pair(assay: AssaySpec, native: Path, out_dir: Path, findings: list[ ) _add_audit_findings(audit_dir / "audit_summary.json", findings, side) extract = _table(assay.data, extract_key) + reads_key = "reads" if "reads" in assay.data else f"{side}_reads" top = out_dir / f"top_unmatched.{side}.tsv" artifacts[f"top_unmatched_{side}"] = top cmd = [ - str(native), "inspect-unmatched", "--targets", str(_spec_path(assay, target_key)), "--reads", str(_spec_path(assay, "reads")), + str(native), "inspect-unmatched", "--targets", str(_spec_path(assay, target_key)), "--reads", str(_spec_path(assay, reads_key)), "--target-start", str(extract["start"]), "--target-length", str(extract["length"]), "--k", str(min(int(assignment.get("k", 1)), 1)), "--offset-window", "5", "--top", "100", "--out", str(top), ] @@ -2046,25 +2058,38 @@ def _compile_pair(assay: AssaySpec, steps: list[PlanStep], artifacts: dict[str, str(_spec_path(assay, "left_targets")), "--right-targets", str(_spec_path(assay, "right_targets")), - "--reads", - str(_spec_path(assay, "reads")), - "--left-start", - str(left["start"]), - "--left-length", - str(left["length"]), - "--right-start", - str(right["start"]), - "--right-length", - str(right["length"]), - "--k", - str(assignment.get("k", 1)), - "--metric", - str(assignment.get("metric", "levenshtein")), - "--out", - str(artifacts["pair_counts"]), - "--summary", - str(artifacts["pair_summary"]), ] + if "reads" in data: + cmd.extend(["--reads", str(_spec_path(assay, "reads"))]) + else: + cmd.extend( + [ + "--left-reads", + str(_spec_path(assay, "left_reads")), + "--right-reads", + str(_spec_path(assay, "right_reads")), + ] + ) + cmd.extend( + [ + "--left-start", + str(left["start"]), + "--left-length", + str(left["length"]), + "--right-start", + str(right["start"]), + "--right-length", + str(right["length"]), + "--k", + str(assignment.get("k", 1)), + "--metric", + str(assignment.get("metric", "levenshtein")), + "--out", + str(artifacts["pair_counts"]), + "--summary", + str(artifacts["pair_summary"]), + ] + ) cmd.extend(["--ambiguity-policy", str(assignment.get("ambiguity_policy", "radius"))]) if outputs.get("assignments"): artifacts["pair_assignments"] = out_dir / "pair_assignments.tsv" @@ -2250,6 +2275,11 @@ def _methods_sample_lines(assay: AssaySpec) -> list[str]: f"- Sample `{sample.get('id', '')}` FASTQ: `{sample.get('fastq', '')}`" for sample in _samples(assay.data) ] + if assay.mode == "pair-count" and "reads" not in assay.data: + return [ + f"- Left FASTQ: `{assay.data.get('left_reads', '')}`", + f"- Right FASTQ: `{assay.data.get('right_reads', '')}`", + ] return [f"- Reads: `{assay.data.get('reads', '')}`"] @@ -4033,12 +4063,19 @@ def _samples_table(assay: AssaySpec) -> str: html.escape(Path(str(sample.get("fastq", ""))).name), ) ) + elif assay.mode == "pair-count" and "reads" not in assay.data: + for side, key in [("left", "left_reads"), ("right", "right_reads")]: + rows.append( + "{}{}".format( + html.escape(side), + html.escape(Path(str(assay.data.get(key, ""))).name), + ) + ) else: - reads_key = "reads" rows.append( "{}{}".format( html.escape(assay.mode), - html.escape(Path(str(assay.data.get(reads_key, ""))).name), + html.escape(Path(str(assay.data.get("reads", ""))).name), ) ) rows.append("") @@ -4235,7 +4272,11 @@ def _template_text(template: str) -> str: assay_type = "generic" left_targets = "left_targets.tsv" right_targets = "right_targets.tsv" +# Use one reads input when both windows are present in the same read. reads = "reads.fastq.gz" +# Or use synchronized paired FASTQs and remove the reads line above: +# left_reads = "sample_R1.fastq.gz" +# right_reads = "sample_R2.fastq.gz" [run] out_dir = "dotmatch_pair_out" diff --git a/python/dotmatch/cli.py b/python/dotmatch/cli.py index 7867d015..90199f95 100644 --- a/python/dotmatch/cli.py +++ b/python/dotmatch/cli.py @@ -29,6 +29,7 @@ assign_posterior, distance, ) +from .feature_matrix import build_feature_matrix from .native import find_native_cli, run_native_cli @@ -484,6 +485,57 @@ def command_count(args: argparse.Namespace) -> int: return 0 +def command_feature_namespace(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="dotmatch feature", + description="Assign pre-extracted feature observations and write a sparse cell-by-feature matrix.", + ) + sub = parser.add_subparsers(dest="command", required=True) + matrix = sub.add_parser( + "matrix", + help="write a deterministic cell-by-feature matrix from an observation table", + description=( + "Input rows must already contain an explicit cell identifier and feature sequence. " + "This command does not perform FASTQ pairing, barcode correction, UMI deduplication, or cell calling." + ), + ) + matrix.add_argument("--observations", required=True, help="headered TSV/CSV with one pre-extracted observation per row") + matrix.add_argument("--targets", required=True, help="feature library TSV/CSV with target_id and target_seq") + matrix.add_argument("--cell-column", required=True, help="explicit input column containing cell identifiers") + matrix.add_argument("--sequence-column", required=True, help="input column containing feature sequence windows") + matrix.add_argument("--id-column", help="optional input column retained as observation_id in assignments.tsv") + matrix.add_argument("--k", type=int, default=1, help="maximum edit distance; exact requires 0") + matrix.add_argument("--metric", choices=["hamming", "levenshtein", "exact"], default="hamming") + matrix.add_argument("--ambiguity-policy", choices=["radius", "best"], default="radius") + matrix.add_argument("--batch-size", type=int, default=4096) + matrix.add_argument("--out-dir", required=True, help="new output directory for matrix and QC artifacts") + + args = parser.parse_args(list(argv)) + try: + if args.command == "matrix": + result = build_feature_matrix( + args.observations, + args.targets, + args.out_dir, + cell_column=args.cell_column, + sequence_column=args.sequence_column, + id_column=args.id_column, + k=args.k, + metric=args.metric, + ambiguity_policy=args.ambiguity_policy, + batch_size=args.batch_size, + ) + print(json.dumps(result.summary, indent=2, sort_keys=True)) + return 0 + except BrokenPipeError: + return 1 + except Exception as exc: + print(f"dotmatch feature: {exc}", file=sys.stderr) + return 2 + parser.error("unreachable") + return 2 + + def command_audit_targets(args: argparse.Namespace) -> int: targets = _read_targets(args.targets) pairs = list(_near_target_pairs(targets, args.k)) @@ -2374,6 +2426,8 @@ def print_top_level_help() -> None: Validate, plan, and run AssaySpec TOML workflows. barcode Infer barcode windows, audit barcode sets, demultiplex reads, and write autopsy reports. + feature + Build a cell-by-feature matrix from pre-extracted observations with explicit cell identifiers. panel Design, certify, simulate, lay out, and export barcode panels. crispr @@ -2408,6 +2462,8 @@ def print_top_level_help() -> None: --target-start 23 --target-length 20 --k 1 --metric hamming --out counts.tsv dotmatch assay check assay.toml dotmatch barcode infer --barcodes barcodes.tsv --reads pooled.fastq.gz --out offset_scan.tsv + dotmatch feature matrix --observations observations.tsv --targets features.tsv \\ + --cell-column cell_barcode --sequence-column feature_seq --out-dir feature_matrix dotmatch panel design --preset illumina-inline-96 --out-dir panel """ ) @@ -2453,6 +2509,7 @@ def print_manual() -> None: Workflow namespaces: assay validate, plan, and run AssaySpec TOML workflows barcode infer barcode windows, audit, demux, and write reports + feature build a cell-by-feature matrix from pre-extracted observations panel design, check, optimize, simulate, lay out, and export barcode panels crispr CRISPR guide-count project helpers and QC @@ -2487,6 +2544,12 @@ def print_manual() -> None: TOML workflow specification consumed by dotmatch assay. Start with dotmatch assay new, dotmatch crispr new, or an example under examples/. + Feature observations: + Headered TSV/CSV with one pre-extracted observation per row. Supply the + explicit cell and sequence column names to `dotmatch feature matrix`. + It does not perform FASTQ pairing, barcode correction, UMI deduplication, + or cell calling. + OUTPUTS count: Counts TSV, optional per-read assignments TSV, optional JSON summary. @@ -2499,6 +2562,11 @@ def print_manual() -> None: offset_scan.tsv, collision/safety audit outputs, demux outputs, top_unmatched.tsv, findings.tsv, provenance.json, report.md, report.html, and MultiQC custom content. + feature matrix: + matrix.mtx (cells x features), barcodes.tsv, features.tsv, + cell_feature_counts.tsv, assignments.tsv, cell_qc.tsv, and summary.json. + Only uniquely assigned observations contribute to the matrix. + COMMON RECIPES Count fixed-window CRISPR guides: dotmatch count --targets guides.tsv --reads sample.fastq.gz \\ @@ -2513,6 +2581,11 @@ def print_manual() -> None: dotmatch barcode autopsy --barcodes barcodes.tsv --reads pooled.fastq.gz \\ --scan-starts 0:30 --barcode-length auto --k-values 0,1 --out-dir barcode_report/ + Build a feature matrix from extracted observations: + dotmatch feature matrix --observations observations.tsv --targets features.tsv \\ + --cell-column cell_barcode --sequence-column feature_seq --metric hamming --k 1 \\ + --out-dir feature_matrix/ + Validate and run an assay workflow: dotmatch assay check assay.toml dotmatch assay start assay.toml @@ -2585,6 +2658,8 @@ def main(argv: Sequence[str] | None = None) -> int: return command_crispr_namespace(raw_args[1:]) if raw_args and raw_args[0] == "barcode": return command_barcode_namespace(raw_args[1:]) + if raw_args and raw_args[0] == "feature": + return command_feature_namespace(raw_args[1:]) if raw_args and raw_args[0] == "panel": from .panel import command_panel_namespace diff --git a/python/dotmatch/core.py b/python/dotmatch/core.py index 087673d1..3465dbd3 100644 --- a/python/dotmatch/core.py +++ b/python/dotmatch/core.py @@ -1168,6 +1168,7 @@ def assignments_to_anndata( *, cell_col: str = "cell_barcode", feature_col: str = "target_name", + status_col: str | None = None, count_unique_only: bool = True, include_ambiguous_per_cell: bool = False, ) -> Any: @@ -1180,7 +1181,12 @@ def assignments_to_anndata( By default (count_unique_only=True) only status==unique reads contribute to the count matrix. This preserves DotMatch's core scientific contract: ambiguous reads are never silently assigned. - If your assignments came from the CLI, the read_id often encodes cell info (parse or supply cell_col). + ``status_col`` defaults to ``status_name`` or ``status`` when either is + present. Text outcomes (``unique``/``ambiguous``) and native numeric status + values are both accepted. + + If your assignments came from the CLI, provide an explicit cell column from + the upstream workflow rather than inferring it from a read identifier. """ _ensure_anndata() _ensure_pandas() @@ -1208,13 +1214,21 @@ def assignments_to_anndata( else: feature_col = "target_index" # fallback - # Always compute per-cell stats for QC/accuracy visibility - if "status_name" in df.columns: - unique_mask = df["status_name"].isin(["unique"]) - ambig_mask = df["status_name"].isin(["ambiguous"]) - else: - unique_mask = df.get("status", 0) == 1 - ambig_mask = df.get("status", 0) == 2 + # Always compute per-cell stats for QC/accuracy visibility. The native API + # has historically emitted integer status values while table artifacts use + # the readable names, so accept both forms without silently dropping rows. + if status_col is None: + if "status_name" in df.columns: + status_col = "status_name" + elif "status" in df.columns: + status_col = "status" + else: + raise ValueError("Could not find assignment status column; pass status_col=") + if status_col not in df.columns: + raise ValueError(f"Could not find status column '{status_col}'") + statuses = df[status_col].astype(str).str.strip().str.lower() + unique_mask = statuses.isin(["unique", "1", "1.0"]) + ambig_mask = statuses.isin(["ambiguous", "2", "2.0"]) if count_unique_only: df_unique = df[unique_mask] diff --git a/python/dotmatch/feature_matrix.py b/python/dotmatch/feature_matrix.py new file mode 100644 index 00000000..4f193fcb --- /dev/null +++ b/python/dotmatch/feature_matrix.py @@ -0,0 +1,464 @@ +"""Build deterministic cell-by-feature matrices from pre-extracted observations. + +This module deliberately operates on a tabular observation stream rather than +FASTQ pairs. Each input row must already contain an explicit cell identifier +and a feature sequence window. It assigns the window to a known feature +library and counts only unique assignments. Cell calling, barcode correction, +UMI deduplication, and read-pair extraction are outside this command's scope. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import shutil +import tempfile +from collections import Counter, defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Iterator, Sequence, TextIO + +from .core import MATCH_AMBIGUOUS, MATCH_INVALID, MATCH_NONE, MATCH_UNIQUE, Matcher, MatchResult, load_targets, status_name + + +@dataclass(frozen=True) +class FeatureMatrixResult: + """Locations and summary data produced by :func:`build_feature_matrix`.""" + + output_dir: Path + summary: dict[str, Any] + + +@dataclass(frozen=True) +class _Observation: + observation_id: str + cell_barcode: str + sequence: str + + +def _open_text(path: str | Path, mode: str = "rt") -> TextIO: + source = Path(path) + if str(source).endswith(".gz"): + import gzip + + return gzip.open(source, mode, encoding="utf-8", newline="") + return source.open(mode, encoding="utf-8", newline="") + + +def _delimiter(path: str | Path) -> str: + name = Path(path).name.lower() + return "," if name.endswith(".csv") or name.endswith(".csv.gz") else "\t" + + +def _sha256(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _require_safe_field(value: str, *, field: str, row_number: int) -> str: + normalized = value.strip() + if not normalized: + raise ValueError(f"row {row_number} has an empty {field}") + if any(character in normalized for character in "\t\n\r"): + raise ValueError(f"row {row_number} has a tab or newline in {field}") + return normalized + + +def _read_observations( + path: str | Path, + *, + cell_column: str, + sequence_column: str, + id_column: str | None, + content_digest: Any | None = None, +) -> Iterator[_Observation]: + source = Path(path) + with _open_text(source) as fh: + def lines() -> Iterator[str]: + for line in fh: + if content_digest is not None: + content_digest.update(line.encode("utf-8")) + yield line + + reader = csv.DictReader(lines(), delimiter=_delimiter(source)) + if not reader.fieldnames: + raise ValueError(f"observation table has no header: {source}") + fieldnames = { + field.strip(): field + for field in reader.fieldnames + if field is not None and field.strip() + } + for field in (cell_column, sequence_column): + if field not in fieldnames: + raise ValueError(f"observation table is missing required column '{field}': {source}") + if id_column is not None and id_column not in fieldnames: + raise ValueError(f"observation table is missing requested id column '{id_column}': {source}") + cell_key = fieldnames[cell_column] + sequence_key = fieldnames[sequence_column] + id_key = fieldnames[id_column] if id_column is not None else None + + for row_number, row in enumerate(reader, start=2): + cell = _require_safe_field(row.get(cell_key, "") or "", field=cell_column, row_number=row_number) + sequence = (row.get(sequence_key, "") or "").strip().upper() + if any(character in sequence for character in "\t\n\r"): + raise ValueError(f"row {row_number} has a tab or newline in {sequence_column}") + if id_column is None: + observation_id = f"row_{row_number - 1}" + else: + observation_id = _require_safe_field( + row.get(id_key or id_column, "") or "", + field=id_column, + row_number=row_number, + ) + yield _Observation(observation_id=observation_id, cell_barcode=cell, sequence=sequence) + + +def _chunks(items: Iterable[_Observation], size: int) -> Iterator[list[_Observation]]: + if size <= 0: + raise ValueError("batch_size must be positive") + batch: list[_Observation] = [] + for item in items: + batch.append(item) + if len(batch) == size: + yield batch + batch = [] + if batch: + yield batch + + +def _assign( + matcher: Matcher, + sequences: Sequence[str], + *, + metric: str, + k: int, + ambiguity_policy: str, +) -> list[MatchResult]: + if metric == "hamming": + return matcher.assign_hamming(sequences, k=k, policy=ambiguity_policy) + if metric == "exact": + return matcher.assign_exact(sequences, policy=ambiguity_policy) + return matcher.assign(sequences, k=k, policy=ambiguity_policy) + + +def _write_tsv(path: Path, header: Sequence[str], rows: Iterable[Sequence[object]]) -> None: + with path.open("w", encoding="utf-8", newline="") as fh: + writer = csv.writer(fh, delimiter="\t", lineterminator="\n") + writer.writerow(header) + writer.writerows(rows) + + +def _write_matrix_market(path: Path, *, counts: Counter[tuple[str, str]], cells: Sequence[str], features: Sequence[str]) -> None: + cell_index = {cell: index + 1 for index, cell in enumerate(cells)} + feature_index = {feature: index + 1 for index, feature in enumerate(features)} + ordered_counts = sorted( + ((cell_index[cell], feature_index[feature], count) for (cell, feature), count in counts.items()), + key=lambda row: (row[0], row[1]), + ) + with path.open("w", encoding="utf-8", newline="") as fh: + fh.write("%%MatrixMarket matrix coordinate integer general\n") + fh.write("% DotMatch cell-by-feature unique-assignment counts\n") + fh.write(f"{len(cells)} {len(features)} {len(ordered_counts)}\n") + for row, column, count in ordered_counts: + fh.write(f"{row} {column} {count}\n") + + +def _new_qc() -> dict[str, int]: + return { + "total_observations": 0, + "assigned_unique": 0, + "ambiguous": 0, + "unmatched": 0, + "invalid": 0, + } + + +def _increment_status(qc: dict[str, int], status: int) -> None: + qc["total_observations"] += 1 + if status == MATCH_UNIQUE: + qc["assigned_unique"] += 1 + elif status == MATCH_AMBIGUOUS: + qc["ambiguous"] += 1 + elif status == MATCH_NONE: + qc["unmatched"] += 1 + else: + qc["invalid"] += 1 + + +def _validate_targets(targets: Sequence[tuple[str, str]]) -> list[tuple[str, str]]: + seen: set[str] = set() + normalized: list[tuple[str, str]] = [] + for target_id, sequence in targets: + target_id = _require_safe_field(target_id, field="target_id", row_number=len(normalized) + 1) + if target_id in seen: + raise ValueError(f"target library contains duplicate target_id '{target_id}'") + seen.add(target_id) + normalized.append((target_id, sequence.upper())) + return normalized + + +def build_feature_matrix( + observations: str | Path, + targets: str | Path, + output_dir: str | Path, + *, + cell_column: str, + sequence_column: str, + id_column: str | None = None, + k: int = 1, + metric: str = "hamming", + ambiguity_policy: str = "radius", + batch_size: int = 4096, +) -> FeatureMatrixResult: + """Assign a pre-extracted observation table and write a sparse cell matrix. + + ``observations`` must be a headered TSV (or CSV) with an explicit cell + column and sequence column. The resulting Matrix Market matrix has cells + on rows and features on columns. Only ``unique`` assignments add a count; + every outcome is retained in ``assignments.tsv`` and ``cell_qc.tsv``. + + The output directory must not already exist. This avoids mixing artifacts + from different assignments in one run directory. + """ + if metric not in {"hamming", "levenshtein", "exact"}: + raise ValueError("metric must be 'hamming', 'levenshtein', or 'exact'") + if k < 0: + raise ValueError("k must be non-negative") + if metric == "exact" and k != 0: + raise ValueError("metric='exact' requires k=0") + if metric == "hamming" and k > 3: + raise ValueError("hamming assignment supports k between 0 and 3") + if ambiguity_policy not in {"radius", "best"}: + raise ValueError("ambiguity_policy must be 'radius' or 'best'") + if batch_size <= 0: + raise ValueError("batch_size must be positive") + + observation_path = Path(observations) + target_path = Path(targets) + final_dir = Path(output_dir) + if not observation_path.is_file(): + raise ValueError(f"observation table does not exist: {observation_path}") + if not target_path.is_file(): + raise ValueError(f"target library does not exist: {target_path}") + if final_dir.exists(): + raise ValueError(f"output directory already exists: {final_dir}") + + normalized_targets = _validate_targets(load_targets(target_path)) + target_ids = [target_id for target_id, _sequence in normalized_targets] + target_sequences = [sequence for _target_id, sequence in normalized_targets] + if metric == "hamming" and len({len(sequence) for sequence in target_sequences}) != 1: + raise ValueError("hamming feature assignment requires target sequences with one shared length") + target_by_index = dict(enumerate(normalized_targets)) + observation_digest = hashlib.sha256() + + final_dir.parent.mkdir(parents=True, exist_ok=True) + staging_dir = Path(tempfile.mkdtemp(prefix=f".{final_dir.name}.tmp-", dir=final_dir.parent)) + assignments_path = staging_dir / "assignments.tsv" + counts: Counter[tuple[str, str]] = Counter() + qc_by_cell: defaultdict[str, dict[str, int]] = defaultdict(_new_qc) + summary = { + "schema_version": 1, + "workflow": "feature_matrix", + "matrix_orientation": "cells_by_features", + "metric": metric, + "k": k, + "ambiguity_policy": ambiguity_policy, + "cell_column": cell_column, + "sequence_column": sequence_column, + "id_column": id_column, + "inputs": { + "observations": str(observation_path), + "observations_content_sha256": "", + "targets": str(target_path), + "targets_sha256": _sha256(target_path), + }, + "scope": { + "input": "pre-extracted observations with explicit cell identifiers", + "cell_calling": "not_performed", + "barcode_correction": "not_performed", + "umi_deduplication": "not_performed", + "paired_read_extraction": "not_performed", + }, + "total_observations": 0, + "valid_observations": 0, + "assigned_unique": 0, + "assigned_exact": 0, + "assigned_corrected": 0, + "ambiguous": 0, + "unmatched": 0, + "invalid": 0, + "features": len(target_ids), + } + + try: + with assignments_path.open("w", encoding="utf-8", newline="") as assignment_fh: + writer = csv.writer(assignment_fh, delimiter="\t", lineterminator="\n") + writer.writerow( + [ + "observation_id", + "cell_barcode", + "observed_seq", + "target_id", + "target_seq", + "distance", + "status", + "match_count", + "second_best_distance", + ] + ) + with Matcher(target_sequences) as matcher: + for batch in _chunks( + _read_observations( + observation_path, + cell_column=cell_column, + sequence_column=sequence_column, + id_column=id_column, + content_digest=observation_digest, + ), + batch_size, + ): + valid_positions = [index for index, observation in enumerate(batch) if observation.sequence] + valid_sequences = [batch[index].sequence for index in valid_positions] + results_by_position: dict[int, MatchResult] = { + index: MatchResult(-1, -1, -1, 0, MATCH_INVALID) + for index, observation in enumerate(batch) + if not observation.sequence + } + if valid_sequences: + results_by_position.update( + zip( + valid_positions, + _assign( + matcher, + valid_sequences, + metric=metric, + k=k, + ambiguity_policy=ambiguity_policy, + ), + ) + ) + + for index, observation in enumerate(batch): + result = results_by_position[index] + status = result.status + target_id = "" + target_sequence = "" + if status == MATCH_UNIQUE and 0 <= result.target_index < len(normalized_targets): + target_id, target_sequence = target_by_index[result.target_index] + summary["total_observations"] += 1 + _increment_status(qc_by_cell[observation.cell_barcode], status) + if status == MATCH_UNIQUE and target_id: + counts[(observation.cell_barcode, target_id)] += 1 + summary["assigned_unique"] += 1 + if result.best_distance == 0: + summary["assigned_exact"] += 1 + else: + summary["assigned_corrected"] += 1 + elif status == MATCH_AMBIGUOUS: + summary["ambiguous"] += 1 + elif status == MATCH_NONE: + summary["unmatched"] += 1 + else: + summary["invalid"] += 1 + writer.writerow( + [ + observation.observation_id, + observation.cell_barcode, + observation.sequence, + target_id, + target_sequence, + result.best_distance, + status_name(status), + result.match_count, + result.second_best_distance, + ] + ) + + cells = sorted(qc_by_cell) + features = sorted(target_ids) + target_sequence_by_id = dict(normalized_targets) + cell_feature_sets: defaultdict[str, set[str]] = defaultdict(set) + for cell, feature in counts: + cell_feature_sets[cell].add(feature) + + summary["valid_observations"] = summary["total_observations"] - summary["invalid"] + summary["inputs"]["observations_content_sha256"] = observation_digest.hexdigest() + summary["cells"] = len(cells) + summary["nonzero_entries"] = len(counts) + summary["assignment_rate"] = ( + summary["assigned_unique"] / summary["valid_observations"] + if summary["valid_observations"] + else 0.0 + ) + summary["artifacts"] = [ + "assignments.tsv", + "barcodes.tsv", + "cell_feature_counts.tsv", + "cell_qc.tsv", + "features.tsv", + "matrix.mtx", + "summary.json", + ] + + _write_matrix_market(staging_dir / "matrix.mtx", counts=counts, cells=cells, features=features) + _write_tsv(staging_dir / "barcodes.tsv", ["cell_barcode"], ((cell,) for cell in cells)) + _write_tsv( + staging_dir / "features.tsv", + ["target_id", "target_seq"], + ((feature, target_sequence_by_id[feature]) for feature in features), + ) + _write_tsv( + staging_dir / "cell_feature_counts.tsv", + ["cell_barcode", "target_id", "count"], + ( + (cell, feature, count) + for (cell, feature), count in sorted(counts.items(), key=lambda item: (item[0][0], item[0][1])) + ), + ) + _write_tsv( + staging_dir / "cell_qc.tsv", + [ + "cell_barcode", + "total_observations", + "valid_observations", + "assigned_unique", + "ambiguous", + "unmatched", + "invalid", + "unique_features", + "assignment_rate", + ], + ( + ( + cell, + qc_by_cell[cell]["total_observations"], + qc_by_cell[cell]["total_observations"] - qc_by_cell[cell]["invalid"], + qc_by_cell[cell]["assigned_unique"], + qc_by_cell[cell]["ambiguous"], + qc_by_cell[cell]["unmatched"], + qc_by_cell[cell]["invalid"], + len(cell_feature_sets[cell]), + ( + qc_by_cell[cell]["assigned_unique"] + / (qc_by_cell[cell]["total_observations"] - qc_by_cell[cell]["invalid"]) + if qc_by_cell[cell]["total_observations"] - qc_by_cell[cell]["invalid"] + else 0.0 + ), + ) + for cell in cells + ), + ) + with (staging_dir / "summary.json").open("w", encoding="utf-8", newline="") as summary_fh: + json.dump(summary, summary_fh, indent=2, sort_keys=True) + summary_fh.write("\n") + os.replace(staging_dir, final_dir) + except Exception: + shutil.rmtree(staging_dir, ignore_errors=True) + raise + + return FeatureMatrixResult(output_dir=final_dir, summary=summary) diff --git a/python/dotmatch/tl.py b/python/dotmatch/tl.py index 17a1145a..9cc688db 100644 --- a/python/dotmatch/tl.py +++ b/python/dotmatch/tl.py @@ -35,6 +35,7 @@ from __future__ import annotations +from collections.abc import Mapping from pathlib import Path from typing import Any, Sequence @@ -66,7 +67,7 @@ def _ensure_anndata() -> None: def _load_library(library: Any) -> list[tuple[str, str]]: - """Accept path, DataFrame, list of tuples, or list of seqs (auto ids).""" + """Accept a path, DataFrame, mappings, tuples, or sequences (auto ids).""" if isinstance(library, (str, Path)): # Assume tsv/csv with id,seq or just seqs try: @@ -84,6 +85,31 @@ def _load_library(library: Any) -> list[tuple[str, str]]: if isinstance(library, (list, tuple)): if library and isinstance(library[0], (list, tuple)) and len(library[0]) == 2: return list(library) + if library and isinstance(library[0], Mapping): + normalized: list[tuple[str, str]] = [] + for index, item in enumerate(library): + if not isinstance(item, Mapping): + raise TypeError("library mappings cannot be mixed with other entry types") + target_id = next( + ( + item[key] + for key in ("target_id", "id", "guide_id", "feature_id", "name") + if key in item and str(item[key]).strip() + ), + f"target_{index}", + ) + sequence = next( + ( + item[key] + for key in ("target_seq", "sequence", "seq", "guide_seq", "feature_seq") + if key in item and str(item[key]).strip() + ), + None, + ) + if sequence is None: + raise ValueError(f"library mapping at position {index} has no sequence field") + normalized.append((str(target_id), str(sequence))) + return normalized else: return [(f"target_{i}", str(s)) for i, s in enumerate(library)] raise TypeError("library must be path, DataFrame, or list of (id,seq) / seqs") diff --git a/python/tests/test_assaycode_bioconda.py b/python/tests/test_assaycode_bioconda.py index 4e7742f5..1018897b 100644 --- a/python/tests/test_assaycode_bioconda.py +++ b/python/tests/test_assaycode_bioconda.py @@ -6,7 +6,7 @@ import pytest from scripts.check_assaycode_bioconda_recipe import audit -from scripts.prepare_bioconda_handoff import PLACEHOLDER, render +from scripts.prepare_bioconda_handoff import PLACEHOLDER, _project_version, render def test_assaycode_metapackage_contract() -> None: @@ -14,7 +14,7 @@ def test_assaycode_metapackage_contract() -> None: def test_handoff_renders_both_recipes_with_real_checksum(tmp_path: Path) -> None: - archive = tmp_path / "v0.2.2.tar.gz" + archive = tmp_path / f"v{_project_version()}.tar.gz" archive.write_bytes(b"immutable release fixture") dotmatch_dir, assaycode_dir, digest = render(archive, tmp_path / "handoff") @@ -33,5 +33,6 @@ def test_handoff_renders_both_recipes_with_real_checksum(tmp_path: Path) -> None def test_handoff_rejects_archive_for_another_version(tmp_path: Path) -> None: archive = tmp_path / "v9.9.9.tar.gz" archive.write_bytes(b"wrong release") - with pytest.raises(ValueError, match="must identify 0.2.2"): + with pytest.raises(ValueError) as exc_info: render(archive, tmp_path / "handoff") + assert f"must identify {_project_version()}" in str(exc_info.value) diff --git a/python/tests/test_assayspec.py b/python/tests/test_assayspec.py index 856e3bdc..39990dc8 100644 --- a/python/tests/test_assayspec.py +++ b/python/tests/test_assayspec.py @@ -145,6 +145,57 @@ def _write_pair_spec(tmp_path: Path) -> Path: return spec +def _write_paired_pair_spec(tmp_path: Path) -> Path: + left = tmp_path / "paired_left.tsv" + right = tmp_path / "paired_right.tsv" + left_reads = tmp_path / "pair_R1.fastq" + right_reads = tmp_path / "pair_R2.fastq" + left.write_text("L0\tACGT\nL1\tTTTT\n", encoding="utf-8") + right.write_text("R0\tGGAA\nR1\tCCCC\n", encoding="utf-8") + left_reads.write_text( + "@p0/1\nACGT\n+\nIIII\n" + "@p1 1:N:0:1\nTTTT\n+\nIIII\n", + encoding="utf-8", + ) + right_reads.write_text( + "@p0/2\nGGAA\n+\nIIII\n" + "@p1 2:N:0:1\nCCCC\n+\nIIII\n", + encoding="utf-8", + ) + spec = tmp_path / "paired_pair.toml" + spec.write_text( + f""" +schema_version = 1 +mode = "pair-count" +assay_type = "generic" +left_targets = "{left}" +right_targets = "{right}" +left_reads = "{left_reads}" +right_reads = "{right_reads}" + +[run] +out_dir = "{tmp_path / 'paired_pair_out'}" + +[left] +start = 0 +length = 4 + +[right] +start = 0 +length = 4 + +[assignment] +k = 1 +metric = "hamming" + +[outputs] +assignments = true +""".lstrip(), + encoding="utf-8", + ) + return spec + + def _write_inference_targets(tmp_path: Path) -> Path: targets = tmp_path / "targets.tsv" targets.write_text("guide_a\tACGT\tGENEA\nguide_b\tTTTT\tGENEB\n", encoding="utf-8") @@ -574,7 +625,7 @@ def test_assay_run_count_reproduces_existing_crispr_fixture(tmp_path: Path) -> N assert "ambiguous reads were not silently counted" in methods citation = (out_dir / "CITATION.bib").read_text(encoding="utf-8") assert "@software{dotmatch" in citation - assert "doi = {10.5281/zenodo.21511337}" in citation + assert "doi = {10.5281/zenodo.20541628}" in citation versions = (out_dir / "software_versions.yml").read_text(encoding="utf-8") assert "dotmatch_python:" in versions assert "dotmatch_native:" in versions @@ -661,10 +712,12 @@ def test_assay_run_demux_and_pair_count_specs(tmp_path: Path) -> None: subprocess.run(["make", "dotmatch"], cwd=ROOT, check=True) demux_spec = _write_demux_spec(tmp_path) pair_spec = _write_pair_spec(tmp_path) + paired_pair_spec = _write_paired_pair_spec(tmp_path) env = {"DOTMATCH_NATIVE_CLI": str(ROOT / "dotmatch")} demux = _run_cli(["assay", "run", str(demux_spec)], env=env) pair = _run_cli(["assay", "run", str(pair_spec)], env=env) + paired_pair = _run_cli(["assay", "run", str(paired_pair_spec)], env=env) assert demux.returncode == 0, demux.stderr assert (tmp_path / "demux_out" / "demuxed" / "bc0.fastq").exists() @@ -675,6 +728,34 @@ def test_assay_run_demux_and_pair_count_specs(tmp_path: Path) -> None: pair_reliability = json.loads((tmp_path / "pair_out" / "reliability_summary.json").read_text(encoding="utf-8")) assert pair_reliability["evidence_boundary"]["status"] == "smoke" assert any(finding["finding_id"] == "evidence_boundary_not_supported" for finding in pair_reliability["findings"]) + assert paired_pair.returncode == 0, paired_pair.stderr + paired_out = tmp_path / "paired_pair_out" + assert "L0\tR0\t1" in (paired_out / "pair_counts.tsv").read_text(encoding="utf-8") + assert "L1\tR1\t1" in (paired_out / "pair_counts.tsv").read_text(encoding="utf-8") + paired_summary = json.loads((paired_out / "pair_summary.json").read_text(encoding="utf-8")) + assert paired_summary["input_mode"] == "paired-fastq" + assert paired_summary["input_sync"] == "canonical-read-id" + assert paired_summary["total_pairs"] == 2 + assert (paired_out / "pair_assignments.tsv").read_text(encoding="utf-8").splitlines()[1].startswith("p0\t") + methods = (paired_out / "methods.md").read_text(encoding="utf-8") + assert "Left FASTQ" in methods + assert "Right FASTQ" in methods + + +def test_pair_assayspec_rejects_mixed_fastq_layouts(tmp_path: Path) -> None: + from dotmatch.assayspec import AssaySpecError, load_assay_spec + + spec = _write_paired_pair_spec(tmp_path) + spec.write_text( + spec.read_text(encoding="utf-8").replace( + f'left_reads = "{tmp_path / "pair_R1.fastq"}"', + f'reads = "{tmp_path / "pair_R1.fastq"}"\nleft_reads = "{tmp_path / "pair_R1.fastq"}"', + ), + encoding="utf-8", + ) + + with pytest.raises(AssaySpecError, match="must use reads or both left_reads and right_reads"): + load_assay_spec(spec) def test_demux_gpu_metadata_requires_public_gpu_gate(tmp_path: Path) -> None: diff --git a/python/tests/test_check_bioconda_recipe.py b/python/tests/test_check_bioconda_recipe.py index 2aa55d81..8f6566e9 100644 --- a/python/tests/test_check_bioconda_recipe.py +++ b/python/tests/test_check_bioconda_recipe.py @@ -60,6 +60,7 @@ def _meta(version: str = "0.1.0") -> str: " - dotmatch audit --help | grep 'safe_at_hamming_k3'\n" " - dotmatch assay --help | grep 'dotmatch assay'\n" " - dotmatch barcode --help | grep 'dotmatch barcode'\n" + " - dotmatch feature --help | grep 'cell-by-feature matrix'\n" " - dotmatch panel --help | grep 'dotmatch panel'\n" " - test -f \"${PREFIX}/include/qdalign.h\"\n" " - test -f \"${PREFIX}/lib/libdotmatch.a\"\n" @@ -89,6 +90,9 @@ def _meta(version: str = "0.1.0") -> str: " - printf '@r1\\nNACGTAAAA\\n+\\nIIIIIIIII\\n@r2\\nNTTTTAAAA\\n+\\nIIIIIIIII\\n' > barcode_reads.fastq\n" " - dotmatch barcode infer --barcodes barcodes.tsv --reads barcode_reads.fastq --scan-starts 0:2 --barcode-length 4 --sample-reads 10 --out offset_scan.tsv --summary barcode_summary.json\n" " - \"grep '\\\"recommended_start\\\": 1' barcode_summary.json\"\n" + " - printf 'observation_id\\tcell_barcode\\tfeature_seq\\nfeature0\\tcell0\\tACGT\\n' > feature_observations.tsv\n" + " - dotmatch feature matrix --observations feature_observations.tsv --targets targets.tsv --id-column observation_id --cell-column cell_barcode --sequence-column feature_seq --metric hamming --k 0 --out-dir feature_matrix_out\n" + " - test -f feature_matrix_out/matrix.mtx\n" " - dotmatch panel design --n 2 --length 4 --candidate-pool-size 100 --restarts 1 --min-hamming-distance 2 --min-levenshtein-distance 2 --out-dir panel_out\n" " - test -f panel_out/barcodes.tsv\n" " - test -f panel_out/design_report.json\n" diff --git a/python/tests/test_check_distribution_channels.py b/python/tests/test_check_distribution_channels.py index 0fcff000..627e6b3c 100644 --- a/python/tests/test_check_distribution_channels.py +++ b/python/tests/test_check_distribution_channels.py @@ -27,6 +27,15 @@ def _write_repo(root: Path, doi: str = "10.5281/zenodo.1234567") -> None: full.write_text(text, encoding="utf-8") +def _write_distribution_record(root: Path, channels: list[dict]) -> None: + path = root / "docs" / "distribution-release.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"schema_version": 1, "release_version": "0.1.0", "channels": channels}), + encoding="utf-8", + ) + + def test_distribution_channels_accepts_mocked_public_release(tmp_path, monkeypatch): checker = _load_checker() _write_repo(tmp_path) @@ -280,6 +289,112 @@ def fake_fetch_json(url: str): assert any("must not include raw linux_x86_64 wheels" in failure.message for failure in result.failures) +def test_distribution_channels_requires_recorded_arm64_pypi_wheels(tmp_path, monkeypatch): + checker = _load_checker() + _write_repo(tmp_path) + _write_distribution_record( + tmp_path, + [{"id": "pypi", "linux_wheel_architectures": ["x86_64", "aarch64"]}], + ) + + monkeypatch.setattr( + checker, + "fetch_json", + lambda url: { + "info": {"version": "0.1.0"}, + "urls": [ + {"packagetype": "sdist", "filename": "dotmatch-0.1.0.tar.gz"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-macosx_11_0_universal2.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-manylinux_2_28_x86_64.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-musllinux_1_2_x86_64.whl"}, + ], + }, + ) + + result = checker.AuditResult() + checker.check_pypi(tmp_path, "0.1.0", result) + + assert any("manylinux_aarch64" in failure.message for failure in result.failures) + assert any("musllinux_aarch64" in failure.message for failure in result.failures) + + +def test_distribution_channels_rejects_raw_arm64_pypi_wheel(tmp_path, monkeypatch): + checker = _load_checker() + _write_repo(tmp_path) + _write_distribution_record( + tmp_path, + [{"id": "pypi", "linux_wheel_architectures": ["x86_64", "aarch64"]}], + ) + + monkeypatch.setattr( + checker, + "fetch_json", + lambda url: { + "info": {"version": "0.1.0"}, + "urls": [ + {"packagetype": "sdist", "filename": "dotmatch-0.1.0.tar.gz"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-macosx_11_0_universal2.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-manylinux_2_28_x86_64.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-manylinux_2_28_aarch64.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-musllinux_1_2_x86_64.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-musllinux_1_2_aarch64.whl"}, + {"packagetype": "bdist_wheel", "filename": "dotmatch-0.1.0-py3-none-linux_aarch64.whl"}, + ], + }, + ) + + result = checker.AuditResult() + checker.check_pypi(tmp_path, "0.1.0", result) + + assert any("must not include raw linux_aarch64 wheels" in failure.message for failure in result.failures) + + +def test_ghcr_manifest_requires_every_recorded_platform(monkeypatch): + checker = _load_checker() + manifest = { + "schemaVersion": 2, + "manifests": [ + {"platform": {"os": "linux", "architecture": "amd64"}}, + {"platform": {"os": "unknown", "architecture": "unknown"}}, + ], + } + monkeypatch.setattr(checker, "fetch_registry_manifest", lambda image: (manifest, "sha256:" + "0" * 64)) + + try: + checker.verify_ghcr_manifest("ghcr.io/dnncha/dotmatch:v0.1.0", ("linux/amd64", "linux/arm64")) + except RuntimeError as exc: + assert "linux/arm64" in str(exc) + else: + raise AssertionError("expected missing linux/arm64 GHCR manifest descriptor to fail") + + +def test_distribution_channels_uses_recorded_ghcr_platforms(tmp_path, monkeypatch): + checker = _load_checker() + _write_repo(tmp_path) + _write_distribution_record( + tmp_path, + [{"id": "ghcr", "platforms": ["linux/amd64", "linux/arm64"]}], + ) + observed: dict[str, object] = {} + + def fake_verify(image: str, required_platforms=checker.DEFAULT_GHCR_PLATFORMS) -> str: + observed["image"] = image + observed["platforms"] = required_platforms + return "sha256:" + "0" * 64 + + monkeypatch.setattr(checker, "verify_ghcr_manifest", fake_verify) + monkeypatch.setattr(checker, "verify_ghcr_run", lambda image, version: None) + + result = checker.AuditResult() + checker.check_ghcr(tmp_path, "0.1.0", result) + + assert result.failures == [] + assert observed == { + "image": "ghcr.io/dnncha/dotmatch:v0.1.0", + "platforms": ("linux/amd64", "linux/arm64"), + } + + def test_distribution_channels_reports_bioconda_missing_version(tmp_path, monkeypatch): checker = _load_checker() _write_repo(tmp_path) diff --git a/python/tests/test_check_distribution_record.py b/python/tests/test_check_distribution_record.py index a727d5fa..2bba0b50 100644 --- a/python/tests/test_check_distribution_record.py +++ b/python/tests/test_check_distribution_record.py @@ -34,6 +34,9 @@ def _channel(channel_id: str, **overrides) -> dict: if channel_id == "pypi": item["blocker"] = "The source distribution and repaired manylinux/musllinux wheels are not public yet." item["next_action"] = "Publish the source distribution and repaired Linux wheels, then rerun make distribution-channels." + item["linux_wheel_architectures"] = ["x86_64"] + if channel_id == "ghcr": + item["platforms"] = ["linux/amd64"] item.update(overrides) return item @@ -296,3 +299,45 @@ def test_distribution_record_rejects_invalid_verified_date(tmp_path): result = checker.audit(tmp_path) assert any("pypi must declare verified_date as YYYY-MM-DD" in failure for failure in result.failures) + + +def test_distribution_record_requires_declared_verified_linux_architectures(tmp_path): + checker = _load_checker() + manifest = _manifest(status="partially_verified") + manifest["channels"][0].update( + { + "status": "verified", + "public_url": "https://pypi.org/project/dotmatch/0.1.0/", + "evidence_url": "https://github.com/dnncha/dotmatch/actions/runs/123456", + "verified_date": "2026-07-23", + "blocker": "", + "linux_wheel_architectures": ["arm64"], + } + ) + _write_repo(tmp_path, manifest) + + result = checker.audit(tmp_path) + + assert any("pypi linux_wheel_architectures contains unsupported value(s): arm64" in failure for failure in result.failures) + assert any("pypi linux_wheel_architectures must include x86_64" in failure for failure in result.failures) + + +def test_distribution_record_requires_declared_verified_ghcr_platforms(tmp_path): + checker = _load_checker() + manifest = _manifest(status="partially_verified") + ghcr = next(channel for channel in manifest["channels"] if channel["id"] == "ghcr") + ghcr.update( + { + "status": "verified", + "public_url": "https://github.com/dnncha/dotmatch/pkgs/container/dotmatch", + "evidence_url": "https://github.com/dnncha/dotmatch/actions/runs/123456", + "verified_date": "2026-07-23", + "blocker": "", + "platforms": ["linux/arm64"], + } + ) + _write_repo(tmp_path, manifest) + + result = checker.audit(tmp_path) + + assert any("ghcr platforms must include linux/amd64" in failure for failure in result.failures) diff --git a/python/tests/test_check_oci_manifest.py b/python/tests/test_check_oci_manifest.py new file mode 100644 index 00000000..abfeeb4f --- /dev/null +++ b/python/tests/test_check_oci_manifest.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "scripts" / "check_oci_manifest.py" + + +def _load_checker(): + spec = importlib.util.spec_from_file_location("check_oci_manifest", CHECKER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _manifest(*platforms: str) -> dict[str, object]: + descriptors: list[dict[str, object]] = [] + for platform in platforms: + operating_system, architecture = platform.split("/", 1) + descriptors.append( + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": f"sha256:{architecture}", + "platform": {"os": operating_system, "architecture": architecture}, + } + ) + return {"schemaVersion": 2, "manifests": descriptors} + + +def test_manifest_checker_accepts_required_linux_platforms() -> None: + checker = _load_checker() + manifest = _manifest("linux/amd64", "linux/arm64", "unknown/unknown") + + platforms = checker.check_manifest(manifest, ["linux/amd64", "linux/arm64"]) + + assert platforms == {"linux/amd64", "linux/arm64", "unknown/unknown"} + + +def test_manifest_checker_rejects_missing_required_platform() -> None: + checker = _load_checker() + + try: + checker.check_manifest(_manifest("linux/amd64"), ["linux/amd64", "linux/arm64"]) + except ValueError as exc: + assert "linux/arm64" in str(exc) + else: + raise AssertionError("expected missing linux/arm64 platform to fail") + + +def test_manifest_checker_requires_an_image_index_or_manifest_list() -> None: + checker = _load_checker() + + try: + checker.check_manifest({"schemaVersion": 2, "config": {}}, ["linux/amd64"]) + except ValueError as exc: + assert "image index or manifest list" in str(exc) + else: + raise AssertionError("expected single-image manifest to fail") diff --git a/python/tests/test_check_python_wheel_metadata.py b/python/tests/test_check_python_wheel_metadata.py index 4a4f8ad3..6f61ea42 100644 --- a/python/tests/test_check_python_wheel_metadata.py +++ b/python/tests/test_check_python_wheel_metadata.py @@ -171,6 +171,46 @@ def test_existing_wheel_clean_install_skips_unsupported_linux_libc_tags(monkeypa checker = _load_checker() monkeypatch.setattr(checker.platform, "system", lambda: "Linux") monkeypatch.setattr(checker.platform, "libc_ver", lambda: ("glibc", "2.39")) + monkeypatch.setattr(checker.platform, "machine", lambda: "x86_64") assert checker.wheel_supported_by_current_platform(Path("dotmatch-0.1.0-py3-none-manylinux_2_28_x86_64.whl")) assert not checker.wheel_supported_by_current_platform(Path("dotmatch-0.1.0-py3-none-musllinux_1_2_x86_64.whl")) + + +def test_existing_wheel_clean_install_skips_non_native_linux_architecture(monkeypatch): + checker = _load_checker() + monkeypatch.setattr(checker.platform, "system", lambda: "Linux") + monkeypatch.setattr(checker.platform, "libc_ver", lambda: ("glibc", "2.39")) + monkeypatch.setattr(checker.platform, "machine", lambda: "x86_64") + + assert not checker.wheel_supported_by_current_platform( + Path("dotmatch-0.1.0-py3-none-manylinux_2_28_aarch64.whl") + ) + + +def test_repaired_linux_wheel_coverage_requires_each_family_and_architecture() -> None: + checker = _load_checker() + wheels = [ + Path("dotmatch-0.1.0-py3-none-manylinux_2_28_x86_64.manylinux2014_x86_64.whl"), + Path("dotmatch-0.1.0-py3-none-manylinux_2_28_aarch64.whl"), + Path("dotmatch-0.1.0-py3-none-musllinux_1_2_x86_64.whl"), + Path("dotmatch-0.1.0-py3-none-musllinux_1_2_aarch64.whl"), + ] + + checker.require_repaired_linux_wheel_architectures(wheels, ["x86_64", "aarch64"]) + + +def test_repaired_linux_wheel_coverage_rejects_missing_arm64_wheels() -> None: + checker = _load_checker() + wheels = [ + Path("dotmatch-0.1.0-py3-none-manylinux_2_28_x86_64.whl"), + Path("dotmatch-0.1.0-py3-none-musllinux_1_2_x86_64.whl"), + ] + + try: + checker.require_repaired_linux_wheel_architectures(wheels, ["x86_64", "aarch64"]) + except SystemExit as exc: + assert "manylinux_aarch64" in str(exc) + assert "musllinux_aarch64" in str(exc) + else: + raise AssertionError("expected missing aarch64 repaired wheels to fail") diff --git a/python/tests/test_check_release_readiness.py b/python/tests/test_check_release_readiness.py index 1f19706f..f07d7225 100644 --- a/python/tests/test_check_release_readiness.py +++ b/python/tests/test_check_release_readiness.py @@ -16,7 +16,12 @@ def _load_checker(): def _write_release_repo(root: Path) -> None: files = { - "pyproject.toml": '[project]\nname = "dotmatch"\nversion = "0.1.0"\nlicense = "Apache-2.0"\n', + "pyproject.toml": ( + '[project]\nname = "dotmatch"\nversion = "0.1.0"\nlicense = "Apache-2.0"\n' + "[tool.cibuildwheel]\n" + 'build = "cp39-manylinux_aarch64 cp312-manylinux_aarch64 cp39-musllinux_aarch64 cp312-musllinux_aarch64"\n' + 'test-command = "dotmatch leq 1 ACGT AGGT"\n' + ), "package.json": '{"version": "0.1.0", "license": "Apache-2.0"}\n', "CITATION.cff": ( 'cff-version: 1.2.0\n' @@ -100,6 +105,8 @@ def _write_release_repo(root: Path) -> None: " container:\n" " needs: [preflight]\n" " steps:\n" + " - uses: docker/setup-qemu-action@v4\n" + " - run: docker buildx build --platform linux/arm64 --load -t dotmatch:ci-arm64 .\n" " - uses: docker/metadata-action@v5\n" " with:\n" " images: ghcr.io/dnncha/dotmatch\n" @@ -117,6 +124,10 @@ def _write_release_repo(root: Path) -> None: " username: ${{ github.actor }}\n" " password: ${{ secrets.GITHUB_TOKEN }}\n" " - uses: docker/build-push-action@v6\n" + " with:\n" + " platforms: linux/amd64,linux/arm64\n" + " - run: docker buildx imagetools inspect ghcr.io/dnncha/dotmatch:v0.1.0 --raw > ghcr-manifest.json\n" + " - run: python scripts/check_oci_manifest.py ghcr-manifest.json --require-platform linux/amd64 --require-platform linux/arm64\n" " - run: docker image inspect dotmatch:ci --format '{{ index .Config.Labels \"org.opencontainers.image.version\" }}'\n" " sdist:\n" " steps:\n" @@ -135,8 +146,11 @@ def _write_release_repo(root: Path) -> None: " linux-repaired-wheels:\n" " needs: [preflight]\n" " steps:\n" + " - uses: docker/setup-qemu-action@v4\n" " - uses: pypa/cibuildwheel@v3.3.0\n" - " - run: python scripts/check_python_wheel.py --wheel-only --out-dir dist-linux\n" + " env:\n" + " CIBW_ARCHS_LINUX: \"x86_64 aarch64\"\n" + " - run: python scripts/check_python_wheel.py --wheel-only --out-dir dist-linux --require-repaired-linux-architectures x86_64 aarch64\n" " - uses: actions/upload-artifact@v7\n" " with:\n" " name: dotmatch-linux-repaired-wheels\n" @@ -260,6 +274,20 @@ def test_release_readiness_rejects_unaligned_container_label(tmp_path): assert any("Dockerfile" in failure and "version" in failure for failure in result.failures) +def test_release_readiness_requires_linux_arm64_distribution_configuration(tmp_path): + checker = _load_checker() + _write_release_repo(tmp_path) + workflow_path = tmp_path / ".github" / "workflows" / "release.yml" + workflow_path.write_text( + workflow_path.read_text(encoding="utf-8").replace("platforms: linux/amd64,linux/arm64", "platforms: linux/amd64"), + encoding="utf-8", + ) + + result = checker.audit(tmp_path) + + assert any("platforms: linux/amd64,linux/arm64" in failure for failure in result.failures) + + def test_release_readiness_rejects_stale_container_smoke_version(tmp_path): checker = _load_checker() _write_release_repo(tmp_path) diff --git a/python/tests/test_check_repository_ready.py b/python/tests/test_check_repository_ready.py index 512731bb..230b4107 100644 --- a/python/tests/test_check_repository_ready.py +++ b/python/tests/test_check_repository_ready.py @@ -120,6 +120,7 @@ def _write_minimal_repo(root: Path) -> None: "scripts/check_citation_metadata.py": "#!/usr/bin/env python3\n", "scripts/check_distribution_channels.py": "#!/usr/bin/env python3\n", "scripts/check_distribution_record.py": "#!/usr/bin/env python3\n", + "scripts/check_oci_manifest.py": "#!/usr/bin/env python3\n", "scripts/check_bioconda_recipe.py": "#!/usr/bin/env python3\n", "scripts/check_native_comparator_scope.py": "#!/usr/bin/env python3\n", "scripts/check_workflow_adoption.py": "#!/usr/bin/env python3\n", @@ -278,6 +279,16 @@ def test_repository_ready_reports_missing_distribution_channel_verifier(tmp_path assert any("scripts/check_distribution_channels.py" in failure for failure in result.failures) +def test_repository_ready_reports_missing_oci_manifest_verifier(tmp_path): + checker = _load_checker() + _write_minimal_repo(tmp_path) + (tmp_path / "scripts" / "check_oci_manifest.py").unlink() + + result = checker.audit(tmp_path) + + assert any("scripts/check_oci_manifest.py" in failure for failure in result.failures) + + def test_repository_ready_reports_missing_alphabet_policy_verifier(tmp_path): checker = _load_checker() _write_minimal_repo(tmp_path) diff --git a/python/tests/test_check_workflow_examples.py b/python/tests/test_check_workflow_examples.py index 3ec670d4..7c954383 100644 --- a/python/tests/test_check_workflow_examples.py +++ b/python/tests/test_check_workflow_examples.py @@ -253,11 +253,12 @@ def _write_workflow_repo(root: Path) -> None: "# Galaxy Wrapper\n\nLocal Galaxy XML examples for DotMatch native and AssaySpec runs.\n" ), "examples/workflows/galaxy/dotmatch_crispr_count.xml": ( - "\n" - " dotmatch\n" - " dotmatch crispr-count --ambiguity-policy radius --ambiguous discard --summary '$summary' --sample-qc '$sample_qc'\n" + "\n" + " dotmatch\n" + " #for $sample in $reads:\nln -s '$sample' '$sample.element_identifier'\n#end for\ndotmatch crispr-count --ambiguity-policy radius --ambiguous discard --summary '$summary' --sample-qc '$sample_qc'\n" + " \n" " \n" - " \n" + " \n" "\n" ), "examples/workflows/galaxy/dotmatch_assay_run.xml": ( diff --git a/python/tests/test_feature_matrix.py b/python/tests/test_feature_matrix.py new file mode 100644 index 00000000..ef57000b --- /dev/null +++ b/python/tests/test_feature_matrix.py @@ -0,0 +1,173 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from dotmatch.feature_matrix import build_feature_matrix + + +ROOT = Path(__file__).resolve().parents[2] +LEGACY_ENV = {**os.environ, "DOTMATCH_PYTHON_NO_DELEGATE": "1"} + + +def _write_inputs(tmp_path: Path) -> tuple[Path, Path]: + targets = tmp_path / "features.tsv" + targets.write_text( + "target_id\ttarget_seq\n" + "feature_a\tAAAA\n" + "feature_b\tAACC\n" + "feature_c\tTTTT\n", + encoding="utf-8", + ) + observations = tmp_path / "observations.tsv" + observations.write_text( + "observation_id\tcell_barcode\tfeature_seq\n" + "obs_001\tcell_z\tAAAA\n" + "obs_002\tcell_z\tAAAT\n" + "obs_003\tcell_a\tTTTT\n" + "obs_004\tcell_a\tAAAC\n" + "obs_005\tcell_a\tCCCC\n" + "obs_006\tcell_a\t\n", + encoding="utf-8", + ) + return observations, targets + + +def test_build_feature_matrix_writes_deterministic_sparse_artifacts(tmp_path: Path) -> None: + observations, targets = _write_inputs(tmp_path) + result = build_feature_matrix( + observations, + targets, + tmp_path / "matrix", + cell_column="cell_barcode", + sequence_column="feature_seq", + id_column="observation_id", + k=1, + metric="hamming", + ) + + assert result.summary["matrix_orientation"] == "cells_by_features" + assert result.summary["total_observations"] == 6 + assert result.summary["valid_observations"] == 5 + assert result.summary["assigned_unique"] == 3 + assert result.summary["assigned_exact"] == 2 + assert result.summary["assigned_corrected"] == 1 + assert result.summary["ambiguous"] == 1 + assert result.summary["unmatched"] == 1 + assert result.summary["invalid"] == 1 + assert result.summary["cells"] == 2 + assert result.summary["features"] == 3 + assert result.summary["nonzero_entries"] == 2 + assert result.summary["scope"]["umi_deduplication"] == "not_performed" + + output = result.output_dir + assert (output / "barcodes.tsv").read_text(encoding="utf-8") == "cell_barcode\ncell_a\ncell_z\n" + assert (output / "features.tsv").read_text(encoding="utf-8") == ( + "target_id\ttarget_seq\nfeature_a\tAAAA\nfeature_b\tAACC\nfeature_c\tTTTT\n" + ) + assert (output / "cell_feature_counts.tsv").read_text(encoding="utf-8") == ( + "cell_barcode\ttarget_id\tcount\ncell_a\tfeature_c\t1\ncell_z\tfeature_a\t2\n" + ) + assert (output / "matrix.mtx").read_text(encoding="utf-8") == ( + "%%MatrixMarket matrix coordinate integer general\n" + "% DotMatch cell-by-feature unique-assignment counts\n" + "2 3 2\n" + "1 3 1\n" + "2 1 2\n" + ) + + assignments = (output / "assignments.tsv").read_text(encoding="utf-8") + assert "obs_004\tcell_a\tAAAC\t\t\t1\tambiguous\t2\t-1" in assignments + assert "obs_005\tcell_a\tCCCC\t\t\t-1\tnone\t0\t-1" in assignments + assert "obs_006\tcell_a\t\t\t\t-1\tinvalid\t0\t-1" in assignments + + qc = (output / "cell_qc.tsv").read_text(encoding="utf-8").splitlines() + assert qc[1] == "cell_a\t4\t3\t1\t1\t1\t1\t1\t0.3333333333333333" + assert qc[2] == "cell_z\t2\t2\t2\t0\t0\t0\t1\t1.0" + + written_summary = json.loads((output / "summary.json").read_text(encoding="utf-8")) + assert written_summary == result.summary + + +def test_build_feature_matrix_requires_explicit_existing_columns(tmp_path: Path) -> None: + observations, targets = _write_inputs(tmp_path) + + with pytest.raises(ValueError, match="missing required column 'wrong_cell'"): + build_feature_matrix( + observations, + targets, + tmp_path / "matrix", + cell_column="wrong_cell", + sequence_column="feature_seq", + ) + + assert not (tmp_path / "matrix").exists() + + +def test_feature_matrix_cli_writes_full_artifact_set(tmp_path: Path) -> None: + observations, targets = _write_inputs(tmp_path) + output = tmp_path / "matrix" + completed = subprocess.run( + [ + sys.executable, + "-m", + "dotmatch.cli", + "feature", + "matrix", + "--observations", + str(observations), + "--targets", + str(targets), + "--id-column", + "observation_id", + "--cell-column", + "cell_barcode", + "--sequence-column", + "feature_seq", + "--k", + "1", + "--metric", + "hamming", + "--out-dir", + str(output), + ], + check=False, + env=LEGACY_ENV, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + assert completed.returncode == 0, completed.stderr + assert json.loads(completed.stdout)["assigned_unique"] == 3 + assert {path.name for path in output.iterdir()} == { + "assignments.tsv", + "barcodes.tsv", + "cell_feature_counts.tsv", + "cell_qc.tsv", + "features.tsv", + "matrix.mtx", + "summary.json", + } + + +def test_assignments_to_anndata_accepts_text_status_when_optional_deps_are_available() -> None: + pd = pytest.importorskip("pandas") + pytest.importorskip("anndata") + import dotmatch + + assignments = pd.DataFrame( + { + "cell_barcode": ["cell_1", "cell_1", "cell_2"], + "target_id": ["feature_a", "feature_b", "feature_a"], + "status": ["unique", "ambiguous", "unique"], + } + ) + adata = dotmatch.assignments_to_anndata(assignments, feature_col="target_id") + + assert list(adata.obs_names) == ["cell_1", "cell_2"] + assert list(adata.var_names) == ["feature_a"] + assert adata.X.tolist() == [[1], [1]] diff --git a/python/tests/test_fetch_10x_crispr_guide_demo.py b/python/tests/test_fetch_10x_crispr_guide_demo.py index 40d6e247..f891f6a4 100644 --- a/python/tests/test_fetch_10x_crispr_guide_demo.py +++ b/python/tests/test_fetch_10x_crispr_guide_demo.py @@ -2,6 +2,7 @@ import importlib.util import json from pathlib import Path +from types import SimpleNamespace ROOT = Path(__file__).resolve().parents[2] @@ -74,6 +75,15 @@ def test_main_records_repo_relative_crispr_metadata(tmp_path, monkeypatch): ), ) monkeypatch.setattr(fetcher, "md5_file", lambda path: fetcher.FEATURE_REF_MD5) + monkeypatch.setattr( + fetcher, + "find_tar_member", + lambda url, suffix, tar_bytes=None: SimpleNamespace( + name=fetcher.R2_SUFFIX, + data_offset=0, + size=0, + ), + ) def fake_write_fastq(tar_url, suffix, dest, records, prefix_bytes=fetcher.FASTQ_PREFIX_BYTES, tar_bytes=None): _write_fastq( diff --git a/python/tests/test_packaging_artifacts.py b/python/tests/test_packaging_artifacts.py index 65f30381..6edc4230 100644 --- a/python/tests/test_packaging_artifacts.py +++ b/python/tests/test_packaging_artifacts.py @@ -48,6 +48,8 @@ def test_bioconda_recipe_builds_python_console_script_and_smoke_tests() -> None: assert "dotmatch leq 1 ACGT AGGT | grep '^true$'" in recipe assert "dotmatch assay --help" in recipe assert "dotmatch barcode --help" in recipe + assert "dotmatch feature --help" in recipe + assert "dotmatch feature matrix" in recipe assert "dotmatch panel --help" in recipe assert "dotmatch assay init" in recipe assert "dotmatch barcode infer" in recipe @@ -72,7 +74,7 @@ def test_bioconda_recipe_gate_is_wired_into_release_ready() -> None: assert "python3 scripts/check_bioconda_recipe.py" in makefile -def test_zenodo_metadata_tracks_minted_release_doi() -> None: +def test_zenodo_metadata_tracks_the_concept_doi_before_release_minting() -> None: metadata = json.loads((ROOT / ".zenodo.json").read_text(encoding="utf-8")) assert metadata["title"] == "DotMatch: deterministic known-target short-DNA assignment for sequencing workflows" @@ -92,7 +94,7 @@ def test_zenodo_metadata_tracks_minted_release_doi() -> None: assert metadata["conceptdoi"] == "10.5281/zenodo.20541628" -def test_codemeta_tracks_package_citation_and_minted_doi() -> None: +def test_codemeta_tracks_package_citation_and_concept_doi_before_minting() -> None: codemeta = json.loads((ROOT / "codemeta.json").read_text(encoding="utf-8")) citation = (ROOT / "CITATION.cff").read_text(encoding="utf-8") zenodo = json.loads((ROOT / ".zenodo.json").read_text(encoding="utf-8")) @@ -115,7 +117,7 @@ def test_codemeta_tracks_package_citation_and_minted_doi() -> None: } ] assert f"version: \"{_pyproject_version()}\"" in citation - assert "doi: 10.5281/zenodo.21511337" in citation + assert "doi:" not in citation assert codemeta["softwareVersion"] == zenodo["version"] assert "known-target assignment" in codemeta["keywords"] assert "CRISPR" in codemeta["keywords"] @@ -153,6 +155,19 @@ def test_python_package_verifier_checks_installed_cli_version() -> None: assert '"autopsy"' in verifier +def test_python_package_verifier_smokes_feature_matrix_and_paired_fastq() -> None: + verifier = (ROOT / "scripts" / "check_python_wheel.py").read_text(encoding="utf-8") + + assert '"feature",' in verifier + assert '"matrix",' in verifier + assert "feature_matrix" in verifier + assert "matrix.mtx" in verifier + assert '"pair-count",' in verifier + assert '"--left-reads",' in verifier + assert '"--right-reads",' in verifier + assert "paired FASTQ pair-count" in verifier + + def test_python_package_build_bundles_native_cli() -> None: setup = (ROOT / "setup.py").read_text(encoding="utf-8") pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") @@ -258,7 +273,7 @@ def test_distribution_docs_include_clean_pypi_install_verification() -> None: assert "pip install dotmatch==" in checker assert "must include repaired manylinux and musllinux wheels" in checker - assert "must not include raw linux_x86_64 wheels" in checker + assert "must not include raw" in checker assert "source distribution plus a macOS wheel" in packaging assert "repaired manylinux/musllinux wheels" in packaging assert "rejects raw" in packaging and "linux_x86_64" in packaging diff --git a/python/tests/test_public_language_guardrail.py b/python/tests/test_public_language_guardrail.py index a3b60dd2..9cad4805 100644 --- a/python/tests/test_public_language_guardrail.py +++ b/python/tests/test_public_language_guardrail.py @@ -14,7 +14,7 @@ ROOT / ".github", ] -PUBLIC_SUFFIXES = {".css", ".html", ".json", ".md", ".mdx", ".nf", ".py", ".ts", ".tsx", ".yaml", ".yml"} +PUBLIC_SUFFIXES = {".css", ".html", ".json", ".md", ".mdx", ".nf", ".py", ".ts", ".tsx", ".xml", ".yaml", ".yml"} SKIP_PARTS = {"_build", "node_modules", "__pycache__"} FORBIDDEN_PHRASES = [ @@ -33,6 +33,8 @@ "without turning private feedback into public " + "evidence", "industry " + "penetration", "massive industry " + "penetration", + "international computational biology " + "infiltration", + "totally super " + "massive", "excellent " + "ux", "perfect scientific " + "accuracy", "adoption " + "flywheel", diff --git a/python/tests/test_run_workflow_integration_tests.py b/python/tests/test_run_workflow_integration_tests.py new file mode 100644 index 00000000..510a0ba1 --- /dev/null +++ b/python/tests/test_run_workflow_integration_tests.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +RUNNER = ROOT / "scripts" / "run_workflow_integration_tests.py" + + +def _load_runner(): + spec = importlib.util.spec_from_file_location("run_workflow_integration_tests", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_planemo_lints_all_wrappers_and_runs_the_scoped_crispr_test(monkeypatch) -> None: + runner = _load_runner() + calls: list[tuple[list[str], Path, dict[str, str]]] = [] + + monkeypatch.setattr(runner, "_tool", lambda name: name) + monkeypatch.setattr( + runner, + "_run", + lambda command, *, cwd, env: calls.append((command, cwd, env)), + ) + + environment = {"PATH": ""} + runner.run_planemo(environment) + + assert calls == [ + (["planemo", "lint", *[str(path) for path in runner.GALAXY_WRAPPERS]], runner.ROOT, environment), + ( + ["planemo", "test", "--install_galaxy", str(runner.GALAXY_CRISPR_WRAPPER)], + runner.ROOT, + environment, + ), + ] diff --git a/python/tests/test_workflow_examples.py b/python/tests/test_workflow_examples.py index a90a75ee..b5a1bad0 100644 --- a/python/tests/test_workflow_examples.py +++ b/python/tests/test_workflow_examples.py @@ -175,15 +175,26 @@ def test_galaxy_wrapper_has_dotmatch_crispr_count_surface() -> None: assert root.tag == "tool" assert root.attrib["id"] == "dotmatch_crispr_count" + assert root.attrib["version"] == "0.2.1+galaxy0" - requirement_names = [node.text for node in root.findall("./requirements/requirement")] - assert "dotmatch" in requirement_names + requirements = {node.text: node.attrib.get("version") for node in root.findall("./requirements/requirement")} + assert requirements["dotmatch"] == "0.2.1" + + reads = root.find("./inputs/param[@name='reads']") + assert reads is not None + assert reads.attrib["multiple"] == "true" command = root.findtext("command") or "" assert "dotmatch crispr-count" in command + assert "element_identifier" in command + assert "ln -s" in command assert "--ambiguous" in command assert "--summary" in command assert "--sample-qc" in command + crispr_command = next(line for line in command.splitlines() if line.startswith("dotmatch crispr-count")) + assert "\\" not in crispr_command + assert "--out" in crispr_command + assert "--no-progress" in crispr_command output_names = {node.attrib["name"] for node in root.findall("./outputs/data")} assert {"counts", "summary", "sample_qc"} <= output_names @@ -197,8 +208,7 @@ def test_galaxy_wrapper_has_planemo_fixture_test() -> None: assert test is not None params = {node.attrib["name"]: node.attrib.get("value", "") for node in test.findall("param")} assert params["library"] == "crispr_library.csv" - assert params["sample1_fastq"] == "sample_a.fastq" - assert params["sample2_fastq"] == "sample_b.fastq" + assert params["reads"] == "sample_a.fastq,sample_b.fastq" outputs = {node.attrib["name"]: node.attrib.get("file", "") for node in test.findall("output")} assert outputs["counts"] == "expected_counts.mageck.tsv" sample_qc = next(node for node in test.findall("output") if node.attrib["name"] == "sample_qc") @@ -206,6 +216,18 @@ def test_galaxy_wrapper_has_planemo_fixture_test() -> None: assert sample_qc.find("./assert_contents/has_text[@text='sample_a']") is not None +def test_galaxy_count_fixture_keeps_the_exact_guide_assignment() -> None: + wrapper_path = ROOT / "examples" / "workflows" / "galaxy" / "dotmatch_crispr_count.xml" + wrapper = ET.parse(wrapper_path).getroot() + requirement = wrapper.find("./requirements/requirement") + assert requirement is not None + assert requirement.text == "dotmatch" + assert requirement.attrib["version"] == "0.2.1" + + expected = ROOT / "examples" / "workflows" / "galaxy" / "test-data" / "expected_counts.mageck.tsv" + assert "guide_a\tGENEA\t0\t0" in expected.read_text(encoding="utf-8") + + def test_workflow_fixtures_cover_core_outcomes() -> None: fixtures = ROOT / "examples" / "workflows" / "fixtures" readme = (fixtures / "README.md").read_text(encoding="utf-8") diff --git a/scripts/check_bioconda_recipe.py b/scripts/check_bioconda_recipe.py index aa9db81e..274841de 100644 --- a/scripts/check_bioconda_recipe.py +++ b/scripts/check_bioconda_recipe.py @@ -122,11 +122,14 @@ def _check_meta(meta: str, result: AuditResult) -> None: "dotmatch audit --help", "dotmatch assay --help", "dotmatch barcode --help", + "dotmatch feature --help", "dotmatch panel --help", "dotmatch assay init", "dotmatch audit --targets audit_targets.tsv --k 3 --audit-mode exact", "chmod -R a+rwX audit_out", "dotmatch barcode infer", + "dotmatch feature matrix", + "feature_matrix_out/matrix.mtx", "dotmatch panel design", "chmod -R a+rwX panel_out", "from dotmatch.native import find_native_cli", diff --git a/scripts/check_distribution_channels.py b/scripts/check_distribution_channels.py index ae4548c3..1a3c40e0 100644 --- a/scripts/check_distribution_channels.py +++ b/scripts/check_distribution_channels.py @@ -30,6 +30,11 @@ ) BIOCONTAINERS_IMAGE = "quay.io/biocontainers/dotmatch:{tag}" ZENODO_RECORD_URL = "https://zenodo.org/api/records/{record_id}" +DISTRIBUTION_RECORD_PATH = Path("docs") / "distribution-release.json" +DEFAULT_PYPI_LINUX_WHEEL_ARCHITECTURES = ("x86_64",) +DEFAULT_GHCR_PLATFORMS = ("linux/amd64",) +SUPPORTED_PYPI_LINUX_WHEEL_ARCHITECTURES = {"x86_64", "aarch64"} +SUPPORTED_GHCR_PLATFORMS = {"linux/amd64", "linux/arm64"} @dataclass(frozen=True) @@ -104,6 +109,61 @@ def project_version(root: Path) -> str: return match.group(1) +def release_channel_record(root: Path, version: str, channel_id: str) -> dict[str, object]: + path = root / DISTRIBUTION_RECORD_PATH + if not path.is_file(): + return {} + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"{DISTRIBUTION_RECORD_PATH.as_posix()} could not be read: {exc}") from exc + if not isinstance(manifest, dict) or str(manifest.get("release_version") or "") != version: + return {} + channels = manifest.get("channels") + if not isinstance(channels, list): + return {} + for item in channels: + if isinstance(item, dict) and str(item.get("id") or "") == channel_id: + return item + return {} + + +def recorded_string_list( + record: dict[str, object], + field: str, + default: tuple[str, ...], + supported_values: set[str], +) -> tuple[str, ...]: + if field not in record: + return default + value = record[field] + if not isinstance(value, list) or not value or not all(isinstance(item, str) and item for item in value): + raise ValueError(f"distribution record field {field} must be a non-empty list of strings") + values = tuple(dict.fromkeys(value)) + invalid = sorted(set(values) - supported_values) + if invalid: + raise ValueError(f"distribution record field {field} contains unsupported value(s): {', '.join(invalid)}") + return values + + +def required_pypi_linux_wheel_architectures(root: Path, version: str) -> tuple[str, ...]: + return recorded_string_list( + release_channel_record(root, version, "pypi"), + "linux_wheel_architectures", + DEFAULT_PYPI_LINUX_WHEEL_ARCHITECTURES, + SUPPORTED_PYPI_LINUX_WHEEL_ARCHITECTURES, + ) + + +def required_ghcr_platforms(root: Path, version: str) -> tuple[str, ...]: + return recorded_string_list( + release_channel_record(root, version, "ghcr"), + "platforms", + DEFAULT_GHCR_PLATFORMS, + SUPPORTED_GHCR_PLATFORMS, + ) + + def citation_doi(root: Path) -> str: text = (root / "CITATION.cff").read_text(encoding="utf-8") match = re.search(r'^\s*doi\s*:\s*["\']?([^"\'\s]+)', text, flags=re.MULTILINE) @@ -187,22 +247,25 @@ def verify_ghcr_run(image: str, version: str) -> None: raise RuntimeError(f"docker image dist smoke test reported {observed_distance!r}, expected '1'") -def verify_ghcr_manifest(image: str) -> str: +def verify_ghcr_manifest(image: str, required_platforms: tuple[str, ...] = DEFAULT_GHCR_PLATFORMS) -> str: data, digest = fetch_registry_manifest(image) if int(data.get("schemaVersion") or 0) != 2: raise RuntimeError("GHCR manifest must use schemaVersion 2") - manifests = data.get("manifests") or [] - if manifests: - linux_amd64 = [ - item - for item in manifests - if isinstance(item, dict) - and isinstance(item.get("platform"), dict) - and item["platform"].get("os") == "linux" - and item["platform"].get("architecture") == "amd64" - ] - if not linux_amd64: - raise RuntimeError("GHCR manifest list must include linux/amd64") + manifests = data.get("manifests") + if not isinstance(manifests, list): + raise RuntimeError("GHCR image must publish an OCI image index or Docker manifest list") + platforms = { + f"{platform.get('os')}/{platform.get('architecture')}" + for item in manifests + if isinstance(item, dict) + and isinstance(item.get("platform"), dict) + and (platform := item["platform"]) + and platform.get("os") + and platform.get("architecture") + } + missing = sorted(set(required_platforms) - platforms) + if missing: + raise RuntimeError(f"GHCR manifest list must include {', '.join(missing)}") if not digest: digest = str(data.get("config", {}).get("digest") or "") if not digest.startswith("sha256:"): @@ -314,8 +377,40 @@ def verify_biocontainers_run(image: str, version: str) -> None: raise RuntimeError(f"BioContainers dotmatch leq smoke test reported {observed_threshold!r}, expected 'true'") -def check_pypi(version: str, result: AuditResult) -> None: +def wheel_platform_tags(filename: str) -> list[str]: + if not filename.endswith(".whl"): + return [] + fields = filename[:-4].rsplit("-", 3) + if len(fields) != 4: + return [] + return fields[-1].split(".") + + +def has_repaired_linux_wheel(wheels: list[dict], family: str, architecture: str) -> bool: + return any( + any(tag.startswith(family) and tag.endswith(f"_{architecture}") for tag in wheel_platform_tags(str(item.get("filename") or ""))) + for item in wheels + ) + + +def raw_linux_wheel_architectures(wheels: list[dict]) -> set[str]: + return { + architecture + for architecture in SUPPORTED_PYPI_LINUX_WHEEL_ARCHITECTURES + if any( + f"linux_{architecture}" in wheel_platform_tags(str(item.get("filename") or "")) + for item in wheels + ) + } + + +def check_pypi(root: Path, version: str, result: AuditResult) -> None: channel = "pypi" + try: + required_architectures = required_pypi_linux_wheel_architectures(root, version) + except ValueError as exc: + result.failures.append(ChannelMessage(channel, str(exc))) + return try: data = fetch_json(PYPI_URL.format(version=version)) except Exception as exc: @@ -325,30 +420,37 @@ def check_pypi(version: str, result: AuditResult) -> None: has_sdist = any(item.get("packagetype") == "sdist" for item in urls if isinstance(item, dict)) wheels = [item for item in urls if isinstance(item, dict) and item.get("packagetype") == "bdist_wheel"] has_macos_wheel = any("macosx_" in str(item.get("filename") or "") for item in wheels) - has_manylinux_wheel = any("manylinux" in str(item.get("filename") or "") for item in wheels) - has_musllinux_wheel = any("musllinux" in str(item.get("filename") or "") for item in wheels) - has_raw_linux_wheel = any( - "linux_x86_64" in str(item.get("filename") or "") - and "manylinux" not in str(item.get("filename") or "") - and "musllinux" not in str(item.get("filename") or "") - for item in wheels - ) if data.get("info", {}).get("version") != version or not has_sdist: result.failures.append(ChannelMessage(channel, f"PyPI version {version} is not available as an sdist")) return if not has_macos_wheel: result.failures.append(ChannelMessage(channel, f"PyPI version {version} must include a macOS wheel")) return - if not has_manylinux_wheel or not has_musllinux_wheel: + missing_repaired_wheels = [ + f"{family}_{architecture}" + for family in ["manylinux", "musllinux"] + for architecture in required_architectures + if not has_repaired_linux_wheel(wheels, family, architecture) + ] + if missing_repaired_wheels: result.failures.append( - ChannelMessage(channel, f"PyPI version {version} must include repaired manylinux and musllinux wheels") + ChannelMessage( + channel, + f"PyPI version {version} must include repaired manylinux and musllinux wheels " + f"for {', '.join(required_architectures)} (missing: {', '.join(missing_repaired_wheels)})", + ) ) return - if has_raw_linux_wheel: - result.failures.append(ChannelMessage(channel, f"PyPI version {version} must not include raw linux_x86_64 wheels")) + raw_architectures = raw_linux_wheel_architectures(wheels) + if raw_architectures: + formatted = ", ".join(f"linux_{architecture}" for architecture in sorted(raw_architectures)) + result.failures.append(ChannelMessage(channel, f"PyPI version {version} must not include raw {formatted} wheels")) return result.passed.append( - ChannelMessage(channel, f"PyPI sdist, macOS wheel, and repaired Linux wheels are available for {version}") + ChannelMessage( + channel, + f"PyPI sdist, macOS wheel, and repaired Linux wheels ({', '.join(required_architectures)}) are available for {version}", + ) ) try: verify_pypi_install(version) @@ -453,15 +555,21 @@ def check_biocontainers(version: str, result: AuditResult) -> None: result.passed.append(ChannelMessage("biocontainers-run", f"BioContainers docker run smoke tests pass for {image}")) -def check_ghcr(version: str, result: AuditResult) -> None: +def check_ghcr(root: Path, version: str, result: AuditResult) -> None: channel = "ghcr" image = GHCR_IMAGE.format(version=version) try: - digest = verify_ghcr_manifest(image) + required_platforms = required_ghcr_platforms(root, version) + if required_platforms == DEFAULT_GHCR_PLATFORMS: + digest = verify_ghcr_manifest(image) + else: + digest = verify_ghcr_manifest(image, required_platforms) except Exception as exc: result.failures.append(ChannelMessage(channel, f"GHCR image tag {image} is not available: {exc}")) return - result.passed.append(ChannelMessage(channel, f"GHCR image tag is available: {image} ({digest})")) + result.passed.append( + ChannelMessage(channel, f"GHCR image tag is available: {image} ({digest}; {', '.join(required_platforms)})") + ) try: verify_ghcr_run(image, version) except FileNotFoundError: @@ -516,10 +624,10 @@ def audit(root: Path, version: Optional[str] = None) -> AuditResult: except Exception as exc: result.failures.append(ChannelMessage("metadata", str(exc))) return result - check_pypi(release_version, result) + check_pypi(root, release_version, result) check_bioconda(release_version, result) check_biocontainers(release_version, result) - check_ghcr(release_version, result) + check_ghcr(root, release_version, result) check_zenodo(root, release_version, result) return result diff --git a/scripts/check_distribution_record.py b/scripts/check_distribution_record.py index 61fb6cfe..faba3b1f 100644 --- a/scripts/check_distribution_record.py +++ b/scripts/check_distribution_record.py @@ -17,6 +17,9 @@ REQUIRED_CHANNELS = ["pypi", "bioconda", "ghcr", "biocontainers", "zenodo"] VALID_OVERALL_STATUSES = {"not_released", "partially_verified", "released"} VALID_CHANNEL_STATUSES = {"prepared", "blocked", "manifest_verified", "verified"} +VERIFIED_CHANNEL_STATUSES = {"manifest_verified", "verified"} +SUPPORTED_PYPI_LINUX_WHEEL_ARCHITECTURES = {"x86_64", "aarch64"} +SUPPORTED_GHCR_PLATFORMS = {"linux/amd64", "linux/arm64"} def _project_version(root: Path) -> str: @@ -29,6 +32,28 @@ def _check_https_url(channel_id: str, field: str, value: str, result: AuditResul return check_https_url(channel_id, field, value, result) +def _check_declared_values( + channel_id: str, + item: dict[str, object], + field: str, + supported_values: set[str], + required_value: str, + result: AuditResult, +) -> None: + value = item.get(field) + if not isinstance(value, list) or not value or not all(isinstance(entry, str) and entry for entry in value): + result.failures.append(f"{channel_id} must declare {field} as a non-empty list of strings") + return + entries = [str(entry) for entry in value] + if len(entries) != len(set(entries)): + result.failures.append(f"{channel_id} {field} must not contain duplicates") + invalid = sorted(set(entries) - supported_values) + if invalid: + result.failures.append(f"{channel_id} {field} contains unsupported value(s): {', '.join(invalid)}") + if required_value not in entries: + result.failures.append(f"{channel_id} {field} must include {required_value}") + + def _check_channel(item: object, overall_status: str, result: AuditResult) -> str: if not isinstance(item, dict): result.failures.append("distribution channels must be objects") @@ -40,6 +65,25 @@ def _check_channel(item: object, overall_status: str, result: AuditResult) -> st status = str(item.get("status") or "").strip() if status not in VALID_CHANNEL_STATUSES: result.failures.append(f"{channel_id} has invalid distribution channel status: {status}") + if status in VERIFIED_CHANNEL_STATUSES: + if channel_id == "pypi": + _check_declared_values( + channel_id, + item, + "linux_wheel_architectures", + SUPPORTED_PYPI_LINUX_WHEEL_ARCHITECTURES, + "x86_64", + result, + ) + elif channel_id == "ghcr": + _check_declared_values( + channel_id, + item, + "platforms", + SUPPORTED_GHCR_PLATFORMS, + "linux/amd64", + result, + ) verification = str(item.get("verification_command") or "").strip() if verification != "make distribution-channels": diff --git a/scripts/check_oci_manifest.py b/scripts/check_oci_manifest.py new file mode 100644 index 00000000..a93e6548 --- /dev/null +++ b/scripts/check_oci_manifest.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Check required platforms in an OCI image index or Docker manifest list.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def available_platforms(manifest: object) -> set[str]: + if not isinstance(manifest, dict): + raise ValueError("OCI manifest must be a JSON object") + if int(manifest.get("schemaVersion") or 0) != 2: + raise ValueError("OCI manifest must use schemaVersion 2") + descriptors = manifest.get("manifests") + if not isinstance(descriptors, list): + raise ValueError("OCI manifest must be an image index or manifest list") + + platforms: set[str] = set() + for descriptor in descriptors: + if not isinstance(descriptor, dict): + continue + platform = descriptor.get("platform") + if not isinstance(platform, dict): + continue + operating_system = str(platform.get("os") or "").strip() + architecture = str(platform.get("architecture") or "").strip() + if operating_system and architecture: + platforms.add(f"{operating_system}/{architecture}") + return platforms + + +def check_manifest(manifest: object, required_platforms: list[str]) -> set[str]: + platforms = available_platforms(manifest) + missing = sorted(set(required_platforms) - platforms) + if missing: + raise ValueError(f"OCI manifest is missing required platform(s): {', '.join(missing)}") + return platforms + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path, help="OCI image index or Docker manifest-list JSON file") + parser.add_argument( + "--require-platform", + action="append", + dest="required_platforms", + default=[], + metavar="OS/ARCH", + help="required platform, for example linux/arm64; may be repeated", + ) + args = parser.parse_args() + + try: + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + platforms = check_manifest(manifest, args.required_platforms) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"OCI MANIFEST: FAIL ({exc})") + return 1 + + rendered = ", ".join(sorted(platforms)) or "none" + print(f"OCI MANIFEST: PASS ({rendered})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_python_wheel.py b/scripts/check_python_wheel.py index 55e1885d..91b776d6 100644 --- a/scripts/check_python_wheel.py +++ b/scripts/check_python_wheel.py @@ -20,6 +20,13 @@ ROOT = Path(__file__).resolve().parents[1] +REPAIRED_LINUX_WHEEL_ARCHITECTURES = ("x86_64", "aarch64") +HOST_ARCHITECTURE_ALIASES = { + "amd64": "x86_64", + "arm64": "aarch64", + "x86_64": "x86_64", + "aarch64": "aarch64", +} def run(cmd: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None) -> None: @@ -256,6 +263,90 @@ def verify_clean_install(artifact: Path, install_root: Path, expected_version: s if dist_observed != "1": raise SystemExit(f"{artifact.name} console CLI distance smoke test returned {dist_observed!r}") + feature_targets = probe_dir / "feature_targets.tsv" + feature_observations = probe_dir / "feature_observations.tsv" + feature_output = probe_dir / "feature_matrix" + feature_targets.write_text("target_id\ttarget_seq\nfeature_a\tACGT\nfeature_b\tTTTT\n", encoding="utf-8") + feature_observations.write_text( + "observation_id\tcell_barcode\tfeature_seq\n" + "feature_read_1\tcell_a\tACGT\n" + "feature_read_2\tcell_b\tTTTT\n" + "feature_read_3\tcell_b\tCCCC\n", + encoding="utf-8", + ) + run( + [ + str(venv_script(env_dir, "dotmatch")), + "feature", + "matrix", + "--observations", + str(feature_observations), + "--targets", + str(feature_targets), + "--id-column", + "observation_id", + "--cell-column", + "cell_barcode", + "--sequence-column", + "feature_seq", + "--metric", + "hamming", + "--k", + "0", + "--out-dir", + str(feature_output), + ], + cwd=probe_dir, + env=env, + ) + feature_summary = json.loads((feature_output / "summary.json").read_text(encoding="utf-8")) + if feature_summary.get("assigned_unique") != 2 or feature_summary.get("unmatched") != 1: + raise SystemExit(f"{artifact.name} installed feature matrix summary is invalid: {feature_summary!r}") + if not (feature_output / "matrix.mtx").is_file(): + raise SystemExit(f"{artifact.name} installed feature matrix command did not write matrix.mtx") + + left_targets = probe_dir / "left_targets.tsv" + right_targets = probe_dir / "right_targets.tsv" + left_reads = probe_dir / "pair_R1.fastq" + right_reads = probe_dir / "pair_R2.fastq" + pair_counts = probe_dir / "pair_counts.tsv" + left_targets.write_text("left_a\tACGT\n", encoding="utf-8") + right_targets.write_text("right_a\tGGAA\n", encoding="utf-8") + left_reads.write_text("@pair_1/1\nACGT\n+\nIIII\n", encoding="utf-8") + right_reads.write_text("@pair_1/2\nGGAA\n+\nIIII\n", encoding="utf-8") + run( + [ + str(venv_script(env_dir, "dotmatch")), + "pair-count", + "--left-targets", + str(left_targets), + "--right-targets", + str(right_targets), + "--left-reads", + str(left_reads), + "--right-reads", + str(right_reads), + "--left-start", + "0", + "--left-length", + "4", + "--right-start", + "0", + "--right-length", + "4", + "--k", + "0", + "--metric", + "hamming", + "--out", + str(pair_counts), + ], + cwd=probe_dir, + env=env, + ) + if "left_a\tright_a\t1" not in pair_counts.read_text(encoding="utf-8"): + raise SystemExit(f"{artifact.name} installed paired FASTQ pair-count output is invalid") + targets = probe_dir / "targets.tsv" reads = probe_dir / "reads.fastq" spec = probe_dir / "assay.toml" @@ -464,14 +555,49 @@ def check_macos_architecture(wheel: Path, native_member: str) -> None: ) +def wheel_platform_tags(wheel: Path) -> list[str]: + if not wheel.name.endswith(".whl"): + return [] + fields = wheel.name[:-4].rsplit("-", 3) + if len(fields) != 4: + return [] + return fields[-1].split(".") + + +def repaired_linux_wheel_architectures(wheel: Path, family: str) -> set[str]: + return { + architecture + for architecture in REPAIRED_LINUX_WHEEL_ARCHITECTURES + if any( + tag.startswith(family) and tag.endswith(f"_{architecture}") + for tag in wheel_platform_tags(wheel) + ) + } + + +def require_repaired_linux_wheel_architectures(wheels: list[Path], required_architectures: list[str]) -> None: + missing = [ + f"{family}_{architecture}" + for family in ["manylinux", "musllinux"] + for architecture in required_architectures + if not any(architecture in repaired_linux_wheel_architectures(wheel, family) for wheel in wheels) + ] + if missing: + raise SystemExit("missing repaired Linux wheel coverage: " + ", ".join(missing)) + + def wheel_supported_by_current_platform(wheel: Path) -> bool: name = wheel.name system = platform.system() libc_name = platform.libc_ver()[0].lower() if "musllinux" in name: - return system == "Linux" and libc_name == "musl" + architectures = repaired_linux_wheel_architectures(wheel, "musllinux") + host_architecture = HOST_ARCHITECTURE_ALIASES.get(platform.machine().lower(), platform.machine().lower()) + return system == "Linux" and libc_name == "musl" and host_architecture in architectures if "manylinux" in name: - return system == "Linux" and libc_name == "glibc" + architectures = repaired_linux_wheel_architectures(wheel, "manylinux") + host_architecture = HOST_ARCHITECTURE_ALIASES.get(platform.machine().lower(), platform.machine().lower()) + return system == "Linux" and libc_name == "glibc" and host_architecture in architectures if "macosx" in name: return system == "Darwin" if "win_" in name or "win32" in name or "win_amd64" in name: @@ -544,6 +670,13 @@ def main() -> int: parser.add_argument("--out-dir", default="", help="optional wheel output directory") parser.add_argument("--sdist-only", action="store_true", help="build and verify only the source distribution") parser.add_argument("--wheel-only", action="store_true", help="verify existing wheels in --out-dir without building") + parser.add_argument( + "--require-repaired-linux-architectures", + choices=REPAIRED_LINUX_WHEEL_ARCHITECTURES, + metavar="ARCH", + nargs="+", + help="require repaired manylinux and musllinux wheels for each architecture", + ) args = parser.parse_args() if args.out_dir: @@ -560,6 +693,8 @@ def main() -> int: expected_version = project_version() if args.wheel_only: wheels = verify_existing_wheels(out_dir, install_root / "existing-wheel-install", expected_version) + if args.require_repaired_linux_architectures: + require_repaired_linux_wheel_architectures(wheels, args.require_repaired_linux_architectures) print("verified existing wheels: " + ", ".join(wheel.name for wheel in wheels)) return 0 sdist_out_dir = out_dir if args.sdist_only else install_root / "sdist" diff --git a/scripts/check_release_readiness.py b/scripts/check_release_readiness.py index a9bac2d1..c885f73e 100644 --- a/scripts/check_release_readiness.py +++ b/scripts/check_release_readiness.py @@ -203,6 +203,7 @@ def check_sdist_metadata(root: Path, result: ReleaseAudit) -> None: def check_distribution_surfaces(root: Path, result: ReleaseAudit) -> None: workflow = _read(root / ".github" / "workflows" / "release.yml") + pyproject = _read(root / "pyproject.toml") dockerfile = _read(root / "Dockerfile") bioconda = _read(root / "packaging" / "bioconda" / "meta.yaml") packaging = _read(root / "docs" / "packaging.md") @@ -221,22 +222,39 @@ def check_distribution_surfaces(root: Path, result: ReleaseAudit) -> None: "docker/build-push-action", "ghcr.io/dnncha/dotmatch", "python scripts/check_python_wheel.py --wheel-only --out-dir dist-linux", + "CIBW_ARCHS_LINUX: \"x86_64 aarch64\"", + "--require-repaired-linux-architectures x86_64 aarch64", + "docker/setup-qemu-action@v4", + "docker buildx build --platform linux/arm64", + "platforms: linux/amd64,linux/arm64", + "docker buildx imagetools inspect", + "scripts/check_oci_manifest.py", "docker image inspect dotmatch:ci", "SHA256SUMS.txt", ] for fragment in required_workflow_fragments: if fragment not in workflow: result.failures.append(f"release workflow missing {fragment}") + required_cibuildwheel_fragments = [ + "cp39-manylinux_aarch64", + "cp312-manylinux_aarch64", + "cp39-musllinux_aarch64", + "cp312-musllinux_aarch64", + "test-command", + "dotmatch leq 1 ACGT AGGT", + ] + for fragment in required_cibuildwheel_fragments: + if fragment not in pyproject: + result.failures.append(f"pyproject.toml cibuildwheel configuration missing {fragment}") if "dotmatch-wheel-Linux" in workflow: result.failures.append("release workflow must not publish raw Linux wheels to PyPI") container_version_check = re.search( - r"docker image inspect dotmatch:ci.*?\| grep '\^([^']+)\$'", + r"docker image inspect dotmatch:ci[^\n]*\| grep [\"']\^([^\"']+)\$[\"']", workflow, - flags=re.S, ) if container_version_check and project_version: workflow_version = container_version_check.group(1) - if workflow_version != project_version: + if workflow_version not in {project_version, "${VERSION}"}: result.failures.append( "release workflow container version smoke test must match " f"pyproject.toml ({project_version}); saw {workflow_version}" diff --git a/scripts/check_repository_ready.py b/scripts/check_repository_ready.py index a08d55bd..a6986427 100644 --- a/scripts/check_repository_ready.py +++ b/scripts/check_repository_ready.py @@ -67,6 +67,7 @@ "scripts/check_citation_metadata.py", "scripts/check_distribution_channels.py", "scripts/check_distribution_record.py", + "scripts/check_oci_manifest.py", "scripts/check_bioconda_recipe.py", "scripts/check_native_comparator_scope.py", "scripts/check_workflow_adoption.py", diff --git a/scripts/check_workflow_examples.py b/scripts/check_workflow_examples.py index 1b59b76a..19e17194 100644 --- a/scripts/check_workflow_examples.py +++ b/scripts/check_workflow_examples.py @@ -540,15 +540,22 @@ def check_galaxy(root: Path, result: WorkflowAudit) -> None: if wrapper.tag != "tool" or wrapper.attrib.get("id") != "dotmatch_crispr_count": result.failures.append("Galaxy wrapper must be tool id dotmatch_crispr_count") + if wrapper.attrib.get("version") != "0.2.1+galaxy0": + result.failures.append("Galaxy CRISPR wrapper must track the public Bioconda 0.2.1 package") command = wrapper.findtext("command") or "" _require(command, "dotmatch crispr-count", "Galaxy wrapper command must run dotmatch crispr-count", result) _require(command, "--ambiguity-policy radius", "Galaxy wrapper command must keep assignment ambiguity policy explicit", result) _require(command, "--ambiguous", "Galaxy wrapper command must expose --ambiguous", result) _require(command, "--summary", "Galaxy wrapper command must include --summary", result) _require(command, "--sample-qc", "Galaxy wrapper command must include --sample-qc", result) - requirements = [node.text for node in wrapper.findall("./requirements/requirement")] - if "dotmatch" not in requirements: - result.failures.append("Galaxy wrapper requirements must include dotmatch") + _require(command, "element_identifier", "Galaxy wrapper command must derive sample IDs from Galaxy datasets", result) + _require(command, "ln -s", "Galaxy wrapper command must stage input FASTQs", result) + requirements = {node.text: node.attrib.get("version", "") for node in wrapper.findall("./requirements/requirement")} + if requirements.get("dotmatch") != "0.2.1": + result.failures.append("Galaxy wrapper must require public Bioconda dotmatch=0.2.1") + reads = wrapper.find("./inputs/param[@name='reads']") + if reads is None or reads.attrib.get("multiple") != "true": + result.failures.append("Galaxy wrapper must accept one or more FASTQ datasets through reads") output_names = {node.attrib.get("name", "") for node in wrapper.findall("./outputs/data")} if not {"counts", "summary", "sample_qc"} <= output_names: result.failures.append("Galaxy wrapper outputs must include counts, summary, and sample_qc") @@ -559,10 +566,7 @@ def check_galaxy(root: Path, result: WorkflowAudit) -> None: params = {node.attrib.get("name", ""): node.attrib.get("value", "") for node in test.findall("param")} for name, value in [ ("library", "crispr_library.csv"), - ("sample1_fastq", "sample_a.fastq"), - ("sample1_label", "sample_a"), - ("sample2_fastq", "sample_b.fastq"), - ("sample2_label", "sample_b"), + ("reads", "sample_a.fastq,sample_b.fastq"), ]: if params.get(name) != value: result.failures.append(f"Galaxy Planemo test must set {name}={value}") @@ -575,6 +579,9 @@ def check_galaxy(root: Path, result: WorkflowAudit) -> None: result.failures.append("Galaxy Planemo test must assert sample_qc output") elif sample_qc.find("./assert_contents/has_text[@text='assignment_rate']") is None: result.failures.append("Galaxy Planemo test must assert sample_qc assignment_rate content") + expected_counts = test_data / "expected_counts.mageck.tsv" + if expected_counts.is_file() and "guide_a\tGENEA\t0\t0" not in expected_counts.read_text(encoding="utf-8"): + result.failures.append("Galaxy expected counts must match the pinned dotmatch=0.2.1 guide_a assignment") for filename in GALAXY_TEST_DATA: if not (test_data / filename).is_file(): result.failures.append(f"Galaxy Planemo test-data file is missing: {filename}") diff --git a/scripts/run_workflow_integration_tests.py b/scripts/run_workflow_integration_tests.py index 04b2bdb6..e54a1870 100644 --- a/scripts/run_workflow_integration_tests.py +++ b/scripts/run_workflow_integration_tests.py @@ -3,7 +3,8 @@ This runner keeps the expensive ecosystem checks in one place so local release work and CI exercise the same artifacts: nf-test modules, the small Nextflow -pipeline, Snakemake, Galaxy wrapper linting, and MultiQC custom/plugin reports. +pipeline, Snakemake, Galaxy wrapper linting and CRISPR-count execution, and +MultiQC custom/plugin reports. """ from __future__ import annotations @@ -26,6 +27,7 @@ ROOT / "examples" / "workflows" / "galaxy" / "dotmatch_demux.xml", ROOT / "examples" / "workflows" / "galaxy" / "dotmatch_panel_check.xml", ] +GALAXY_CRISPR_WRAPPER = GALAXY_WRAPPERS[0] class Failure(Exception): @@ -132,9 +134,14 @@ def run_snakemake(tmp: Path, env: dict[str, str]) -> None: ) -def run_planemo_lint(env: dict[str, str]) -> None: +def run_planemo(env: dict[str, str]) -> None: _tool("planemo") _run(["planemo", "lint", *[str(path) for path in GALAXY_WRAPPERS]], cwd=ROOT, env=env) + _run( + ["planemo", "test", "--install_galaxy", str(GALAXY_CRISPR_WRAPPER)], + cwd=ROOT, + env=env, + ) def run_multiqc(tmp: Path, env: dict[str, str]) -> None: @@ -180,7 +187,11 @@ def main() -> int: parser.add_argument("--skip-nf-test", action="store_true", help="skip nf-test module checks") parser.add_argument("--skip-nextflow", action="store_true", help="skip the Nextflow pipeline check") parser.add_argument("--skip-snakemake", action="store_true", help="skip the Snakemake workflow check") - parser.add_argument("--skip-planemo", action="store_true", help="skip Galaxy wrapper linting") + parser.add_argument( + "--skip-planemo", + action="store_true", + help="skip Galaxy wrapper linting and CRISPR-count execution", + ) parser.add_argument("--skip-multiqc", action="store_true", help="skip MultiQC report checks") args = parser.parse_args() @@ -195,7 +206,7 @@ def main() -> int: if not args.skip_snakemake: run_snakemake(tmp, env) if not args.skip_planemo: - run_planemo_lint(env) + run_planemo(env) if not args.skip_multiqc: run_multiqc(tmp, env) except subprocess.CalledProcessError as exc: diff --git a/src/qda.c b/src/qda.c index c1e5afb4..2a0f63fa 100644 --- a/src/qda.c +++ b/src/qda.c @@ -75,7 +75,7 @@ static void usage(const char *argv0) { fprintf(stderr, " %s assign K barcodes.txt reads.txt [--ambiguity-policy radius|best]\n", argv0); fprintf(stderr, " %s match K targets.txt reads.txt [--ambiguity-policy radius|best]\n", argv0); fprintf(stderr, " %s fastq-assign --barcodes barcodes.tsv --reads reads.fastq[.gz] --barcode-start N --barcode-length L --k 0|1 [--ambiguity-policy radius|best] --out assignments.tsv\n", argv0); - fprintf(stderr, " %s pair-count --left-targets left.tsv --right-targets right.tsv --reads reads.fastq[.gz] --left-start N --left-length L --right-start N --right-length L --k 0|1|2 --metric hamming|levenshtein [--ambiguity-policy radius|best] --out pair_counts.tsv [--summary summary.json]\n", argv0); + fprintf(stderr, " %s pair-count --left-targets left.tsv --right-targets right.tsv (--reads reads.fastq[.gz] | --left-reads left.fastq[.gz] --right-reads right.fastq[.gz]) --left-start N --left-length L --right-start N --right-length L --k 0|1|2 --metric hamming|levenshtein [--ambiguity-policy radius|best] --out pair_counts.tsv [--summary summary.json]\n", argv0); fprintf(stderr, " %s demux --barcodes barcodes.tsv|barcodes.csv --reads reads.fastq[.gz] --barcode-start N --barcode-length L|auto --k 0|1|2 --metric hamming|levenshtein [--ambiguity-policy radius|best] [--max-correction-qual Q] --out-dir demux_dir [--summary qc.json]\n", argv0); fprintf(stderr, " %s bcl-demux --run-folder RUN --sample-sheet SampleSheet.csv --out-dir demux_dir --barcode-mismatches 0|1|1,1 [--threads N] (0=auto) [--gzip-level 0..9] [--emit-index-fastqs] [--summary summary.json]\n", argv0); fprintf(stderr, " %s bcl-validate --dotmatch-out DIR --truth-out DIR\n", argv0); @@ -126,9 +126,10 @@ static void help_manual(FILE *out, const char *argv0) { fprintf(out, " demux --barcodes barcodes.tsv|barcodes.csv --reads reads.fastq[.gz] \\\n"); fprintf(out, " --barcode-start N --barcode-length L|auto --k 0|1|2 --out-dir demux_dir\n"); fprintf(out, " Split reads by fixed-position inline barcodes.\n"); - fprintf(out, " pair-count --left-targets left.tsv --right-targets right.tsv --reads reads.fastq[.gz] \\\n"); + fprintf(out, " pair-count --left-targets left.tsv --right-targets right.tsv \\\n"); + fprintf(out, " (--reads reads.fastq[.gz] | --left-reads left.fastq[.gz] --right-reads right.fastq[.gz]) \\\n"); fprintf(out, " --left-start N --left-length L --right-start N --right-length L --out pair_counts.tsv\n"); - fprintf(out, " Count pairs of independent fixed-window targets.\n"); + fprintf(out, " Count independent fixed-window targets from one read or synchronized FASTQ mates.\n"); fprintf(out, "\n"); fprintf(out, "Diagnostics and validation:\n"); fprintf(out, " audit --targets targets.tsv|targets.csv --k K --out-dir audit_dir\n"); @@ -894,6 +895,20 @@ static void fastq_read_id(const char *header, char *out, size_t out_cap) { out[n] = '\0'; } +/* + * Paired FASTQ files commonly use either a shared Illumina identifier followed + * by a read-number field, or a terminal /1 and /2 suffix. Compare the stable + * identifier in both forms while retaining fastq_read_id behavior for + * single-read commands. + */ +static void fastq_pair_read_id(const char *header, char *out, size_t out_cap) { + fastq_read_id(header, out, out_cap); + size_t n = strlen(out); + if (n >= 2 && out[n - 2] == '/' && (out[n - 1] == '1' || out[n - 1] == '2')) { + out[n - 2] = '\0'; + } +} + static void print_fastq_row(FILE *out, const seq_table *targets, const char *read_id, const char *observed, qdaln_match_result r) { const char *target_id = ""; @@ -7475,6 +7490,8 @@ typedef struct pair_count_stats { unsigned long long left_unmatched; unsigned long long right_unmatched; unsigned long long invalid; + unsigned long long left_invalid; + unsigned long long right_invalid; unsigned long long candidates_considered; unsigned long long candidates_verified; } pair_count_stats; @@ -7503,6 +7520,8 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { const char *left_path = NULL; const char *right_path = NULL; const char *reads_path = NULL; + const char *left_reads_path = NULL; + const char *right_reads_path = NULL; const char *out_path = NULL; const char *summary_path = NULL; const char *assignments_path = NULL; @@ -7523,6 +7542,10 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { right_path = argv[i++]; } else if (strcmp(arg, "--reads") == 0 && i < argc) { reads_path = argv[i++]; + } else if (strcmp(arg, "--left-reads") == 0 && i < argc) { + left_reads_path = argv[i++]; + } else if (strcmp(arg, "--right-reads") == 0 && i < argc) { + right_reads_path = argv[i++]; } else if (strcmp(arg, "--left-start") == 0 && i < argc) { if (parse_size_value(argv[i++], &left_start) != 0) { usage(argv0); @@ -7580,8 +7603,11 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { } } - if (left_path == NULL || right_path == NULL || reads_path == NULL || out_path == NULL || - left_len == 0 || right_len == 0 || k < 0) { + int paired_fastq = reads_path == NULL; + if (left_path == NULL || right_path == NULL || out_path == NULL || left_len == 0 || right_len == 0 || k < 0 || + (reads_path != NULL && (left_reads_path != NULL || right_reads_path != NULL)) || + (paired_fastq && (left_reads_path == NULL || right_reads_path == NULL))) { + fprintf(stderr, "pair-count requires --reads or both --left-reads and --right-reads\n"); usage(argv0); return 2; } @@ -7598,7 +7624,8 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { size_t *right_lens = NULL; qdaln_index *left_index = NULL; qdaln_index *right_index = NULL; - fastq_reader reader = {0}; + fastq_reader left_reader = {0}; + fastq_reader right_reader = {0}; FILE *out = NULL; FILE *summary = NULL; FILE *assignments = NULL; @@ -7642,7 +7669,8 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { fprintf(stderr, "out of memory\n"); goto done; } - if (fastq_reader_open(&reader, reads_path) != 0) { + if (fastq_reader_open(&left_reader, paired_fastq ? left_reads_path : reads_path) != 0 || + (paired_fastq && fastq_reader_open(&right_reader, right_reads_path) != 0)) { fprintf(stderr, "failed to open FASTQ input\n"); goto done; } @@ -7655,30 +7683,67 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { fprintf(assignments, "read_id\tleft_observed\tleft_index\tleft_id\tleft_status\tleft_distance\tright_observed\tright_index\tright_id\tright_status\tright_distance\tpair_status\n"); } - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; + char left_header[8192]; + char left_seq[8192]; + char left_plus[8192]; + char left_qual[8192]; + char right_header[8192]; + char right_seq[8192]; + char right_plus[8192]; + char right_qual[8192]; char read_id[8192]; + char right_read_id[8192]; char left_observed[8192]; char right_observed[8192]; - size_t seq_len = 0; - int got = 0; - while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - (void)plus; - (void)qual; + for (;;) { + size_t left_seq_len = 0; + size_t right_seq_len = 0; + int left_got = fastq_read_record_len(&left_reader, left_header, left_seq, left_plus, left_qual, + sizeof(left_header), &left_seq_len); + if (left_got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + goto done; + } + const char *right_seq_ptr = left_seq; + if (paired_fastq) { + int right_got = fastq_read_record_len(&right_reader, right_header, right_seq, right_plus, right_qual, + sizeof(right_header), &right_seq_len); + if (right_got < 0) { + fprintf(stderr, "malformed paired FASTQ input\n"); + goto done; + } + if (left_got != right_got) { + fprintf(stderr, "paired FASTQ inputs have different record counts\n"); + goto done; + } + if (left_got == 0) break; + fastq_pair_read_id(left_header, read_id, sizeof(read_id)); + fastq_pair_read_id(right_header, right_read_id, sizeof(right_read_id)); + if (read_id[0] == '\0' || right_read_id[0] == '\0') { + fprintf(stderr, "paired FASTQ records require non-empty read IDs\n"); + goto done; + } + if (strcmp(read_id, right_read_id) != 0) { + fprintf(stderr, "paired FASTQ read IDs do not match: %s != %s\n", read_id, right_read_id); + goto done; + } + right_seq_ptr = right_seq; + } else { + if (left_got == 0) break; + right_seq_len = left_seq_len; + fastq_read_id(left_header, read_id, sizeof(read_id)); + } qdaln_match_result left = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; qdaln_match_result right = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; qdaln_index_stats left_stats = {0, 0}; qdaln_index_stats right_stats = {0, 0}; - fastq_read_id(header, read_id, sizeof(read_id)); left_observed[0] = '\0'; right_observed[0] = '\0'; ++stats.total_reads; - if (assign_count_window(left_index, seq, seq_len, left_start, left_len, k, metric, 0, + if (assign_count_window(left_index, left_seq, left_seq_len, left_start, left_len, k, metric, 0, &left, &left_stats, left_observed, sizeof(left_observed), 0) != 0 || - assign_count_window(right_index, seq, seq_len, right_start, right_len, k, metric, 0, + assign_count_window(right_index, right_seq_ptr, right_seq_len, right_start, right_len, k, metric, 0, &right, &right_stats, right_observed, sizeof(right_observed), 0) != 0) { fprintf(stderr, "FASTQ pair assignment failed\n"); goto done; @@ -7695,6 +7760,8 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { ++stats.assigned_pairs; } else if (strcmp(pair_status, "invalid") == 0) { ++stats.invalid; + if (left.status == QDALN_MATCH_INVALID) ++stats.left_invalid; + if (right.status == QDALN_MATCH_INVALID) ++stats.right_invalid; } else { if (left.status == QDALN_MATCH_AMBIGUOUS || right.status == QDALN_MATCH_AMBIGUOUS) ++stats.pair_ambiguous; if (left.status == QDALN_MATCH_NONE) ++stats.left_unmatched; @@ -7705,10 +7772,6 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { left_observed, left, right_observed, right); } } - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - goto done; - } out = open_output_file(out_path); if (out == NULL) { @@ -7731,11 +7794,12 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { goto done; } fprintf(summary, - "{\n \"workflow\": \"pair-count\",\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"left_start\": %zu,\n \"left_length\": %zu,\n \"right_start\": %zu,\n \"right_length\": %zu,\n \"n_left_targets\": %zu,\n \"n_right_targets\": %zu,\n \"total_reads\": %llu,\n \"assigned_pairs\": %llu,\n \"pair_ambiguous\": %llu,\n \"left_unmatched\": %llu,\n \"right_unmatched\": %llu,\n \"invalid\": %llu,\n \"candidates_considered\": %llu,\n \"candidates_verified\": %llu\n}\n", + "{\n \"workflow\": \"pair-count\",\n \"input_mode\": \"%s\",\n \"input_sync\": \"%s\",\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"left_start\": %zu,\n \"left_length\": %zu,\n \"right_start\": %zu,\n \"right_length\": %zu,\n \"n_left_targets\": %zu,\n \"n_right_targets\": %zu,\n \"total_reads\": %llu,\n \"total_pairs\": %llu,\n \"assigned_pairs\": %llu,\n \"pair_ambiguous\": %llu,\n \"left_unmatched\": %llu,\n \"right_unmatched\": %llu,\n \"invalid\": %llu,\n \"left_invalid\": %llu,\n \"right_invalid\": %llu,\n \"candidates_considered\": %llu,\n \"candidates_verified\": %llu\n}\n", + paired_fastq ? "paired-fastq" : "single-read", paired_fastq ? "canonical-read-id" : "not-applicable", k, metric_name(metric), ambiguity_policy_name(assignment_policy), qdaln_alphabet_policy(), left_start, left_len, right_start, right_len, - left_targets.count, right_targets.count, stats.total_reads, stats.assigned_pairs, + left_targets.count, right_targets.count, stats.total_reads, stats.total_reads, stats.assigned_pairs, stats.pair_ambiguous, stats.left_unmatched, stats.right_unmatched, stats.invalid, - stats.candidates_considered, stats.candidates_verified); + stats.left_invalid, stats.right_invalid, stats.candidates_considered, stats.candidates_verified); } rc = 0; @@ -7744,7 +7808,8 @@ static int run_pair_count(const char *argv0, int argc, char **argv) { if (out != NULL) fclose(out); if (summary != NULL) fclose(summary); if (assignments != NULL) fclose(assignments); - fastq_reader_close(&reader); + fastq_reader_close(&left_reader); + fastq_reader_close(&right_reader); qdaln_index_free(left_index); qdaln_index_free(right_index); free(left_ptrs); diff --git a/tests/test_cli_fastq.sh b/tests/test_cli_fastq.sh index 075a5fda..44d49376 100644 --- a/tests/test_cli_fastq.sh +++ b/tests/test_cli_fastq.sh @@ -250,6 +250,9 @@ PAIRFASTQ grep '^L0 R0 2$' "$TMPDIR/pair_counts.tsv" >/dev/null grep '^L1 R1 1$' "$TMPDIR/pair_counts.tsv" >/dev/null +grep '"input_mode": "single-read"' "$TMPDIR/pair_summary.json" >/dev/null +grep '"input_sync": "not-applicable"' "$TMPDIR/pair_summary.json" >/dev/null +grep '"total_pairs": 7' "$TMPDIR/pair_summary.json" >/dev/null grep '"assigned_pairs": 3' "$TMPDIR/pair_summary.json" >/dev/null grep '"pair_ambiguous": 1' "$TMPDIR/pair_summary.json" >/dev/null grep '"left_unmatched": 1' "$TMPDIR/pair_summary.json" >/dev/null @@ -258,6 +261,137 @@ grep '"invalid": 1' "$TMPDIR/pair_summary.json" >/dev/null grep '^p5 AGGT 0 L0 ambiguous 1 GGAA 0 R0 unique 0 ambiguous$' "$TMPDIR/pair_assignments.tsv" >/dev/null grep '^p6 -1 invalid -1 -1 invalid -1 invalid$' "$TMPDIR/pair_assignments.tsv" >/dev/null +cat > "$TMPDIR/pair_left_reads.fastq" <<'PAIRFASTQ' +@pair_a/1 +ACGT ++ +IIII +@pair_b 1:N:0:1 +TTTT ++ +IIII +@pair_c/1 +AC ++ +II +@pair_d/1 +ACGT ++ +IIII +PAIRFASTQ + +cat > "$TMPDIR/pair_right_reads.fastq" <<'PAIRFASTQ' +@pair_a/2 +GGAA ++ +IIII +@pair_b 2:N:0:1 +CCCC ++ +IIII +@pair_c/2 +GGAA ++ +IIII +@pair_d/2 +AC ++ +II +PAIRFASTQ + +"$DOTMATCH_BIN" pair-count \ + --left-targets "$TMPDIR/pair_left.tsv" \ + --right-targets "$TMPDIR/pair_right.tsv" \ + --left-reads "$TMPDIR/pair_left_reads.fastq" \ + --right-reads "$TMPDIR/pair_right_reads.fastq" \ + --left-start 0 \ + --left-length 4 \ + --right-start 0 \ + --right-length 4 \ + --k 1 \ + --metric hamming \ + --out "$TMPDIR/pair_paired_counts.tsv" \ + --summary "$TMPDIR/pair_paired_summary.json" \ + --assignments "$TMPDIR/pair_paired_assignments.tsv" + +grep '^L0 R0 1$' "$TMPDIR/pair_paired_counts.tsv" >/dev/null +grep '^L1 R1 1$' "$TMPDIR/pair_paired_counts.tsv" >/dev/null +grep '"input_mode": "paired-fastq"' "$TMPDIR/pair_paired_summary.json" >/dev/null +grep '"input_sync": "canonical-read-id"' "$TMPDIR/pair_paired_summary.json" >/dev/null +grep '"total_pairs": 4' "$TMPDIR/pair_paired_summary.json" >/dev/null +grep '"left_invalid": 1' "$TMPDIR/pair_paired_summary.json" >/dev/null +grep '"right_invalid": 1' "$TMPDIR/pair_paired_summary.json" >/dev/null +grep '^pair_a ACGT 0 L0 unique 0 GGAA 0 R0 unique 0 unique$' "$TMPDIR/pair_paired_assignments.tsv" >/dev/null +grep '^pair_c -1 invalid -1 GGAA 0 R0 unique 0 invalid$' "$TMPDIR/pair_paired_assignments.tsv" >/dev/null +grep '^pair_d ACGT 0 L0 unique 0 -1 invalid -1 invalid$' "$TMPDIR/pair_paired_assignments.tsv" >/dev/null + +if "$DOTMATCH_BIN" pair-count \ + --left-targets "$TMPDIR/pair_left.tsv" \ + --right-targets "$TMPDIR/pair_right.tsv" \ + --left-reads "$TMPDIR/pair_left_reads.fastq" \ + --left-start 0 \ + --left-length 4 \ + --right-start 0 \ + --right-length 4 \ + --k 1 \ + --metric hamming \ + --out "$TMPDIR/pair_missing_mate_counts.tsv" 2> "$TMPDIR/pair_missing_mate.err"; then + echo "pair-count accepted one paired FASTQ mate" >&2 + exit 1 +fi +grep 'pair-count requires --reads or both --left-reads and --right-reads' "$TMPDIR/pair_missing_mate.err" >/dev/null +test ! -e "$TMPDIR/pair_missing_mate_counts.tsv" + +cat > "$TMPDIR/pair_right_mismatch.fastq" <<'PAIRFASTQ' +@other/2 +GGAA ++ +IIII +PAIRFASTQ + +if "$DOTMATCH_BIN" pair-count \ + --left-targets "$TMPDIR/pair_left.tsv" \ + --right-targets "$TMPDIR/pair_right.tsv" \ + --left-reads "$TMPDIR/pair_left_reads.fastq" \ + --right-reads "$TMPDIR/pair_right_mismatch.fastq" \ + --left-start 0 \ + --left-length 4 \ + --right-start 0 \ + --right-length 4 \ + --k 1 \ + --metric hamming \ + --out "$TMPDIR/pair_mismatch_counts.tsv" 2> "$TMPDIR/pair_mismatch.err"; then + echo "pair-count accepted unsynchronized paired FASTQ IDs" >&2 + exit 1 +fi +grep 'paired FASTQ read IDs do not match: pair_a != other' "$TMPDIR/pair_mismatch.err" >/dev/null +test ! -e "$TMPDIR/pair_mismatch_counts.tsv" + +cat > "$TMPDIR/pair_right_short.fastq" <<'PAIRFASTQ' +@pair_a/2 +GGAA ++ +IIII +PAIRFASTQ + +if "$DOTMATCH_BIN" pair-count \ + --left-targets "$TMPDIR/pair_left.tsv" \ + --right-targets "$TMPDIR/pair_right.tsv" \ + --left-reads "$TMPDIR/pair_left_reads.fastq" \ + --right-reads "$TMPDIR/pair_right_short.fastq" \ + --left-start 0 \ + --left-length 4 \ + --right-start 0 \ + --right-length 4 \ + --k 1 \ + --metric hamming \ + --out "$TMPDIR/pair_short_counts.tsv" 2> "$TMPDIR/pair_short.err"; then + echo "pair-count accepted paired FASTQ files with different record counts" >&2 + exit 1 +fi +grep 'paired FASTQ inputs have different record counts' "$TMPDIR/pair_short.err" >/dev/null +test ! -e "$TMPDIR/pair_short_counts.tsv" + cat > "$TMPDIR/pair_left_duplicate.tsv" <<'TARGETS' Ldup ACGT Ldup TTTT @@ -1798,7 +1932,29 @@ grep '^bc3 TTTT G3 0 0 0 0 0 0 0$' "$TMPDIR/counts_exact_radius.tsv" >/dev/null grep '"count_engine": "hamming_lookup_direct_single_offset"' "$TMPDIR/summary_exact_radius.json" >/dev/null grep '"hamming_index": "exact"' "$TMPDIR/summary_exact_radius.json" >/dev/null +METAL_AVAILABLE=0 if [ "$(uname -s)" = "Darwin" ]; then + if "$DOTMATCH_BIN" count \ + --targets "$TMPDIR/targets.csv" \ + --reads "$TMPDIR/reads.fastq.gz" \ + --sample-label metal_probe \ + --target-start 0 \ + --target-length 4 \ + --k 1 \ + --metric hamming \ + --ambiguity-policy best \ + --format mageck \ + --backend gpu-metal-experimental \ + --out "$TMPDIR/counts_metal_probe.tsv" \ + --summary "$TMPDIR/summary_metal_probe.json" \ + 2> "$TMPDIR/metal_probe.err"; then + METAL_AVAILABLE=1 + else + grep 'Metal backend unavailable' "$TMPDIR/metal_probe.err" >/dev/null + fi +fi + +if [ "$METAL_AVAILABLE" = "1" ]; then "$DOTMATCH_BIN" count \ --targets "$TMPDIR/targets.csv" \ --reads "$TMPDIR/reads.fastq.gz" \ @@ -2350,7 +2506,7 @@ LIBRARYALIASES diff -u "$TMPDIR/expected_mageck.tsv" "$TMPDIR/crispr_mageck_reordered.tsv" -if [ "$(uname -s)" = "Darwin" ]; then +if [ "$METAL_AVAILABLE" = "1" ]; then "$DOTMATCH_BIN" crispr-count \ --library "$TMPDIR/targets.csv" \ --samples "$TMPDIR/samples.tsv" \