From d33fe2985c63e25645f5dce4227162b8c1af7018 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Mon, 13 Jul 2026 15:01:52 +0100 Subject: [PATCH 01/93] Implement industry evaluation and CRISPR quickstart --- .github/ISSUE_TEMPLATE/pilot_feedback.yml | 62 ++++ .../ISSUE_TEMPLATE/workflow_integration.yml | 54 ++++ .github/PULL_REQUEST_TEMPLATE.md | 6 + .github/workflows/release.yml | 4 +- .zenodo.json | 2 +- CITATION.cff | 2 +- Dockerfile | 2 +- Makefile | 10 +- README.md | 78 +++-- app/page.tsx | 26 +- codemeta.json | 4 +- docs/adopters/README.md | 6 +- docs/adopters/record-template.md | 38 +++ docs/adoption-metrics.json | 78 +++++ docs/adoption-metrics.md | 30 ++ .../performance-improvements-2026-07-05.md | 82 +++++ docs/bioinformatics-evaluation.md | 138 ++++++++ docs/citation-flywheel.md | 12 +- docs/conf.py | 2 +- docs/distribution-release.json | 52 ++- docs/external-review-packet.md | 87 +++++ docs/getting-started.md | 32 +- docs/index.md | 13 +- docs/industry-exposure-plan.json | 4 +- docs/industry-exposure.md | 12 +- docs/industry-next-wins.md | 18 +- docs/integration-targets.json | 86 +++++ docs/methods-and-citation.md | 8 +- docs/packaging.md | 37 ++- docs/pilot-program.md | 80 +++++ docs/registries/biotools.yml | 49 +++ docs/release-process.md | 2 +- docs/resubmission-evidence.md | 4 +- docs/reviewer-readiness.json | 67 ++++ docs/scientific-claims.md | 2 +- docs/scientific-readiness.json | 2 +- docs/tutorials/crispr-count-first-run.md | 4 +- docs/workflow-adoption.json | 2 +- docs/workflow-integration-kit.md | 163 ++++++++++ docs/workflow-integration-plan.json | 77 +++++ docs/workflow-integration-roadmap.md | 232 +++++++++++++ examples/workflows/nf-core/README.md | 2 +- .../modules/local/dotmatch/assay_run/main.nf | 4 +- .../modules/local/dotmatch/audit/main.nf | 4 +- .../modules/local/dotmatch/count/main.nf | 4 +- .../local/dotmatch/crispr_count/main.nf | 4 +- .../modules/local/dotmatch/demux/main.nf | 4 +- .../local/dotmatch/panel_check/main.nf | 4 +- .../nf-core/dotmatch/assay_run/main.nf | 4 +- .../modules/nf-core/dotmatch/audit/main.nf | 4 +- .../modules/nf-core/dotmatch/count/main.nf | 4 +- .../nf-core/dotmatch/crispr_count/main.nf | 4 +- .../modules/nf-core/dotmatch/demux/main.nf | 4 +- .../nf-core/dotmatch/panel_check/main.nf | 4 +- include/qdalign.h | 2 +- package-lock.json | 4 +- package.json | 2 +- packaging/bioconda/meta.yaml | 2 +- paper/paper.bib | 2 +- pyproject.toml | 2 +- python/dotmatch/__init__.py | 2 +- python/dotmatch/cli.py | 115 ++++++- python/tests/test_assayspec.py | 33 ++ python/tests/test_packaging_artifacts.py | 2 +- .../tests/test_public_language_guardrail.py | 57 ++++ scripts/check_adoption_metrics.py | 109 +++++++ scripts/check_reviewer_readiness_assets.py | 305 ++++++++++++++++++ scripts/check_site_assets.mjs | 4 +- src/qda.c | 83 +---- 69 files changed, 2199 insertions(+), 244 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/pilot_feedback.yml create mode 100644 .github/ISSUE_TEMPLATE/workflow_integration.yml create mode 100644 docs/adopters/record-template.md create mode 100644 docs/adoption-metrics.json create mode 100644 docs/adoption-metrics.md create mode 100644 docs/benchmarks/performance-improvements-2026-07-05.md create mode 100644 docs/bioinformatics-evaluation.md create mode 100644 docs/external-review-packet.md create mode 100644 docs/integration-targets.json create mode 100644 docs/pilot-program.md create mode 100644 docs/registries/biotools.yml create mode 100644 docs/reviewer-readiness.json create mode 100644 docs/workflow-integration-kit.md create mode 100644 docs/workflow-integration-plan.json create mode 100644 docs/workflow-integration-roadmap.md create mode 100644 python/tests/test_public_language_guardrail.py create mode 100644 scripts/check_adoption_metrics.py create mode 100644 scripts/check_reviewer_readiness_assets.py diff --git a/.github/ISSUE_TEMPLATE/pilot_feedback.yml b/.github/ISSUE_TEMPLATE/pilot_feedback.yml new file mode 100644 index 00000000..42968987 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/pilot_feedback.yml @@ -0,0 +1,62 @@ +name: Evaluation feedback +description: Record scoped feedback from a DotMatch technical evaluation. +title: "[Evaluation]: " +labels: ["evaluation", "feedback", "needs-triage"] +body: + - type: dropdown + id: assay_context + attributes: + label: Assay Context + options: + - CRISPR guide counting + - Inline barcode demultiplexing + - Feature barcode assignment + - Perturb-seq guide capture + - Amplicon or panel target assignment + - Oligo or adapter-prefix assignment + - Other known-target workflow + validations: + required: true + - type: textarea + id: install_path + attributes: + label: Install Path + description: State PyPI, Bioconda, source checkout, container, or workflow wrapper version. + validations: + required: true + - type: textarea + id: inputs + attributes: + label: Inputs + description: Describe the target table and read window without sharing private raw data. + validations: + required: true + - type: checkboxes + id: outputs_reviewed + attributes: + label: Outputs Reviewed + options: + - label: sample_qc.tsv + - label: summary.json + - label: assignments.tsv + - label: top_unmatched.tsv + - label: HTML report + - label: methods.md / CITATION.bib / software_versions.yml + - type: textarea + id: findings + attributes: + label: Findings + description: What became clearer or blocked the evaluation? + validations: + required: true + - type: dropdown + id: public_use_permission + attributes: + label: Public Use Permission + options: + - No public use record approved + - Anonymized summary approved + - Public project name approved + - Approved public use record text and URL + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/workflow_integration.yml b/.github/ISSUE_TEMPLATE/workflow_integration.yml new file mode 100644 index 00000000..46adfec2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/workflow_integration.yml @@ -0,0 +1,54 @@ +name: Workflow integration review +description: Propose or review an external DotMatch workflow integration. +title: "[Integration]: " +labels: ["workflow", "integration", "needs-triage"] +body: + - type: dropdown + id: workflow_manager + attributes: + label: Workflow Manager + options: + - nf-core / Nextflow + - MultiQC + - Galaxy / IUC + - Snakemake + - bio.tools + - Institutional workflow + validations: + required: true + - type: textarea + id: workflow_context + attributes: + label: Workflow Context + description: Which known-target assay workflow would this integration support? + validations: + required: true + - type: textarea + id: source_assets + attributes: + label: Source Assets + description: Link the local module, wrapper, fixture, parser, schema, or registry draft. + validations: + required: true + - type: textarea + id: expected_outputs + attributes: + label: Expected Outputs + description: List the TSV, JSON, FASTQ, HTML, MultiQC, or citation artifacts the integration should expose. + validations: + required: true + - type: textarea + id: review_evidence + attributes: + label: Review Evidence + description: Include commands, CI links, fixture paths, lint output, or external review links. + render: bash + validations: + required: false + - type: textarea + id: validated_scope + attributes: + label: Validated Scope + description: State what the integration supports and what it must not imply. + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1ce91d81..c4487ce6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,6 +20,12 @@ - [ ] Large generated datasets or scratch outputs are not committed. - [ ] No real FASTQ/BAM/BCL/customer assay data is committed; examples are synthetic, minimized, or public and scoped. +## Public Language and Reviewer Readiness + +- [ ] This PR does not imply accepted external workflow integration. +- [ ] Broad replacement wording or launch copy was removed or avoided. +- [ ] Public-facing claims link to a checked artifact, gate, or scoped status record. + ## Acceptance Criteria - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 911fe3dc..ec2f7a0b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -113,10 +113,10 @@ jobs: run: docker build -t dotmatch:ci . - name: Smoke test container run: | - docker run --rm dotmatch:ci --version | grep '^dotmatch 0.1.8$' + docker run --rm dotmatch:ci --version | grep '^dotmatch 0.1.9$' docker run --rm dotmatch:ci dist ACGT AGGT | grep '^1$' docker run --rm dotmatch:ci leq 1 ACGT AGGT | grep '^true$' - docker image inspect dotmatch:ci --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' | grep '^0.1.8$' + docker image inspect dotmatch:ci --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' | grep '^0.1.9$' - uses: docker/metadata-action@v6 id: meta with: diff --git a/.zenodo.json b/.zenodo.json index 44394c84..282ef8f1 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,7 +1,7 @@ { "title": "DotMatch: deterministic known-target short-DNA assignment for sequencing workflows", "upload_type": "software", - "version": "0.1.8", + "version": "0.1.9", "conceptdoi": "10.5281/zenodo.20541628", "creators": [ { diff --git a/CITATION.cff b/CITATION.cff index f4c5043c..c17296c3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,7 +8,7 @@ authors: orcid: "https://orcid.org/0009-0003-5012-7229" repository-code: "https://github.com/dnncha/dotmatch" license: Apache-2.0 -version: "0.1.8" +version: "0.1.9" doi: 10.5281/zenodo.20541628 abstract: "DotMatch is a deterministic known-target short-DNA assignment engine for CRISPR guide counting, barcode demultiplexing, and fixed-target FASTQ workflows." keywords: diff --git a/Dockerfile b/Dockerfile index dec6d930..c93415bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ LABEL org.opencontainers.image.title="DotMatch" \ org.opencontainers.image.source="https://github.com/dnncha/dotmatch" \ org.opencontainers.image.url="https://dotmatch.readthedocs.io/" \ org.opencontainers.image.documentation="https://dotmatch.readthedocs.io/" \ - org.opencontainers.image.version="0.1.8" \ + org.opencontainers.image.version="0.1.9" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.authors="Donncha O'Toole" diff --git a/Makefile b/Makefile index 733ca826..62cbeee2 100644 --- a/Makefile +++ b/Makefile @@ -454,7 +454,7 @@ repository-ready: python3 scripts/check_evidence_gallery.py $(MAKE) docs-ready -release-ready: python-test python-package-test docs-ready scientific-readiness-ready assay-evidence-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready evidence-gallery-ready distribution-record-ready bioconda-recipe-ready gpu-evidence-gate native-exact-gate public-crispr-evidence-gate crispr-comparison-gate barcode-comparison-gate feature-barcode-public-gate perturb-seq-public-gate amplicon-panel-public-gate bcl-tiny-public-gate oligo-adapter-public-gate +release-ready: python-test python-package-test docs-ready scientific-readiness-ready assay-evidence-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready evidence-gallery-ready distribution-record-ready bioconda-recipe-ready gpu-evidence-gate native-exact-gate public-crispr-evidence-gate crispr-comparison-gate barcode-comparison-gate feature-barcode-public-gate perturb-seq-public-gate amplicon-panel-public-gate bcl-tiny-public-gate oligo-adapter-public-gate reviewer-readiness-ready adoption-metrics-ready python3 scripts/check_release_readiness.py pretag-ready: @@ -498,6 +498,14 @@ workflow-integration-test: dotmatch shared workflow-adoption-status: python3 scripts/check_workflow_adoption.py +.PHONY: reviewer-readiness-ready +reviewer-readiness-ready: + python3 scripts/check_reviewer_readiness_assets.py + +.PHONY: adoption-metrics-ready +adoption-metrics-ready: + python3 scripts/check_adoption_metrics.py + distribution-record-ready: python3 scripts/check_distribution_record.py diff --git a/README.md b/README.md index 975c1510..20e6cb81 100644 --- a/README.md +++ b/README.md @@ -45,16 +45,32 @@ scoped handoff documents: - [Homepage](https://dnncha.github.io/dotmatch): short positioning and audience routes for known-target sequencing assignment. -- [Industry Exposure Kit](docs/industry-exposure.md): the five highest-leverage - adoption moves, copy-paste outreach, and claim guardrails. -- [Next 10 Industry Exposure Wins](docs/industry-next-wins.md): decision tree, +- [Outreach and Integration Kit](docs/industry-exposure.md): the five + highest-leverage distribution moves, copy-paste outreach, and claim guardrails. +- [Next 10 Distribution Actions](docs/industry-next-wins.md): decision tree, persona one-pagers, integration tracker, reviewer packet, outreach copy, pilot scorecard, KPIs, and release communication calendar. +- [Bioinformatics Evaluation Packet](docs/bioinformatics-evaluation.md): + installation, evidence boundaries, evaluation commands, and output review + criteria for independent technical reviewers. +- [Pilot Program](docs/pilot-program.md): a reproducible intake, migration, + review, and public-use process for external laboratories. +- [Workflow Integration Roadmap](docs/workflow-integration-roadmap.md): the + maintainer handoff plan for nf-core, MultiQC, Galaxy/IUC, Snakemake, and + bio.tools. +- [Workflow Integration Kit](docs/workflow-integration-kit.md): the concrete + handoff assets, launch checklist, and reviewer rules for maintainers. +- [Reviewer Readiness Record](docs/reviewer-readiness.json) and + [Integration Target Tracker](docs/integration-targets.json): machine-readable + records that keep planned ecosystem work separate from accepted adoption. +- [Adoption Metrics](docs/adoption-metrics.json): a measurement contract that + separates package distribution health from completed evaluations and accepted + workflow integrations. - [Workflow Submission Pack](docs/workflow-submissions.md): nf-core, MultiQC, Galaxy, and Snakemake handoff checklist. - [Methods and Citation](docs/methods-and-citation.md): copyable language for reports, manuscripts, and release-specific citation. -- [Adopter Notes](docs/adopters/README.md): rules for public, quote-approved +- [Adopter Notes](docs/adopters/README.md): rules for public, approved used-by records. ## DotMatch Pro @@ -252,11 +268,13 @@ documented scope are especially welcome. ## Installation -DotMatch 0.1.8 is published on PyPI for Linux and macOS. The PyPI package -includes the `dotmatch` command, Python imports, and the bundled native library. +DotMatch 0.1.9 is the current release target. After the tagged release workflow +publishes the package and `make distribution-channels` verifies the channel, +the PyPI package includes the `dotmatch` command, Python imports, and the bundled +native library. ```bash -python3 -m pip install dotmatch==0.1.8 +python3 -m pip install dotmatch==0.1.9 dotmatch --version dotmatch dist ACGT AGGT ``` @@ -289,14 +307,14 @@ docker build -t dotmatch:dev . docker run --rm -v "$PWD:/work" dotmatch:dev dist ACGT AGGT ``` -Bioconda is the Conda-based bioinformatics install path. DotMatch 0.1.8 package -metadata is visible on Anaconda's Bioconda channel for `linux-64`, `osx-64`, -and `osx-arm64`, including Apple Silicon Macs. Treat the install command as -fully released only after `make distribution-channels` verifies that Bioconda +Bioconda is the Conda-based bioinformatics install path. The 0.1.9 recipe update +keeps `linux-64`, `osx-64`, and `osx-arm64` support, including Apple Silicon +Macs. Treat the install command as released only after the Bioconda recipe +update is accepted and `make distribution-channels` verifies that Bioconda repodata and a clean `conda create` both resolve the package: ```bash -conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.1.8 +conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.1.9 conda activate dotmatch dotmatch --version ``` @@ -308,21 +326,22 @@ in [Packaging Notes](docs/packaging.md), the available for a release after `make distribution-channels` verifies public metadata and install smoke tests. -The tagged release workflow published the 0.1.8 source distribution, native +The tagged release workflow publishes the 0.1.9 source distribution, native macOS wheel, and repaired manylinux/musllinux Linux wheels. PyPI trusted publishing is configured for that workflow. The GitHub release workflow builds and smoke-tests repaired manylinux/musllinux wheels before upload. PyPI wheel -availability includes macOS, manylinux, and musllinux artifacts. The 0.1.8 -release files are visible on PyPI; the full multi-channel release record remains -open until `make distribution-channels` verifies clean Bioconda installs, -BioContainers propagation, GHCR runtime behavior, and DOI evidence. Raw +availability should include macOS, manylinux, and musllinux artifacts; release +files are visible on PyPI only after the tagged workflow publishes them. The +full multi-channel release record remains open until `make distribution-channels` +verifies clean Bioconda installs, BioContainers propagation, GHCR runtime +behavior, and DOI evidence. Raw `linux_x86_64` wheels remain GitHub release artifacts only and are not uploaded to PyPI. BioContainers publication is expected through the Bioconda automation rather -than a separate DotMatch container submission. After the accepted Bioconda 0.1.8 +than a separate DotMatch container submission. After the accepted Bioconda 0.1.9 package is converted by BioContainers, the expected image tag shape is -`quay.io/biocontainers/dotmatch:0.1.8--`. +`quay.io/biocontainers/dotmatch:0.1.9--`. Bioconda provides the `dotmatch` command-line tool, Python workflow namespaces, Python imports, and C header/library artifacts for the published package @@ -383,6 +402,25 @@ mode. ## CRISPR Guide Counting +For the fastest safe evaluation, use the reviewable one-command path: + +```bash +dotmatch crispr quickstart \ + --library guides.csv \ + --fastq 'fastqs/*.fastq.gz' \ + --out crispr_screen/ +``` + +This infers the guide window, creates an AssaySpec project, and writes a +self-contained project for review. By default it leaves the inferred spec in +draft form. For an immediate, explicit run, pass `--accept-inference` on the +initial command; otherwise review the inference report, set `status = "ready"`, +and run `dotmatch assay start crispr_screen/assay.toml`. `--no-run` is an +explicit review-only form. Inputs are copied into the project +so the generated run remains portable. The command is a convenience layer over +the same `dotmatch assay start` path used by workflow examples and production +handoffs. + The default production path scaffolds a reviewable assay project, runs preflight `check`, counts guides, and writes a reliability report with suggested `assay.toml` fixes when QC thresholds fail. @@ -762,7 +800,7 @@ A short JOSS software-paper draft is available in [paper/paper.md](paper/paper.m @software{dotmatch_software, author = {{O'Toole}, Donncha}, title = {{DotMatch: deterministic known-target short-DNA assignment for sequencing workflows}}, - version = {0.1.8}, + version = {0.1.9}, date = {2026-06-04}, publisher = {Zenodo}, doi = {10.5281/zenodo.20541628}, diff --git a/app/page.tsx b/app/page.tsx index 0b30b0d2..91b8f904 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -7,6 +7,13 @@ const exposureUrl = `${repoUrl}/blob/main/docs/industry-exposure.md`; const nextWinsUrl = `${repoUrl}/blob/main/docs/industry-next-wins.md`; const workflowSubmissionsUrl = `${repoUrl}/blob/main/docs/workflow-submissions.md`; const adoptersUrl = `${repoUrl}/blob/main/docs/adopters/README.md`; +const evaluationUrl = `${repoUrl}/blob/main/docs/bioinformatics-evaluation.md`; +const pilotUrl = `${repoUrl}/blob/main/docs/pilot-program.md`; +const reviewPacketUrl = `${repoUrl}/blob/main/docs/external-review-packet.md`; +const integrationRoadmapUrl = `${repoUrl}/blob/main/docs/workflow-integration-roadmap.md`; +const integrationTargetsUrl = `${repoUrl}/blob/main/docs/integration-targets.json`; +const reviewerReadinessUrl = `${repoUrl}/blob/main/docs/reviewer-readiness.json`; +const adoptionMetricsUrl = `${repoUrl}/blob/main/docs/adoption-metrics.md`; const pypiUrl = "https://pypi.org/project/dotmatch/"; const basePath = process.env.NEXT_PUBLIC_BASE_PATH ?? ""; @@ -155,12 +162,12 @@ const exposureActions = [ { title: "Pilot", body: - "Record quote-approved external pilots only after a public lab, workflow, or package integration can be linked and reviewed." + "Record approved external pilots only after a public lab, workflow, or package integration can be linked and reviewed." }, { title: "Share", body: - "Use the industry exposure kit for conference abstracts, repository announcements, short social copy, and direct maintainer outreach." + "Use the outreach and integration kit for conference abstracts, repository announcements, short social copy, and direct maintainer outreach." } ] as const; @@ -195,7 +202,7 @@ export default function Home() { Audiences Evidence Adoption - Next wins + Next actions Install GitHub @@ -360,8 +367,15 @@ export default function Home() { ))} @@ -380,7 +394,7 @@ export default function Home() {

-
    +
      {nextWins.map((win) => (
    1. {win}
      2. ))} @@ -390,7 +404,7 @@ export default function Home() {

      The playbook and machine-readable tracker must stay aligned before release. Private outreach and unmerged PRs remain activity, not - adoption evidence. + an accepted external-use record.

      Open the next 10 playbook diff --git a/codemeta.json b/codemeta.json index 150382bf..97b81c35 100644 --- a/codemeta.json +++ b/codemeta.json @@ -9,8 +9,8 @@ "issueTracker": "https://github.com/dnncha/dotmatch/issues", "license": "https://spdx.org/licenses/Apache-2.0", "identifier": "https://doi.org/10.5281/zenodo.20541628", - "version": "0.1.8", - "softwareVersion": "0.1.8", + "version": "0.1.9", + "softwareVersion": "0.1.9", "programmingLanguage": [ "C", "Python", diff --git a/docs/adopters/README.md b/docs/adopters/README.md index f9ea541a..76befe7d 100644 --- a/docs/adopters/README.md +++ b/docs/adopters/README.md @@ -1,8 +1,8 @@ # DotMatch Used By and Adopter Notes -This directory is reserved for quote-approved, independent adoption records and -the public "Used by" page. Do not count private emails, informal calls, or -in-repository examples as external adoption evidence. +This directory is reserved for approved, independent use records and the public +"Used by" page. Do not count private emails, informal calls, or in-repository +examples as an accepted external-use record. When an external workflow, lab, or package maintainer approves a public record, add: diff --git a/docs/adopters/record-template.md b/docs/adopters/record-template.md new file mode 100644 index 00000000..e229a4ea --- /dev/null +++ b/docs/adopters/record-template.md @@ -0,0 +1,38 @@ +# DotMatch Public Use Record Template + +Use this template after a lab, workflow maintainer, package maintainer, or +project has approved a public DotMatch use record. + +## Required Fields + +- Adopter or project name: +- Public URL: +- Evidence URL: +- Approval date: +- DotMatch version: +- Workflow scope: +- Outputs used: +- What this record supports: +- What this record does not support: +- Approval contact or source: + +## Suggested Record Shape + +```markdown +# + +- Public URL: +- Evidence URL: +- Approval date: +- DotMatch version: +- Scope: +- Outputs used: + +## Scope Notes + +This record documents . It does not imply clinical, +diagnostic, workflow-replacement, or private-data claims. +``` + +After adding a record, update `docs/workflow-adoption.json` only if the record +is an accepted external workflow integration covered by that schema. diff --git a/docs/adoption-metrics.json b/docs/adoption-metrics.json new file mode 100644 index 00000000..4335d28f --- /dev/null +++ b/docs/adoption-metrics.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "status": "instrumentation_ready", + "north_star": { + "id": "completed_external_evaluations_30d", + "definition": "Count of independent teams that complete the DotMatch evaluation protocol in a rolling 30-day window.", + "target_next_90_days": 5, + "why": "Completed evaluations measure real workflow use; package downloads alone do not." + }, + "metrics": [ + { + "id": "anaconda_downloads_6m", + "name": "Anaconda package downloads", + "role": "distribution_health", + "definition": "Cumulative downloads reported for the bioconda/dotmatch package over six months.", + "source": "https://api.anaconda.org/package/bioconda/dotmatch", + "cadence": "monthly", + "target": "Establish a baseline; never use this metric as a standalone adoption claim.", + "limitation": "Downloads do not identify active projects, unique teams, successful runs, or retention." + }, + { + "id": "pypi_downloads_30d", + "name": "PyPI downloads", + "role": "distribution_health", + "definition": "Download count for the current package version over the most recent 30-day window.", + "source": "https://pypistats.org/packages/dotmatch", + "cadence": "monthly", + "target": "Establish a baseline and compare with completed evaluations.", + "limitation": "Mirrors, CI jobs, and repeated installs can inflate the count." + }, + { + "id": "completed_external_evaluations_30d", + "name": "Completed external evaluations", + "role": "north_star", + "definition": "Teams that complete the intake, run, output review, and scorecard in docs/pilot-program.md.", + "source": "Approved pilot scorecards; private records may be summarized without exposing assay data.", + "cadence": "monthly", + "target": "5 completed evaluations in the first 90-day push.", + "limitation": "Requires an approved record or auditable private summary; do not infer from clicks." + }, + { + "id": "repeat_workflows_90d", + "name": "Repeat workflows", + "role": "retention", + "definition": "Independent teams that run DotMatch in at least two distinct workflows within 90 days.", + "source": "Approved pilot scorecards and docs/workflow-adoption.json.", + "cadence": "quarterly", + "target": "2 independent teams in the first 90-day push.", + "limitation": "A repeat workflow must be recorded with scope; an unmerged example is not evidence." + }, + { + "id": "accepted_external_integrations", + "name": "Accepted external integrations", + "role": "ecosystem", + "definition": "Workflow, registry, or reporting integrations accepted outside the DotMatch repository.", + "source": "docs/workflow-adoption.json with public adoption and evidence URLs.", + "cadence": "release_cycle", + "target": "1 accepted integration in the first 90-day push.", + "limitation": "Local wrapper payloads and open issues remain planned until accepted externally." + }, + { + "id": "docs_to_install_intent", + "name": "Documentation-to-install intent", + "role": "funnel", + "definition": "Unique visitors reaching a documented install or quickstart command divided by tutorial visitors.", + "source": "Opt-in documentation or homepage analytics, when enabled.", + "cadence": "monthly", + "target": "Establish a baseline before setting a conversion target.", + "limitation": "Analytics are optional; no user-level tracking is required for DotMatch to operate." + } + ], + "decision_rules": [ + "Report distribution health separately from external use, retention, and integration acceptance.", + "Never convert a package download count into a unique-user, active-project, or production-use claim.", + "A public use statement requires an approved record, a public URL, and a scope note.", + "Review the north-star count monthly and update the target only after the baseline is recorded." + ] +} diff --git a/docs/adoption-metrics.md b/docs/adoption-metrics.md new file mode 100644 index 00000000..3a2dae5c --- /dev/null +++ b/docs/adoption-metrics.md @@ -0,0 +1,30 @@ +# Adoption Metrics + +DotMatch reports distribution health separately from evidence of real workflow +use. The machine-readable contract is [`adoption-metrics.json`](adoption-metrics.json) +and is checked by `make adoption-metrics-ready`. + +## What the numbers mean + +- Anaconda and PyPI downloads are package-distribution signals. They include + mirrors, CI, repeated installs, and automated jobs; they do not identify + active teams or successful runs. +- The north-star metric is completed independent evaluations using the intake, + run, output review, and scorecard in [`pilot-program.md`](pilot-program.md). +- Retention is measured by repeat workflows, not by a second download. +- Ecosystem progress is an accepted external integration recorded in + [`workflow-adoption.json`](workflow-adoption.json), with public evidence URLs. + +## Monthly review + +1. Record the Anaconda and PyPI distribution counts and the date collected. +2. Count completed evaluation scorecards, keeping private assay data out of the + repository. +3. Count repeat workflows and accepted external integrations from their source + records. +4. Compare the results with the rolling 90-day targets in the JSON contract. +5. Update positioning or onboarding only when the completed-evaluation pattern + identifies a concrete friction point. + +The project should not publish a unique-user, active-project, production-use, +or market-share claim from package download counts alone. diff --git a/docs/benchmarks/performance-improvements-2026-07-05.md b/docs/benchmarks/performance-improvements-2026-07-05.md new file mode 100644 index 00000000..5eb2dd12 --- /dev/null +++ b/docs/benchmarks/performance-improvements-2026-07-05.md @@ -0,0 +1,82 @@ +# DotMatch Performance Improvements - 2026-07-05 + +This benchmark note covers practical performance improvements in the current +codebase: Python callers were not reaching native exact/Hamming kernels, several +higher-level Python workflows ignored their Hamming metric, and the threaded +native exact-count path mishandled offset windows. + +## Implemented Improvements + +1. Bound `qdaln_index_assign_hamming_stats` in the Python `ctypes` layer. +2. Bound `qdaln_index_lookup_exact_many_stats` in the Python `ctypes` layer. +3. Bound `qdaln_index_lookup_exact_ascii_many_stats` in the Python `ctypes` layer. +4. Bound `qdaln_index_assign_status_stats` in the Python `ctypes` layer. +5. Added top-level `dotmatch.assign_hamming(...)`. +6. Added top-level `dotmatch.assign_exact(...)`. +7. Added `Matcher.assign_hamming(...)`. +8. Added `Matcher.assign_hamming_with_stats(...)`. +9. Added `Matcher.assign_exact(...)`. +10. Added `Matcher.assign_exact_with_stats(...)`. +11. Added `Matcher.assign_status_with_stats(...)` for status-only early-stop use cases. +12. Routed `assign_dataframe(..., metric="hamming")` through the Hamming kernel. +13. Routed `assign_dataframe(..., metric="exact")` through the exact lookup table. +14. Routed `stream_assign(..., metric="hamming")` through the Hamming kernel. +15. Routed `stream_assign(..., metric="exact")` through the exact lookup table. +16. Routed `dotmatch.tl.assign_features(..., metric="hamming"|"exact")` through the native fast paths. +17. Routed `dotmatch.tl.feature_counts(..., metric="hamming"|"exact")` through the native fast paths. +18. Switched AssaySpec fixed-window start scoring to the Hamming kernel. +19. Replaced streamed assignment TSV `DictWriter` rows with direct fixed-column writes. +20. Fixed threaded native Hamming `k=0` offset-window counting so batch mode evaluates extracted windows, records totals, and falls back to the threaded direct worker for large buffers. +21. Replaced the generic exact-assignment fallback in native offset detection with the exact ASCII lookup path. +22. Routed `inspect-unmatched --k 0` primary fixed-window assignment through exact ASCII lookup. +23. Routed `inspect-unmatched --offset-window ... --k 0` shifted-window hints through exact ASCII lookup. +24. Changed `assignment_summary(...)` to accumulate local integer counters instead of mutating a dictionary per row. +25. Changed `write_assignments_tsv(...)` to keep local summary counters while writing rows. + +## Benchmarks + +Hardware/environment: local macOS build, `make dotmatch`, `PYTHONPATH=python`, +local `libdotmatch.dylib`, synthetic fixed-window DNA workloads. + +### Python matcher, 100,000 reads, 4,096 targets, length 20 + +| Path | Seconds | Reads/sec | Notes | +| --- | ---: | ---: | --- | +| `Matcher.assign_with_stats(..., k=1)` | 0.367143 | 272,374 | General Levenshtein path | +| `Matcher.assign_hamming_with_stats(..., k=1)` | 0.200717 | 498,214 | 1.83x faster | +| `Matcher.assign_exact_with_stats(...)` | 0.194195 | 514,946 | 1.89x faster for exact windows | +| `Matcher.assign_status_with_stats(..., k=1)` | 0.207492 | 481,946 | 1.77x faster when lower-bound ambiguity details are acceptable | + +### Python streaming FASTQ, 200,000 reads, 1,024 targets, length 20 + +| Path | Seconds | Reads/sec | Notes | +| --- | ---: | ---: | --- | +| `stream_assign(..., metric="levenshtein", k=1)` | 1.666834 | 119,988 | Previous default route | +| `stream_assign(..., metric="hamming", k=1)` | 1.323489 | 151,116 | 1.26x faster with identical assignments for this substitution-only workload | + +### Native threaded exact count offset regression + +Workload: 200,000 reads, target window starts at offset 5, target length 20, +Hamming `k=0`, 1,024 targets. Before this pass, `--threads 4` fed whole reads +to exact lookup and reported `total_reads=0`, `assigned_unique=0`, +`unmatched=200000`. + +| Path | Count phase seconds | Total reads | Assigned exact | Unmatched | +| --- | ---: | ---: | ---: | ---: | +| `--threads 1` | 0.214056 | 200000 | 190000 | 10000 | +| `--threads 4` | 0.206341 | 200000 | 190000 | 10000 | + +The threaded path now matches single-thread counts and is slightly faster on the +counting phase for this workload. + +### Additional exact-window and streaming-summary checks + +These checks compare the patched tree against a temporary baseline binary built +from the checked-in pre-change `src/qda.c`. Outputs were compared byte-for-byte +for the native commands. + +| Workload | Baseline seconds | Patched seconds | Impact | +| --- | ---: | ---: | --- | +| `count --metric levenshtein --k 0 --auto-offset 8`, 300,000 reads, 4,096 targets | 0.763317 | 0.684115 | 1.12x faster | +| `inspect-unmatched --k 0`, 300,000 reads, 4,096 targets | 0.118202 | 0.116536 | 1.01x faster | +| `assignment_summary(...)`, 1,000,000 streamed rows | 0.401176 | 0.152804 | 2.63x faster | diff --git a/docs/bioinformatics-evaluation.md b/docs/bioinformatics-evaluation.md new file mode 100644 index 00000000..81942b5c --- /dev/null +++ b/docs/bioinformatics-evaluation.md @@ -0,0 +1,138 @@ +# Bioinformatics Evaluation Packet + +This packet is for bioinformatics teams, core facilities, workflow maintainers, +and assay-development groups evaluating whether DotMatch is appropriate for a +known-target sequencing workflow. It records what is available now, what is +validated by repository checks, and what should not be inferred from the current +release. + +DotMatch should be evaluated as a local command-line and Python package for +deterministic assignment of fixed read windows to known short DNA targets. It is +not a genome aligner, basecaller, variant caller, adapter trimmer, cell/UMI +quantifier, or screen-level statistical analysis package. + +## Current Package Surface + +For release target `0.1.9`, the package surface is: + +| Surface | Current state | Evidence or source | +| --- | --- | --- | +| PyPI | Prepared for tagged release publication and install smoke tests | `docs/distribution-release.json` | +| Bioconda | Prepared recipe template with `linux-64`, `osx-64`, and `osx-arm64` support | `docs/distribution-release.json` | +| GHCR container | Prepared for tagged image publication and runtime smoke tests | `docs/distribution-release.json` | +| BioContainers | Prepared; propagation depends on the accepted Bioconda recipe | `docs/distribution-release.json` | +| Documentation | Sphinx docs and public schemas in repository | `docs/index.md`, `docs/schemas.md` | +| Citation | `CITATION.cff`, Zenodo concept DOI, and generated run artifacts | `docs/methods-and-citation.md` | +| Python API | `dotmatch` package, streaming helpers, pandas/polars/AnnData interop, MultiQC parser entry point | `pyproject.toml`, `docs/streaming-api.md` | +| R interface | Reticulate-backed package skeleton and vignette | `R/`, `vignettes/dotmatch.Rmd` | + +The package should not be described as published or verified on a public channel +until `make distribution-channels` passes for that channel and the release record +is updated. + +## Validated Assay Scope + +Use `docs/assay-evidence.json` and `docs/scientific-claims.md` as the source of +truth. A short evaluator summary: + +| Assay or workflow area | Current public status | Boundary | +| --- | --- | --- | +| CRISPR guide counting | Supported for checked public lanes and comparator semantics | Not screen-level effect analysis | +| Fixed-position inline barcode demultiplexing | Supported for checked SRP009896 exact-prefix and fixed-length Hamming lanes | Not arbitrary adapter trimming or raw BCL replacement | +| Feature-barcode assignment | Supported for checked per-read 10x TotalSeq-B fixed-window assignment | Not Cell Ranger feature-matrix parity | +| Perturb-seq guide capture | Gated fixed-window per-read evidence | Not guide-per-cell quantification or perturbation-effect inference | +| Amplicon or panel primer-start assignment | Supported for checked fixed-window ARTIC primer-start lane | Not consensus generation, variant calling, or clinical interpretation | +| Oligo or adapter-prefix assignment | Supported for checked fixed-window adapter-prefix assignment | Not adapter trimming, read merging, or UMI grouping | +| Classic BCL milestone | Narrow checked tiny-BCL milestone | Not production Illumina demultiplexing or CBCL/NovaSeq support | + +If a statement would affect a procurement decision, manuscript claim, pipeline +replacement, or regulated workflow, link it to the relevant raw artifact, +benchmark report, and gate. + +## Minimum Local Evaluation + +Start from the released package, not an unpublished checkout, unless the +evaluation is explicitly for development work: + +```bash +python3 -m pip install dotmatch==0.1.9 +dotmatch --version +dotmatch dist ACGT AGGT +``` + +For Conda-based environments: + +```bash +conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.1.9 +conda activate dotmatch +dotmatch --version +``` + +For a first source-level review: + +```bash +make test +make cli-test +make python-test +make workflow-examples-ready +make repository-ready +``` + +For scope-sensitive review, run the specific evidence gates named in +`docs/methods-and-citation.md` and `docs/scientific-claims.md`; do not infer +broader assay coverage from a generic test pass. + +## Outputs To Inspect + +A useful evaluation should inspect the files a lab or workflow system would +actually receive: + +- `sample_qc.tsv`: assignment rate, ambiguous reads, unmatched reads, invalid + windows, rescue counts, and representation fields. +- `summary.json`: command, version, policy, target-window configuration, and + provenance. +- `assignments.tsv`: per-read states when requested. +- `top_unmatched.tsv`: recurring unmatched sequences for assay review. +- `assay_report.html` or workflow report HTML: human-readable reliability + review. +- `methods.md`, `CITATION.bib`, and `software_versions.yml`: methods and + software citation artifacts produced by AssaySpec run paths. +- `assay_manifest.summary.tsv`: workflow-facing contract for run artifacts. + +The central contract is that `ambiguous`, `none`, and `invalid` outcomes remain +visible instead of being silently collapsed into target counts. + +## Workflow Integration Status + +`docs/workflow-adoption.json` is currently `not_ready` because no external +workflow-manager integration has been accepted yet. Local examples and upstream +payloads are available for review, but external workflow status requires an +accepted PR, package, ToolShed record, wrapper, or public pipeline outside this +repository. + +Priority integration targets remain: + +1. nf-core modules. +2. Released or upstreamed MultiQC module. +3. Galaxy/IUC wrappers. +4. Snakemake wrapper or public lab workflow. +5. bio.tools registry record. + +Record accepted integrations only in `docs/workflow-adoption.json`. Public use +records require approved wording and a public URL. + +## Language Rules For Public Surfaces + +Use plain technical language: + +- say "known-target assignment", not broad "sequencing analysis"; +- say "checked public lane", "gated", or "experimental" when evidence is + narrow; +- say "workflow artifact", "schema", "gate", "report", and "command" when + there is a concrete file or check; +- avoid broad replacement claims, superlatives, and launch copy; +- do not imply external workflow integration before + `docs/workflow-adoption.json` records an accepted public integration. + +The project should feel like a bioinformatics package first: installable, +auditable, scoped, reproducible, and cautious about unsupported claims. diff --git a/docs/citation-flywheel.md b/docs/citation-flywheel.md index 8a24fe41..6051d1dd 100644 --- a/docs/citation-flywheel.md +++ b/docs/citation-flywheel.md @@ -15,15 +15,15 @@ listed in `docs/scientific-claims.md`. | P1 | Generate run citation artifacts | Done in AssaySpec runs: each production run writes `methods.md`, `CITATION.bib`, and `software_versions.yml` beside summary and report outputs. | | P1 | Add copyable methods and citation UI | Workbench and generated HTML reports expose a single "Copy methods and citation" action backed by the generated methods artifact. | | P1 | Recruit documented external pilots | Three to five independent labs, cores, or workflow maintainers approve public pilot records with scope notes. | -| P1 | Maintain the industry exposure kit | Homepage audience routes, outreach copy, workflow handoffs, citation links, and adopter-record rules stay aligned in `docs/industry-exposure.md`. | -| P1 | Execute the next 10 exposure wins | `docs/industry-next-wins.md`, `docs/industry-exposure-plan.json`, and the homepage "Next 10 exposure wins" section stay aligned. | +| P1 | Maintain the outreach and integration kit | Homepage audience routes, outreach copy, workflow handoffs, citation links, and adopter-record rules stay aligned in `docs/industry-exposure.md`. | +| P1 | Execute the next 10 distribution actions | `docs/industry-next-wins.md`, `docs/industry-exposure-plan.json`, and the homepage action section stay aligned. | | P2 | Publish a methods and benchmark preprint | The manuscript demonstrates speed and the consequences of ambiguity, unsafe rescue, and incorrect barcode windows. | -| P2 | Maintain a public used-by page | Quote-approved independent examples are listed in `docs/adopters/` and mirrored in the public docs. | +| P2 | Maintain a public used-by page | Approved independent examples are listed in `docs/adopters/` and mirrored in the public docs. | ## Operating Rules -- Do not count private conversations, internal examples, or unmerged PRs as - adoption evidence. +- Do not count private conversations, internal examples, or unmerged PRs as an + accepted external-use record. - Record external integrations in `docs/workflow-adoption.json` only after they are accepted, released, or published outside this repository. - Keep citation text generated from checked project metadata, not manually @@ -37,7 +37,7 @@ listed in `docs/scientific-claims.md`. [Workflow Submission Pack](workflow-submissions.md). - Methods and citation language: [Methods and Citation Template](methods-and-citation.md). -- Quote-approved adoption records: +- Approved external-use records: [DotMatch Adopter Notes](adopters/README.md). - Evidence limits for public claims: [DotMatch Evidence Notes](scientific-claims.md). diff --git a/docs/conf.py b/docs/conf.py index 85a01869..1c592cfa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = "DotMatch" author = "Donncha O'Toole" copyright = "2026, Donncha O'Toole" -release = "0.1.8" +release = "0.1.9" version = release extensions = [ diff --git a/docs/distribution-release.json b/docs/distribution-release.json index 15c268e4..617fb6cc 100644 --- a/docs/distribution-release.json +++ b/docs/distribution-release.json @@ -1,64 +1,56 @@ { "schema_version": 1, - "status": "partially_verified", - "release_version": "0.1.8", + "status": "not_released", + "release_version": "0.1.9", "post_release_gate": "make distribution-channels", "channels": [ { "id": "pypi", - "status": "verified", - "expected_url": "https://pypi.org/project/dotmatch/0.1.8/", + "status": "prepared", + "expected_url": "https://pypi.org/project/dotmatch/0.1.9/", "verification_command": "make distribution-channels", - "public_url": "https://pypi.org/project/dotmatch/0.1.8/", - "evidence_url": "https://github.com/dnncha/dotmatch/actions/runs/27484079771", - "blocker": "", - "next_action": "Keep this channel verified by rerunning make distribution-channels before any release status change." + "blocker": "The 0.1.9 source distribution and repaired wheel artifacts have not been published by the tagged release workflow yet.", + "next_action": "Tag v0.1.9, let the GitHub release workflow publish the source distribution and repaired wheels, then rerun make distribution-channels." }, { "id": "bioconda", - "status": "verified", + "status": "prepared", "expected_url": "https://anaconda.org/bioconda/dotmatch", - "latest_published_version": "0.1.8", + "latest_published_version": "0.1.9", + "observed_downloads_last_6_months": 0, + "observed_date": "2026-07-13", "initial_recipe_pr_url": "https://github.com/bioconda/bioconda-recipes/pull/65367", - "recipe_pr_url": "https://github.com/bioconda/bioconda-recipes/pull/66291", - "release_tarball_sha256": "ec3819bc773431454910287559d0809aca6ec1d81959f29d3d522650edb74904", - "public_url": "https://anaconda.org/bioconda/dotmatch", - "evidence_url": "https://github.com/bioconda/bioconda-recipes/pull/66291", - "platforms": ["linux-64", "osx-64", "osx-arm64"], "verification_command": "make distribution-channels", - "blocker": "", - "next_action": "Keep this channel verified by rerunning make distribution-channels before any release status change." + "blocker": "The public package page shows 0.1.9, but this record still needs immutable source/SHA256 and clean repodata-install evidence before calling the channel verified.", + "next_action": "Capture the v0.1.9 source SHA256, verify repodata and a clean conda create, then rerun make distribution-channels and update this channel record." }, { "id": "ghcr", - "status": "manifest_verified", + "status": "prepared", "expected_url": "https://github.com/dnncha/dotmatch/pkgs/container/dotmatch", "verification_command": "make distribution-channels", - "public_url": "https://github.com/dnncha/dotmatch/pkgs/container/dotmatch", - "evidence_url": "https://github.com/dnncha/dotmatch/actions/runs/27484079771", - "manifest_digest": "sha256:6c4634f9e61c15c289e1a4238cec341c69d3e2163080f4cf2ab3f9ebe5f50f67", - "blocker": "The v0.1.8 GHCR manifest is reachable, but this workstation does not have Docker, so runtime smoke tests could not be run by make distribution-channels.", - "next_action": "Run make distribution-channels on a host with Docker or another OCI runtime and verify ghcr.io/dnncha/dotmatch:v0.1.8 --version plus CLI distance smoke tests." + "blocker": "The 0.1.9 GHCR image has not been published and runtime-smoke-tested yet.", + "next_action": "Let the release workflow publish ghcr.io/dnncha/dotmatch:v0.1.9, then verify --version and CLI distance smoke tests with an OCI runtime." }, { "id": "biocontainers", "status": "prepared", "expected_url": "https://quay.io/repository/biocontainers/dotmatch", "verification_command": "make distribution-channels", - "blocker": "make distribution-channels can discover a BioContainers tag for 0.1.8, but this workstation does not have Docker, so manifest and runtime smoke tests could not be run.", - "next_action": "Run make distribution-channels on a host with Docker and verify the selected quay.io/biocontainers/dotmatch:0.1.8-- image with CLI distance and threshold smoke tests." + "blocker": "BioContainers 0.1.9 cannot propagate until the Bioconda recipe is accepted.", + "next_action": "After Bioconda publishes 0.1.9, verify the selected quay.io/biocontainers/dotmatch:0.1.9-- image with CLI distance and threshold smoke tests." }, { "id": "zenodo", - "status": "verified", + "status": "prepared", "expected_url": "https://zenodo.org/records/20541628", "verification_command": "make distribution-channels", - "blocker": "", - "next_action": "Keep this channel verified by rerunning make distribution-channels before any release status change." + "blocker": "The Zenodo archive metadata for v0.1.9 has not been verified yet.", + "next_action": "After v0.1.9 is tagged and archived, verify the Zenodo record metadata and DOI resolution, then rerun make distribution-channels." } ], "blockers": [ - "DotMatch 0.1.8 still needs Docker-backed runtime verification for GHCR and BioContainers on a host with an OCI runtime." + "DotMatch 0.1.9 has not been tagged, published to PyPI/GHCR/Zenodo, accepted by Bioconda, or propagated to BioContainers yet." ], - "next_action": "Run make distribution-channels on a host with Docker or another OCI runtime, verify GHCR and BioContainers runtime behavior, then set the overall release record to verified if the full post-release gate passes." + "next_action": "Run make pretag-ready, tag v0.1.9, monitor the release workflow, submit the Bioconda recipe update, and rerun make distribution-channels after public channels propagate." } diff --git a/docs/external-review-packet.md b/docs/external-review-packet.md new file mode 100644 index 00000000..6c2da0a2 --- /dev/null +++ b/docs/external-review-packet.md @@ -0,0 +1,87 @@ +# External Review Packet + +This packet is for maintainers, PIs, core-facility leads, and technical +reviewers who need a concise answer to: what is DotMatch today, what can be +reviewed, and what should not be claimed yet? + +## Review Summary + +DotMatch is a local command-line and Python package for deterministic +known-target assignment of short read windows. It records each read as +`unique`, `ambiguous`, `none`, or `invalid`, and writes workflow-readable TSV, +JSON, FASTQ, and HTML artifacts. + +The package is strongest today where the assay has: + +- a known target table; +- a fixed, scaffolded, or inferable read window; +- a need to preserve ambiguity, unmatched reads, invalid windows, and unsafe + correction states; +- downstream tools that can consume count tables, split FASTQs, QC tables, or + reports. + +It should not be reviewed as a genome aligner, basecaller, adapter trimmer, +variant caller, UMI/cell quantifier, or screen-level statistics package. + +## What To Check First + +| Review area | Start here | Notes | +| --- | --- | --- | +| Installation | `docs/bioinformatics-evaluation.md` | PyPI and Bioconda install checks are verified after public channel propagation; container runtime proof needs an OCI runtime. | +| Validated scope | `docs/scientific-claims.md` | Public claims must stay inside checked lanes and gates. | +| Output contracts | `docs/schemas.md` | TSV/JSON/HTML artifacts are intentionally plain for workflow systems. | +| Workflow handoff | `docs/workflow-submissions.md` | Local nf-core, MultiQC, Galaxy, and Snakemake assets exist. | +| Workflow status | `docs/workflow-adoption.json` | External workflow integration is listed only after acceptance outside this repository. | +| Registry status | `docs/registries/biotools.yml` | Draft metadata only; not an accepted bio.tools record. | +| Citation | `docs/methods-and-citation.md` | Use release-specific citation text and generated methods artifacts. | + +## Minimum Review Commands + +```bash +python3 -m pip install dotmatch==0.1.9 +dotmatch --version +dotmatch dist ACGT AGGT +``` + +For source review: + +```bash +make test +make cli-test +make python-test +make workflow-examples-ready +make reviewer-readiness-ready +make repository-ready +``` + +For a claim-sensitive review, run the assay-specific evidence gate named beside +the claim. A generic test pass is not enough to support a new scientific or +comparative statement. + +## Reviewer Questions + +Ask these before recommending DotMatch for a workflow: + +1. Does the assay have known short targets? +2. Is the read window fixed, scaffolded, or inferable? +3. Are ambiguous, unmatched, or invalid reads important to interpretation? +4. Are TSV, JSON, FASTQ, HTML, and methods/citation artifacts sufficient for the + receiving workflow? +5. Does the claim being made have a matching raw artifact, report, and gate? +6. Is any public adoption record accepted outside this repository? + +## Known Blockers To Keep Visible + +- `docs/workflow-adoption.json` is still `not_ready`; local examples are not + accepted external workflow integrations. +- GHCR and BioContainers runtime smoke tests still need a Docker or OCI host + before the release record can be fully verified. +- Broader BCL/CBCL, Cell Ranger-style quantification, adapter trimming, + variant calling, and screen-level statistical claims remain outside the + current evidence boundary. + +## Public Wording Rule + +Use the narrowest accurate wording. If a sentence would influence purchase, +publication, workflow replacement, or regulated use, it needs a checked artifact +and a scoped wording review. diff --git a/docs/getting-started.md b/docs/getting-started.md index 0c5d0872..21b467c1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -10,14 +10,14 @@ adapter prefixes. For the current PyPI release: ```bash -python3 -m pip install dotmatch==0.1.8 +python3 -m pip install dotmatch==0.1.9 dotmatch --version ``` -For Conda-based environments, Bioconda now publishes DotMatch 0.1.8: +For Conda-based environments, the 0.1.9 Bioconda recipe targets: ```bash -conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.1.8 +conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.1.9 conda activate dotmatch dotmatch --version ``` @@ -35,7 +35,8 @@ dotmatch --version The source build needs a C compiler, `make`, Python 3.9 or newer, and zlib for FASTQ.gz support. -Bioconda 0.1.8 metadata and clean install smoke tests have been verified. PyPI +The 0.1.9 release record tracks PyPI and Bioconda verification separately. Run +`make distribution-channels` before treating a public channel as verified. PyPI remains the simplest cross-platform Python install path; Bioconda is the preferred package-manager path for Conda-based bioinformatics environments. @@ -99,6 +100,29 @@ safe for correction should usually be counted exactly or redesigned. ## Count Known Targets From FASTQ +### Fastest safe CRISPR evaluation + +For a first CRISPR-screen evaluation, `crispr quickstart` turns one or more +FASTQ paths into the same reviewable AssaySpec project used by production runs. +It infers the guide window, writes an inference report, and copies the selected +FASTQs into a self-contained project. By default the project stays in draft +status for review: + +```bash +dotmatch crispr quickstart \ + --library guides.csv \ + --fastq 'fastqs/*.fastq.gz' \ + --out crispr_screen/ +``` + +Review `crispr_screen/inference_report.json`. Then set `status = "ready"` in +`assay.toml` and run `dotmatch assay start crispr_screen/assay.toml`. For an +immediate explicit run, pass `--accept-inference` on the initial quickstart +command. A non-zero result means the run completed but a +reliability gate needs review; inspect `assay_out/reliability_report.html` and +`assay_out/assay_fixes.tsv` before using counts downstream. `--no-run` is an +explicit review-only form. + Use `dotmatch count` when reads contain one fixed target window. ```bash diff --git a/docs/index.md b/docs/index.md index a2147f6c..256cabac 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,10 +34,13 @@ need their own gates before they are public claims. - Labs evaluating scientific claims should read [Trust, Scope, and Evidence](trust-and-scope.md). - Workflow and pipeline authors should use the [public output schemas](schemas.md). - Industry evaluators and maintainers should use the - [Industry Exposure Kit](industry-exposure.md) to route outreach, citations, + [Outreach and Integration Kit](industry-exposure.md) to route outreach, citations, pilots, and workflow submissions without broadening public claims. +- Independent reviewers should start with the + [Bioinformatics Evaluation Packet](bioinformatics-evaluation.md), then use + the [Pilot Program](pilot-program.md) for an external migration. - Maintainers pushing the next adoption layer should use the - [Next 10 Industry Exposure Wins](industry-next-wins.md) and the checked + [Next 10 Distribution Actions](industry-next-wins.md) and the checked `industry-exposure-plan.json` tracker. - Teams evaluating the open-core boundary should read [Commercial Boundary](commercial-boundary.md) and @@ -87,6 +90,12 @@ workflow-submissions resubmission-evidence commercial-boundary evidence-packet-v1 +bioinformatics-evaluation +external-review-packet +pilot-program +workflow-integration-kit +workflow-integration-roadmap +adoption-metrics ``` ```{toctree} diff --git a/docs/industry-exposure-plan.json b/docs/industry-exposure-plan.json index 34115e60..90367276 100644 --- a/docs/industry-exposure-plan.json +++ b/docs/industry-exposure-plan.json @@ -1,7 +1,7 @@ { "schema_version": 1, "status": "ready_to_execute", - "scope": "next 10 industry exposure wins", + "scope": "next 10 distribution actions", "items": [ { "id": "decision_tree", @@ -57,7 +57,7 @@ "title": "Pilot Scorecard", "primary_audience": "labs, cores, CROs, biotech teams", "asset": "docs/industry-next-wins.md#8-pilot-scorecard", - "done_when": "Pilot feedback can be scored without turning private evaluation into public adoption evidence." + "done_when": "Pilot feedback can be scored without turning private evaluation into an external-use record." }, { "id": "adoption_kpis", diff --git a/docs/industry-exposure.md b/docs/industry-exposure.md index fd2f11e6..d1e00a34 100644 --- a/docs/industry-exposure.md +++ b/docs/industry-exposure.md @@ -1,4 +1,4 @@ -# DotMatch Industry Exposure Kit +# DotMatch Outreach and Integration Kit This page turns the adoption roadmap into concrete marketing and distribution work. It is about reaching the right people without overstating claims. Public @@ -6,7 +6,7 @@ performance, correctness, packaging, and adoption statements still need the evidence gates in `docs/scientific-claims.md`, `docs/packaging.md`, and `docs/workflow-adoption.json`. -## The Big 5 Wins +## The Big 5 Distribution Moves | Win | Relevant people | Asset in this repository | Done when | | --- | --- | --- | --- | @@ -14,7 +14,7 @@ evidence gates in `docs/scientific-claims.md`, `docs/packaging.md`, and | Workflow distribution handoff | nf-core, MultiQC, Galaxy/IUC, Snakemake, institutional pipeline owners | `docs/workflow-submissions.md` | At least one external PR, package, or reviewed wrapper is public and tracked in `docs/workflow-adoption.json`. | | Citation and methods flywheel | PIs, methods writers, bioinformatics leads, paper authors | `docs/methods-and-citation.md`, `docs/citation-flywheel.md` | Runs can produce copyable methods and citation artifacts, and external users know how to cite the exact release. | | Evidence-first launch packet | Technical evaluators, skeptical industry users, procurement reviewers | Evidence gallery, benchmark docs, scientific claim boundary | Every public announcement links to scoped evidence instead of broad claims. | -| Public adopter record | Labs, cores, CROs, biotech, pharma teams, workflow projects | `docs/adopters/README.md` | Quote-approved independent use is listed only after the external artifact is public. | +| Public adopter record | Labs, cores, CROs, biotech, pharma teams, workflow projects | `docs/adopters/README.md` | Approved independent use is listed only after the external artifact is public. | ## Positioning @@ -137,7 +137,7 @@ Docs: https://dotmatch.readthedocs.io/ ## Next 10 Wins -The next layer is tracked in [DotMatch Next 10 Industry Exposure Wins](industry-next-wins.md) +The next layer is tracked in [DotMatch Next 10 Distribution Actions](industry-next-wins.md) and mirrored in `docs/industry-exposure-plan.json`. 1. Evaluator decision tree. @@ -153,10 +153,10 @@ and mirrored in `docs/industry-exposure-plan.json`. ## Tracking Rules -- Private feedback can inform the roadmap, but it is not adoption evidence. +- Internal review can inform the roadmap, but it is not an external-use record. - Unmerged PRs can be listed as outreach activity, but not as external adoption. - External use counts only when a public artifact can be linked. -- Quote-approved user names, organization names, and logos need explicit +- Approved user names, organization names, and logos need explicit approval before they appear in the repository or public site. - If a claim would influence purchase, publication, or pipeline replacement, it needs a checked artifact and a scoped wording review. diff --git a/docs/industry-next-wins.md b/docs/industry-next-wins.md index 28f3a555..e831b874 100644 --- a/docs/industry-next-wins.md +++ b/docs/industry-next-wins.md @@ -1,4 +1,4 @@ -# DotMatch Next 10 Industry Exposure Wins +# DotMatch Next 10 Distribution Actions This playbook builds on the first five exposure wins in `docs/industry-exposure.md`. It turns the next adoption layer into concrete @@ -7,9 +7,9 @@ CRISPR teams, and assay developers without expanding DotMatch's public claim boundary. Use `docs/industry-exposure-plan.json` as the machine-readable checklist for -these ten wins. Keep private conversations, unmerged PRs, and internal examples -out of public adopter evidence until the rules in `docs/adopters/README.md` are -satisfied. +these ten actions. Keep private conversations, unmerged PRs, and internal +examples out of public use records until the rules in `docs/adopters/README.md` +are satisfied. ## 1. Evaluator Decision Tree @@ -173,7 +173,7 @@ safety status without implying downstream biological pass/fail calls. ## 8. Pilot Scorecard -Use this privately during pilots. Publish only quote-approved summaries. +Use this privately during pilots. Publish only approved summaries. | Dimension | Score | Notes | | --- | --- | --- | @@ -187,8 +187,8 @@ Use this privately during pilots. Publish only quote-approved summaries. Interpretation: - 0-4: do not publicize; fix product or docs first. -- 5-8: useful private feedback; consider a follow-up pilot. -- 9-12: candidate for quote-approved adopter record. +- 5-8: useful internal review; consider a follow-up pilot. +- 9-12: candidate for an approved adopter record. ## 9. Adoption KPI Dashboard Spec @@ -204,8 +204,8 @@ Track exposure health separately from scientific evidence. | Citation artifacts generated | release or assay outputs | release cycle | | Distribution channel health | `make distribution-channels` | release cycle | -Do not combine these KPIs with performance or correctness claims. Exposure can -increase before external adoption is proven. +Do not combine these KPIs with performance or correctness claims. Interest can +increase before external use is proven. ## 10. Release Communications Calendar diff --git a/docs/integration-targets.json b/docs/integration-targets.json new file mode 100644 index 00000000..28384b5d --- /dev/null +++ b/docs/integration-targets.json @@ -0,0 +1,86 @@ +{ + "schema_version": 1, + "status": "planned", + "adoption_record": "docs/workflow-adoption.json", + "targets": [ + { + "id": "nf_core_modules", + "name": "nf-core modules", + "type": "nf_core_module", + "status": "local_payload_ready", + "external_target": "https://github.com/nf-core/modules", + "source_assets": [ + "examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/count/", + "examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/demux/", + "examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/audit/", + "examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/panel_check/", + "examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/crispr_count/", + "examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/assay_run/" + ], + "review_command": "make workflow-examples-ready", + "next_action": "Open or update an upstream nf-core modules PR after local workflow integration tests pass.", + "public_record_required": true + }, + { + "id": "multiqc_module", + "name": "MultiQC module", + "type": "multiqc_plugin", + "status": "local_parser_ready", + "external_target": "https://github.com/MultiQC/MultiQC", + "source_assets": [ + "python/dotmatch/multiqc.py", + "examples/workflows/multiqc/multiqc_config.yaml", + "examples/workflows/multiqc/data/" + ], + "review_command": "make workflow-examples-ready", + "next_action": "Package or upstream the module after public workflow maintainers confirm report shape.", + "public_record_required": true + }, + { + "id": "galaxy_iuc", + "name": "Galaxy / IUC wrappers", + "type": "galaxy_toolshed", + "status": "local_wrapper_ready", + "external_target": "https://github.com/galaxyproject/tools-iuc", + "source_assets": [ + "examples/workflows/galaxy/dotmatch_crispr_count.xml", + "examples/workflows/galaxy/dotmatch_demux.xml", + "examples/workflows/galaxy/dotmatch_panel_check.xml", + "examples/workflows/galaxy/dotmatch_assay_run.xml" + ], + "review_command": "make workflow-examples-ready", + "next_action": "Run Planemo checks in an IUC checkout and open a scoped wrapper PR.", + "public_record_required": true + }, + { + "id": "snakemake_wrapper", + "name": "Snakemake wrapper", + "type": "snakemake_workflow", + "status": "local_workflow_ready", + "external_target": "https://github.com/snakemake/snakemake-wrappers", + "source_assets": [ + "examples/workflows/snakemake/Snakefile", + "examples/workflows/snakemake/config.json", + "examples/workflows/fixtures/" + ], + "review_command": "make workflow-examples-ready", + "next_action": "Convert the local workflow into an external wrapper or public lab workflow.", + "public_record_required": true + }, + { + "id": "bio_tools_record", + "name": "bio.tools registry record", + "type": "registry_record", + "status": "draft_metadata_ready", + "external_target": "https://bio.tools/", + "source_assets": [ + "docs/registries/biotools.yml", + "codemeta.json", + "CITATION.cff" + ], + "review_command": "make reviewer-readiness-ready", + "next_action": "Submit registry metadata and record only the accepted public URL after review.", + "public_record_required": true + } + ] +} diff --git a/docs/methods-and-citation.md b/docs/methods-and-citation.md index b2e5e790..5945af02 100644 --- a/docs/methods-and-citation.md +++ b/docs/methods-and-citation.md @@ -17,11 +17,11 @@ If you use DotMatch, cite the software release through `CITATION.cff`. Installed packages also provide `dotmatch citation` for a copyable citation. Use the Zenodo concept DOI `10.5281/zenodo.20541628` for general software citation. The DOI `10.5281/zenodo.20541629` is the version DOI for v0.1.7 and -must not be used as the v0.1.8 release DOI. +must not be used as the v0.1.9 release DOI. Suggested citation: -> O'Toole D. DotMatch: deterministic known-target short-DNA assignment for sequencing workflows. Software release v0.1.8. https://github.com/dnncha/dotmatch +> O'Toole D. DotMatch: deterministic known-target short-DNA assignment for sequencing workflows. Software release v0.1.9. https://github.com/dnncha/dotmatch DOI: @@ -29,7 +29,7 @@ DOI: For CRISPR guide-counting workflows: -> Reads were assigned to the guide library using DotMatch v0.1.8 with known-target assignment, literal-byte sequence semantics, and the radius ambiguity policy. Count matrices retained only reads for which exactly one guide lay inside the configured edit-distance radius; ambiguous and unmatched reads were excluded from target counts and retained in diagnostic summaries. +> Reads were assigned to the guide library using DotMatch v0.1.9 with known-target assignment, literal-byte sequence semantics, and the radius ambiguity policy. Count matrices retained only reads for which exactly one guide lay inside the configured edit-distance radius; ambiguous and unmatched reads were excluded from target counts and retained in diagnostic summaries. For one-edit Levenshtein rescue: @@ -121,7 +121,7 @@ statements out until real-data comparator evidence is in the repository. ## Evidence Boundary -Describe DotMatch v0.1.8 as a known-target short-DNA assignment engine. It is +Describe DotMatch v0.1.9 as a known-target short-DNA assignment engine. It is not a genome aligner, general Edlib replacement, production Illumina demultiplexer, full Perturb-seq analysis pipeline, adapter trimmer, UMI grouper, read merger, or amplicon consensus/variant-calling workflow. Current public diff --git a/docs/packaging.md b/docs/packaging.md index 1f2bd3fe..d353c8c8 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -18,7 +18,7 @@ The same verifier also builds the sdist, confirms it contains `src/qdalign.c` an For PyPI, upload the sdist plus the native macOS wheel built on GitHub Actions. Linux binary wheels should go to PyPI only after they are built or repaired as manylinux/musllinux wheels. The release workflow builds repaired Linux wheel artifacts with cibuildwheel for `manylinux_x86_64` and `musllinux_x86_64`, smoke-tests `import dotmatch`, the installed console script, and `dotmatch dist ACGT AGGT`, and uploads them as GitHub release artifacts. Do not upload a raw `linux_x86_64` wheel to PyPI. -DotMatch 0.1.8 is published on PyPI; the `v0.1.8` release workflow publishes the source distribution, the native macOS wheel, and repaired manylinux/musllinux Linux wheels. The release workflow +DotMatch 0.1.9 is the current release target; the `v0.1.9` release workflow publishes the source distribution, the native macOS wheel, and repaired manylinux/musllinux Linux wheels. The release workflow uses PyPI trusted publishing from repository `dnncha/dotmatch`, workflow `.github/workflows/release.yml`, and environment `pypi`; if that publisher is missing or mismatched, the build artifacts are created but the publish job fails @@ -36,17 +36,16 @@ not upload a Conda package directly. [bioconda/bioconda-recipes#65367](https://github.com/bioconda/bioconda-recipes/pull/65367) published DotMatch 0.1.2 as the first Bioconda package. [bioconda/bioconda-recipes#66291](https://github.com/bioconda/bioconda-recipes/pull/66291) -merged the DotMatch 0.1.8 update on 2026-06-17. Anaconda package metadata and a -clean install smoke test now verify DotMatch 0.1.8 on `linux-64`, `osx-64`, and -`osx-arm64`, using immutable `v0.1.8` release sources. Treat future Bioconda +merged the DotMatch 0.1.8 update on 2026-06-17. The public Anaconda page now +shows 0.1.9, but immutable source, checksum, repodata, and clean-install +evidence for that release still needs to be captured. Treat the Bioconda versions as available only after `https://anaconda.org/bioconda/dotmatch`, repodata, and the install smoke tests in `make distribution-channels` all verify the release version. A release recipe template is kept under `packaging/bioconda/`. Before copying it to `bioconda-recipes`, replace `REPLACE_WITH_RELEASE_TARBALL_SHA256` with the -SHA256 for the tagged GitHub release tarball. For 0.1.8 that SHA256 is -`ec3819bc773431454910287559d0809aca6ec1d81959f29d3d522650edb74904`. The checked-in +SHA256 for the tagged GitHub release tarball. The checked-in `docs/distribution-release.json` records the current channel state for the active Bioconda handoff. Run `make bioconda-recipe-ready` before that copy so the checked-in template stays aligned with the release version, native install @@ -90,10 +89,10 @@ The native CLI exposes `dotmatch --version`, so the Bioconda recipe and post-release Bioconda install verifier should check version output as well as functional CLI smoke tests. -### Bioconda 0.1.8 PR changelog draft +### Bioconda 0.1.9 PR changelog draft -- Update DotMatch from the latest accepted Bioconda version to 0.1.8. -- Use the immutable v0.1.8 tag and replace the SHA256 after the release tarball +- Update DotMatch from the latest accepted Bioconda version to 0.1.9. +- Use the immutable v0.1.9 tag and replace the SHA256 after the release tarball is available. - Keep the Python console-script package scope introduced in 0.1.4: `dotmatch` exposes the native commands plus `assay`, `barcode`, `panel`, and @@ -130,16 +129,15 @@ images to `ghcr.io/dnncha/dotmatch`. BioContainers images for DotMatch are generated from the accepted Bioconda recipe; there is no separate DotMatch Dockerfile to submit to BioContainers for -the normal release path. DotMatch 0.1.8 package metadata and clean Bioconda -install smoke tests pass, and `make distribution-channels` can discover a -matching `quay.io/biocontainers/dotmatch:0.1.8--` tag. The remaining -local check is Docker-backed manifest/runtime verification: +the normal release path. The 0.1.9 image is expected only after the Bioconda +recipe is accepted and propagated. The remaining local check is Docker-backed +manifest/runtime verification: ```bash -python3 scripts/check_distribution_channels.py --version 0.1.8 -docker pull quay.io/biocontainers/dotmatch:0.1.8-- -docker run --rm quay.io/biocontainers/dotmatch:0.1.8-- dotmatch dist ACGT AGGT -docker run --rm quay.io/biocontainers/dotmatch:0.1.8-- dotmatch leq 1 ACGT AGGT +python3 scripts/check_distribution_channels.py --version 0.1.9 +docker pull quay.io/biocontainers/dotmatch:0.1.9-- +docker run --rm quay.io/biocontainers/dotmatch:0.1.9-- dotmatch dist ACGT AGGT +docker run --rm quay.io/biocontainers/dotmatch:0.1.9-- dotmatch leq 1 ACGT AGGT ``` Do not publish a manual BioContainers image for DotMatch unless the Bioconda @@ -190,6 +188,7 @@ publication has actually happened. The repository includes `.zenodo.json` metadata for tagged software archives. General software citation uses DOI `10.5281/zenodo.20541628`, which resolves -through Zenodo metadata for DotMatch 0.1.8. Version DOI +through Zenodo metadata for DotMatch. The 0.1.9 archive metadata remains a +post-tag verification step. Version DOI `10.5281/zenodo.20541629` belongs to v0.1.7 and is retained only as explicit -version-specific provenance, not as the v0.1.8 DOI. +version-specific provenance, not as the v0.1.9 DOI. diff --git a/docs/pilot-program.md b/docs/pilot-program.md new file mode 100644 index 00000000..bcafc72d --- /dev/null +++ b/docs/pilot-program.md @@ -0,0 +1,80 @@ +# DotMatch Evaluation Protocol + +This protocol is for core facilities, CRISPR screen teams, assay-development +groups, CROs, biotech teams, and workflow maintainers assessing DotMatch on a +known-target sequencing workflow. + +The goal is to decide whether DotMatch fits an existing assay or pipeline by +checking installation, input mapping, assignment outputs, and methods artifacts +on data the reviewing team is allowed to share or summarize. + +## Suitable Workflows + +DotMatch is appropriate to evaluate when the workflow has: + +- FASTQ or FASTQ.gz reads, or a public/sanitized fixture; +- a known target table such as guides, barcodes, feature tags, primers, or panel + targets; +- a fixed, scaffolded, or inferable read window; +- an existing output to compare against, such as counts, split FASTQs, or QC + tables; +- permission to record commands, package versions, and non-sensitive output + summaries. + +DotMatch is not a substitute for genome alignment, basecalling, adapter +trimming, UMI/cell quantification, variant calling, or screen-level statistics. + +## Intake Fields + +Capture these details before running a comparison: + +| Field | Required note | +| --- | --- | +| Project or assay | Public name, anonymized label, or internal reference | +| Assay context | CRISPR, inline barcode, feature barcode, amplicon, adapter prefix, panel, or other known-target workflow | +| Current workflow | Existing command, wrapper, notebook, or manual process | +| Target table shape | Identifier columns, sequence column, expected target length | +| Read window | Start, length, read mate, and whether the window is inferred | +| Correction policy | Exact, Hamming, Levenshtein, radius, quality filter, or no correction | +| Outputs to inspect | QC report, counts, split FASTQs, per-read assignments, top unmatched, methods text | +| Public-use permission | None, anonymized summary, public project name, or approved record text | + +Do not place private FASTQ/BAM/BCL files, patient data, customer assay designs, +or restricted screenshots in the public repository. + +## Review Steps + +1. Install DotMatch from PyPI or Bioconda unless the review is explicitly testing + an unreleased checkout. +2. Run the closest tutorial or workflow example first. +3. Audit the target table before enabling correction. +4. Run the evaluation on minimized or public data when possible. +5. Inspect `sample_qc.tsv`, `summary.json`, `assignments.tsv`, + `top_unmatched.tsv`, and the HTML report before comparing headline counts. +6. Check whether ambiguous reads, unmatched reads, invalid windows, unsafe + correction, or workflow handoff became easier to review. +7. Record blockers as workflow requirements, not as product claims. + +## Scorecard + +| Dimension | Score | Notes | +| --- | --- | --- | +| Install from released channel | 0-2 | PyPI, Bioconda, source, or container | +| Input mapping | 0-2 | Target table, sample table, read window | +| Assignment review | 0-2 | Ambiguous, unmatched, invalid, unsafe correction | +| Workflow fit | 0-2 | TSV, JSON, FASTQ, HTML, MultiQC, notebook | +| Methods and citation | 0-2 | Version, command, ambiguity policy | +| Public record permission | 0-2 | Approved wording and URL, if any | + +Suggested interpretation: + +- `0-4`: workflow fit is poor or incomplete. +- `5-8`: useful evaluation; record blockers and repeat after fixes. +- `9-12`: strong candidate for a documented public workflow example or approved + use record. + +## Public Use Records + +Public use records belong in `docs/adopters/` only when the reviewing +organization or project has approved the exact name, URL, scope note, and +evidence link. Until then, keep the evaluation as a private review artifact. diff --git a/docs/registries/biotools.yml b/docs/registries/biotools.yml new file mode 100644 index 00000000..b6285582 --- /dev/null +++ b/docs/registries/biotools.yml @@ -0,0 +1,49 @@ +name: DotMatch +biotoolsID: dotmatch +homepage: https://github.com/dnncha/dotmatch +documentation: https://dotmatch.readthedocs.io/ +description: > + DotMatch is a deterministic known-target short-DNA assignment toolkit for + fixed read windows in CRISPR guide counting, inline barcode demultiplexing, + feature-barcode reads, primer or adapter-prefix checks, amplicon-panel starts, + whitelist-style assays, and barcode panel design. +license: Apache-2.0 +version: "0.1.9" +topic: + - Bioinformatics + - Sequencing + - Quality control +function: + - operation: + - Sequence analysis + - Demultiplexing + - Quality control + input: + - data: FASTQ + - data: Sequence set + output: + - data: Count matrix + - data: Report + - data: FASTQ +language: + - Python + - C +operatingSystem: + - Linux + - macOS +download: + - type: Source code + url: https://github.com/dnncha/dotmatch + - type: Package + url: https://pypi.org/project/dotmatch/ + - type: Package + url: https://anaconda.org/bioconda/dotmatch +publication: + - doi: 10.5281/zenodo.20541628 +credit: + - name: Donncha O'Toole + orcid: https://orcid.org/0009-0003-5012-7229 +note: > + Draft registry metadata. Do not describe this as an accepted bio.tools record + until the public registry page exists and is recorded in docs/integration-targets.json + or docs/workflow-adoption.json. diff --git a/docs/release-process.md b/docs/release-process.md index 4ab95f8b..791517d6 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -1,6 +1,6 @@ # Release Process -DotMatch releases should be specific, reproducible, and evidence-bounded. +DotMatch releases should be specific, reproducible, and evidence-scoped. ## Pre-Tag Checks diff --git a/docs/resubmission-evidence.md b/docs/resubmission-evidence.md index 684b2cae..5178d205 100644 --- a/docs/resubmission-evidence.md +++ b/docs/resubmission-evidence.md @@ -14,7 +14,7 @@ and kept current as evidence changes. | Public CRISPR example | Public Yusa/CRISPR guide-counting fixture and comparison reports | See `examples/crispr_guides/`, `examples/crispr_sanson_brunello/`, and `docs/benchmarks/public_crispr/README.md` | | Workflow reuse | nf-core-style, Snakemake, MultiQC, and Galaxy integration examples | See `examples/workflows/` and `docs/workflow-adoption.json` | | Citation and archival record | CITATION metadata and Zenodo DOI badge | See `CITATION.cff` and `docs/methods-and-citation.md` | -| Independent adoption | External confirmations | Track quote-approved confirmations in `docs/adopters/README.md`; do not count unapproved private emails | +| Independent use | External confirmations | Track approved confirmations in `docs/adopters/README.md`; do not count unapproved private emails | ## Minimal Reproduction Command @@ -60,7 +60,7 @@ accepted, released, or published. - Run `make repro`, `make repository-ready`, and the relevant evidence gates for any claim being made in the submission text. - Link to release artifacts: PyPI, Bioconda, GitHub Release, and Zenodo DOI. -- Add only quote-approved, independent adopter statements to +- Add only approved, independent adopter statements to `docs/adopters/README.md`. - Keep claims aligned with `docs/scientific-claims.md`; move aspirational or unverified statements to roadmap text. diff --git a/docs/reviewer-readiness.json b/docs/reviewer-readiness.json new file mode 100644 index 00000000..ae4c8e43 --- /dev/null +++ b/docs/reviewer-readiness.json @@ -0,0 +1,67 @@ +{ + "schema_version": 1, + "status": "ready_to_execute", + "scope": "public reviewer materials that do not broaden DotMatch's validated scientific scope", + "items": [ + { + "id": "bioinformatics_evaluation_packet", + "title": "Bioinformatics evaluation packet", + "asset": "docs/bioinformatics-evaluation.md", + "done_when": "Reviewers can see package channels, validated assay scope, minimum commands, output files, workflow status, and public wording rules." + }, + { + "id": "external_review_packet", + "title": "External review packet", + "asset": "docs/external-review-packet.md", + "done_when": "Workflow maintainers, PIs, and technical reviewers have one checklist for install proof, output contracts, validated scope, and known blockers." + }, + { + "id": "integration_target_tracker", + "title": "Integration target tracker", + "asset": "docs/integration-targets.json", + "done_when": "nf-core, MultiQC, Galaxy/IUC, Snakemake, and bio.tools targets have machine-readable status, source assets, external targets, and next actions." + }, + { + "id": "biotools_registry_draft", + "title": "bio.tools registry draft", + "asset": "docs/registries/biotools.yml", + "done_when": "Registry metadata is ready to review without implying accepted registry status." + }, + { + "id": "evaluation_protocol", + "title": "DotMatch evaluation protocol", + "asset": "docs/pilot-program.md", + "done_when": "Reviewers can assess workflow fit, outputs, limitations, and public-use permission without exposing sensitive data." + }, + { + "id": "public_use_record_template", + "title": "Public use record template", + "asset": "docs/adopters/record-template.md", + "done_when": "Approved public use records use consistent fields and scope notes." + }, + { + "id": "workflow_integration_issue_template", + "title": "Workflow integration issue template", + "asset": ".github/ISSUE_TEMPLATE/workflow_integration.yml", + "done_when": "External integration requests ask for workflow manager, source assets, expected outputs, fixtures, and validated scope." + }, + { + "id": "evaluation_feedback_issue_template", + "title": "Evaluation feedback issue template", + "asset": ".github/ISSUE_TEMPLATE/pilot_feedback.yml", + "done_when": "Evaluation feedback can be submitted with install path, assay context, outputs inspected, blockers, and public-use permission." + }, + { + "id": "pull_request_language_checklist", + "title": "Pull request language checklist", + "asset": ".github/PULL_REQUEST_TEMPLATE.md", + "done_when": "Public wording changes require validated-scope, workflow-status, and unsupported-language checks." + }, + { + "id": "reviewer_readiness_gate", + "title": "Reviewer readiness gate", + "asset": "scripts/check_reviewer_readiness_assets.py", + "done_when": "`make reviewer-readiness-ready` verifies the assets, statuses, and public-language guardrails." + } + ] +} diff --git a/docs/scientific-claims.md b/docs/scientific-claims.md index 83c64f40..b0e18fb5 100644 --- a/docs/scientific-claims.md +++ b/docs/scientific-claims.md @@ -130,7 +130,7 @@ alignment or all demultiplexing tasks. | Oligo/adapter comparisons. | `make oligo-adapter-public-gate` requires the synthetic diagnostic lane plus public DotMatch k=0/k=1 rows and exact-slice baseline agreement for the fast-adapter-trimming TruSeq R1 fixed window. | The checked public lane supports adapter-prefix assignment only; adapter trimming, primer removal, UMI grouping, read merging, and production adapter workflow claims require separate comparator evidence. | | General aligner replacement. | No repository check promotes this. | DotMatch does not currently expose reference-index mapping, traceback/CIGAR, SAM/BAM, paired-end mapping, or genome-scale alignment semantics. | | Native SeqAn/Parasail comparisons. | `make native-comparator-scope-ready` checks the documented scope in `docs/native-comparator-scope.md`. | SeqAn and Parasail are not completed comparator evidence until equivalent scoring semantics, native dependency/version capture, raw CSV rows, generated reports, and zero assignment mismatches are recorded. | -| Package-channel availability. | `make python-package-test` verifies local Linux/macOS wheel and sdist installability; public PyPI and Bioconda availability for DotMatch 0.1.8 is claimed only after `make distribution-channels` verifies metadata and install smoke tests. | Package-channel availability is distribution evidence only. It does not extend CRISPR, barcode, feature-barcode, BCL, amplicon, adapter, GPU, or aligner-comparison claims; `make distribution-channels` records each release channel only after public metadata and install smoke tests pass. | +| Package-channel availability. | `make python-package-test` verifies local Linux/macOS wheel and sdist installability; public PyPI and Bioconda availability for DotMatch 0.1.9 is claimed only after `make distribution-channels` verifies metadata and install smoke tests. | Package-channel availability is distribution evidence only. It does not extend CRISPR, barcode, feature-barcode, BCL, amplicon, adapter, GPU, or aligner-comparison claims; `make distribution-channels` records each release channel only after public metadata and install smoke tests pass. | | Quality-aware assignment. | Current checks cover deterministic `--max-correction-qual` behavior only. A posterior or likelihood model would need fixtures with known truth, calibration diagnostics, threshold semantics, and comparator or oracle validation before appearing in supported public claims. | Do not describe current DotMatch calls as posterior probabilities, calibrated confidence, probabilistic assignment, or quality-weighted target likelihoods. The supported behavior is a hard Phred threshold on observed edited bases for selected one-edit rescue paths. | ## Rules For New Public Statements diff --git a/docs/scientific-readiness.json b/docs/scientific-readiness.json index 1ff47531..a520bce7 100644 --- a/docs/scientific-readiness.json +++ b/docs/scientific-readiness.json @@ -83,7 +83,7 @@ "gates": [ "make scientific-readiness-ready" ], - "acceptance": "Broad public metadata must use deterministic/evidence-bounded assignment language and avoid unsupported global SOTA or fast-exact positioning." + "acceptance": "Broad public metadata must use deterministic, scoped assignment language and avoid unsupported global SOTA or fast-exact positioning." }, { "id": "distribution_reproducibility", diff --git a/docs/tutorials/crispr-count-first-run.md b/docs/tutorials/crispr-count-first-run.md index dfa69eb8..e4e2f319 100644 --- a/docs/tutorials/crispr-count-first-run.md +++ b/docs/tutorials/crispr-count-first-run.md @@ -42,7 +42,7 @@ Key outputs under the configured `out_dir`: - `counts.mageck.tsv` — MAGeCK-style count matrix - `sample_qc.tsv` — per-sample assignment and representation QC - `summary.json` — run metadata and assignment rates -- `reliability_report.html` — evidence-bounded preflight/postrun review +- `reliability_report.html` — scoped preflight/postrun review - `crispr_qc.json` — guide-level QC summary CPU remains the assignment authority. GPU Metal is opt-in via `[backend]` in the @@ -150,4 +150,4 @@ diff -u examples/workflows/fixtures/expected_counts.mageck.tsv \ No diff means the tutorial count matrix matches the repository fixture. For a public-data CRISPR example, use `examples/crispr_guides/run.sh` and the -checked evidence reports under `docs/benchmarks/public_crispr/`. \ No newline at end of file +checked evidence reports under `docs/benchmarks/public_crispr/`. diff --git a/docs/workflow-adoption.json b/docs/workflow-adoption.json index c0daec90..19abed04 100644 --- a/docs/workflow-adoption.json +++ b/docs/workflow-adoption.json @@ -3,7 +3,7 @@ "status": "not_ready", "integrations": [], "blockers": [ - "No external workflow-manager adoption record is listed yet. Bioconda package availability is distribution evidence, not workflow adoption evidence." + "No external workflow-manager record is listed yet. Bioconda package availability is distribution evidence, not workflow integration evidence." ], "next_action": "After a stable Galaxy, Nextflow, Snakemake, nf-core, MultiQC, or other external workflow record is public, add an integration entry with adoption and evidence URLs, then run make workflow-adoption-status." } diff --git a/docs/workflow-integration-kit.md b/docs/workflow-integration-kit.md new file mode 100644 index 00000000..e94bbc05 --- /dev/null +++ b/docs/workflow-integration-kit.md @@ -0,0 +1,163 @@ +# DotMatch Workflow Integration Kit + +This page collects the materials needed to review DotMatch for workflow +integration in nf-core, MultiQC, Galaxy/IUC, Snakemake, or an institutional +pipeline. Public performance, correctness, packaging, and workflow status +statements should remain tied to `docs/scientific-claims.md`, +`docs/packaging.md`, and `docs/workflow-adoption.json`. + +## Priority Integration Work + +| Work item | Relevant reviewers | Asset in this repository | Done when | +| --- | --- | --- | --- | +| Homepage routes | Core facility leads, CRISPR screen teams, assay developers, workflow maintainers | Public homepage section: "Routes into industry workflows" | Each reviewer has a clear use case, proof path, and next-click destination. | +| Workflow distribution handoff | nf-core, MultiQC, Galaxy/IUC, Snakemake, institutional pipeline owners | `docs/workflow-submissions.md` | At least one external PR, package, or reviewed wrapper is public and tracked in `docs/workflow-adoption.json`. | +| Methods and citation artifacts | PIs, methods writers, bioinformatics leads, paper authors | `docs/methods-and-citation.md`, `docs/citation-flywheel.md` | Runs can produce copyable methods and citation artifacts, and reviewers know how to cite the exact release. | +| Reviewer packet | Technical reviewers, core facilities, procurement reviewers | Evidence gallery, benchmark docs, scientific scope notes | Public statements link to scoped evidence instead of broad claims. | +| Public use records | Labs, cores, CROs, biotech, pharma teams, workflow projects | `docs/adopters/README.md` | Independent use is listed only after the named party approves the wording and public URL. | +| Reviewer readiness gate | Maintainers and reviewers | `docs/reviewer-readiness.json`, `scripts/check_reviewer_readiness_assets.py` | `make reviewer-readiness-ready` passes before reviewer, integration, registry, or public-use materials are merged. | + +## Positioning + +Short description: + +> DotMatch is a deterministic known-target sequencing assignment toolkit. It +> counts or demultiplexes fixed read windows against expected short DNA targets +> while keeping unique, ambiguous, unmatched, and invalid outcomes visible. + +One-line audience variants: + +- Core facilities: "DotMatch makes barcode and guide assignment failures visible before results leave the core." +- CRISPR screen teams: "DotMatch produces guide-counting artifacts while preserving ambiguous and unmatched reads for review." +- Workflow maintainers: "DotMatch writes stable TSV, JSON, FASTQ, and HTML outputs that can be wrapped in nf-core, Galaxy, Snakemake, and MultiQC." +- Assay developers: "DotMatch designs and audits barcode panels so unsafe correction rules are caught before sequencing." + +Avoid these unsupported shortcuts unless the linked evidence explicitly supports +them for the exact release and setting: + +- "replacement for genome aligners"; +- "replacement for basecallers or full BCL conversion"; +- "screen-analysis package"; +- "guaranteed production demultiplexing replacement"; +- broad speed claims without benchmark scope, hardware, command, and comparator. + +## Launch Checklist + +Use this checklist when announcing a release, opening an integration PR, or +asking an external maintainer to evaluate DotMatch. + +- Link the homepage first for positioning. +- Link `docs/getting-started.md` or a specific tutorial for a runnable path. +- Link `docs/scientific-claims.md` before making scope-sensitive claims. +- Link `docs/workflow-submissions.md` for pipeline maintainers. +- Link `docs/methods-and-citation.md` for citation and methods text. +- Include PyPI, Bioconda, and repository links only after release smoke tests pass. +- Record any accepted external integration in `docs/workflow-adoption.json`. +- Add public use records to `docs/adopters/` only with approved wording and a + public URL. + +## Copy-Paste Outreach + +### Repository announcement + +```text +DotMatch is a deterministic known-target sequencing assignment toolkit for +CRISPR guides, inline barcodes, feature tags, primers, and panel targets. It +reports every read as unique, ambiguous, unmatched, or invalid, so assignment +failures stay visible in TSV, JSON, FASTQ, and HTML outputs. + +Homepage: https://dnncha.github.io/dotmatch +Docs: https://dotmatch.readthedocs.io/ +Repository: https://github.com/dnncha/dotmatch +``` + +### Workflow maintainer email + +```text +Subject: DotMatch handoff for known-target sequencing workflows + +Hi , + +DotMatch may be a fit for workflows that count CRISPR guides, split inline +barcodes, or audit known-target read windows. The useful distinction is that it +keeps unique, ambiguous, unmatched, and invalid read outcomes explicit instead +of collapsing them into a count-only result. + +I put together a workflow submission pack with expected outputs, review notes, +and public workflow-status rules: +https://github.com/dnncha/dotmatch/blob/main/docs/workflow-submissions.md + +If this overlaps with your pipeline, I would value a review of the output +contract and wrapper shape before opening or expanding an integration PR. +``` + +### Core facility pilot email + +```text +Subject: Pilot request: visible assignment QC for barcodes and guide counts + +Hi , + +I am looking for feedback from core facilities that run known-target sequencing +assays: CRISPR guide counting, inline barcodes, feature tags, or panel starts. +DotMatch focuses on the assignment reliability layer, especially ambiguous +reads, unsafe correction, shifted windows, and recurring unmatched sequences. + +The homepage and evidence boundary are here: +https://dnncha.github.io/dotmatch +https://github.com/dnncha/dotmatch/blob/main/docs/scientific-claims.md + +If a small public or anonymized evaluation is possible, I can help scope the +commands and outputs so the review does not require sensitive sample data. +``` + +### Short social post + +```text +DotMatch is a known-target sequencing assignment toolkit for CRISPR guides, +inline barcodes, feature tags, primers, and panel targets. It keeps unique, +ambiguous, unmatched, and invalid read outcomes visible in workflow-friendly +artifacts. + +Homepage: https://dnncha.github.io/dotmatch +Docs: https://dotmatch.readthedocs.io/ +``` + +## Where To Put Effort First + +1. Open or prepare the official nf-core module PRs after local workflow gates + pass. This gives pipeline authors concrete module code and test fixtures. +2. Package or upstream the MultiQC module so DotMatch outputs are automatically + visible in existing reports. +3. Use the [DotMatch Evaluation Protocol](pilot-program.md) with two to five + core facilities or CRISPR-screen teams. +4. Convert strong evaluations into methods-focused examples only when the + reviewing team approves the public wording and URL. +5. Keep the homepage, docs index, citation page, and public use records aligned + so every external mention points to the same source of truth. + +## Workflow Integration Roadmap + +The next layer is tracked in +[DotMatch Workflow Integration Roadmap](workflow-integration-roadmap.md) +and mirrored in `docs/workflow-integration-plan.json`. + +1. Evaluator decision tree. +2. Persona one-pagers. +3. Integration target tracker. +4. Reviewer evidence packet. +5. Conference abstracts. +6. Social and forum pack. +7. Maintainer issue templates. +8. Evaluation scorecard. +9. Integration tracking metrics. +10. Release communications calendar. + +## Tracking Rules + +- Unmerged PRs can be listed as work in progress, not as accepted integration. +- External use records require an approved public URL. +- Names, organization names, logos, and quotes require approved wording before + they appear in the repository or public site. +- If a statement would influence purchase, publication, or pipeline replacement, + it needs a checked artifact and a scoped wording review. diff --git a/docs/workflow-integration-plan.json b/docs/workflow-integration-plan.json new file mode 100644 index 00000000..2508e8b0 --- /dev/null +++ b/docs/workflow-integration-plan.json @@ -0,0 +1,77 @@ +{ + "schema_version": 1, + "status": "ready_to_execute", + "scope": "workflow integration and external technical review assets", + "items": [ + { + "id": "reviewer_decision_tree", + "title": "Reviewer Decision Tree", + "primary_audience": "technical reviewers", + "asset": "docs/workflow-integration-roadmap.md#1-reviewer-decision-tree", + "done_when": "A reviewer can pick the right DotMatch entry path without broad assay claims." + }, + { + "id": "persona_one_pagers", + "title": "Persona One-Pagers", + "primary_audience": "core facilities, CRISPR teams, workflow maintainers, assay developers", + "asset": "docs/workflow-integration-roadmap.md#2-persona-one-pagers", + "done_when": "Each priority reviewer group has a workflow use case, evidence path, and first command." + }, + { + "id": "integration_tracker", + "title": "Integration Target Tracker", + "primary_audience": "pipeline maintainers", + "asset": "docs/workflow-integration-roadmap.md#3-integration-target-tracker", + "done_when": "Every target integration has an upstream URL, readiness state, source assets, and next action." + }, + { + "id": "reviewer_packet", + "title": "Reviewer Packet", + "primary_audience": "maintainers, reviewers, procurement evaluators", + "asset": "docs/workflow-integration-roadmap.md#4-reviewer-packet", + "done_when": "A reviewer can find install proof, outputs, limitations, evidence, and citation text from one checklist." + }, + { + "id": "conference_abstracts", + "title": "Conference Abstracts", + "primary_audience": "conference reviewers and attendees", + "asset": "docs/workflow-integration-roadmap.md#5-conference-abstracts", + "done_when": "Short and long abstracts are ready for workflow, core facility, and methods venues." + }, + { + "id": "technical_communication_pack", + "title": "Technical Communication Pack", + "primary_audience": "bioinformatics community readers", + "asset": "docs/workflow-integration-roadmap.md#6-technical-communication-pack", + "done_when": "Release notes, technical threads, and forum prompts stay scoped to validated behavior." + }, + { + "id": "maintainer_issue_templates", + "title": "Maintainer Issue Templates", + "primary_audience": "nf-core, MultiQC, Galaxy, Snakemake maintainers", + "asset": "docs/workflow-integration-roadmap.md#7-maintainer-issue-templates", + "done_when": "External issue or PR openings can be copied with scope, outputs, tests, and review asks." + }, + { + "id": "evaluation_scorecard", + "title": "Evaluation Scorecard", + "primary_audience": "labs, cores, CROs, biotech teams", + "asset": "docs/workflow-integration-roadmap.md#8-evaluation-scorecard", + "done_when": "Evaluation feedback can be scored against install, input mapping, outputs, workflow fit, methods text, and public-use permission." + }, + { + "id": "integration_tracking_metrics", + "title": "Integration Tracking Metrics", + "primary_audience": "maintainers", + "asset": "docs/workflow-integration-roadmap.md#9-integration-tracking-metrics", + "done_when": "The project tracks workflow interest separately from scientific claims and release quality." + }, + { + "id": "release_communications_calendar", + "title": "Release Communications Calendar", + "primary_audience": "maintainers", + "asset": "docs/workflow-integration-roadmap.md#10-release-communications-calendar", + "done_when": "Each release has a pre-merge, tag, channel-verification, and follow-up communication checklist." + } + ] +} diff --git a/docs/workflow-integration-roadmap.md b/docs/workflow-integration-roadmap.md new file mode 100644 index 00000000..368da5b7 --- /dev/null +++ b/docs/workflow-integration-roadmap.md @@ -0,0 +1,232 @@ +# DotMatch Workflow Integration Roadmap + +This roadmap builds on the workflow integration kit in +`docs/workflow-integration-kit.md`. It turns reviewer-facing work into concrete +assets for maintainers, workflow reviewers, core facilities, CRISPR teams, and +assay developers without expanding DotMatch's validated scope. + +Use `docs/workflow-integration-plan.json` as the machine-readable checklist for +these ten assets. Reviewer readiness is checked by +`docs/reviewer-readiness.json` and `scripts/check_reviewer_readiness_assets.py`. +Public use records follow the approval requirements in `docs/adopters/README.md`. + +## 1. Reviewer Decision Tree + +Use this when someone asks whether DotMatch is relevant. + +| Question | If yes | If no | +| --- | --- | --- | +| Do you already know the expected short target sequences? | Continue. | DotMatch is probably not the right first tool. | +| Is the read window fixed, scaffolded, or inferable? | Use count, demux, assay, barcode, or panel docs. | Use extraction or preprocessing before DotMatch. | +| Do ambiguous, unmatched, or invalid reads affect interpretation? | Lead with reliability and output artifacts. | Use DotMatch only if ordinary count outputs still help. | +| Do you need genome coordinates, CIGAR strings, UMI deduplication, or hit statistics? | Use established downstream tools. | DotMatch can own the assignment layer. | +| Do you need a public pipeline integration? | Start with workflow submissions. | Start with local install and methods text. | + +## 2. Persona One-Pagers + +### Core Facilities + +- Promise: visible assignment QC for known guides, barcodes, and panels. +- Proof path: homepage, barcode troubleshooting, evidence gallery, packaging. +- First command: `dotmatch barcode autopsy ...` +- Review artifact: HTML report plus TSV/JSON outputs for lab handoff. + +### CRISPR Screen Teams + +- Promise: guide counts with explicit ambiguity and unmatched diagnostics. +- Proof path: CRISPR tutorial, public CRISPR evidence, methods citation text. +- First command: `dotmatch crispr-count ...` +- Review artifact: count matrix, sample QC, top unmatched, MAGeCK-compatible table. + +### Workflow Maintainers + +- Promise: stable TSV, JSON, FASTQ, and HTML artifacts for wrappers. +- Proof path: workflow submission pack, schemas, MultiQC parser, release gates. +- First command: `make workflow-examples-ready` +- Review artifact: wrapper fixture, command log, expected output contract. + +### Assay Developers + +- Promise: barcode panel design and correction-safety review before sequencing. +- Proof path: barcode panel design docs, assay evidence, panel report. +- First command: `dotmatch panel design ...` +- Review artifact: design report, collision tables, plate layout, lab README. + +## 3. Integration Target Tracker + +Track external integration work separately from local examples. + +| Target | Why it matters | Ready asset | Public state to record | +| --- | --- | --- | --- | +| nf-core modules | High-trust workflow reuse and container automation | `docs/workflow-submissions.md` | Merged PR or released module page | +| MultiQC module | Makes DotMatch visible in existing pipeline reports | `python/dotmatch/multiqc.py` | Released plugin or upstream integration | +| Galaxy/IUC | Reaches core facilities and wet-lab teams | Galaxy wrapper examples | IUC acceptance or ToolShed publication | +| Snakemake wrapper | Easy lab workflow reuse | Snakemake example workflow | Public wrapper or external lab pipeline | +| bio.tools entry | Searchable bioinformatics registry presence | Homepage and metadata | Accepted bio.tools record | + +## 4. Reviewer Packet + +Send this packet when a maintainer, reviewer, procurement evaluator, or PI asks +what is real today. + +- Positioning: homepage. +- Bioinformatics evaluation: `docs/bioinformatics-evaluation.md`. +- External review packet: `docs/external-review-packet.md`. +- Install proof: PyPI, Bioconda, packaging notes. +- Output contract: schemas and command reference. +- Validated scope: scientific claims and trust/scope docs. +- Evidence: evidence gallery and benchmark pages. +- Citation: methods and citation template. +- Public use records: adopter notes and workflow status JSON. +- Integration status: `docs/integration-targets.json`. + +## 5. Conference Abstracts + +### Short Abstract + +DotMatch is a deterministic known-target sequencing assignment toolkit for +fixed read windows such as CRISPR guides, inline barcodes, feature tags, +primers, and panel targets. It reports unique, ambiguous, unmatched, and invalid +read outcomes so assignment failures remain visible in workflow artifacts. + +### Methods Abstract + +Known-target sequencing workflows often collapse assignment decisions into count +tables, making ambiguous reads, shifted windows, unsafe correction, and recurring +unmatched sequences hard to inspect. DotMatch separates the assignment layer +from downstream interpretation: it compares configured read windows with known +short DNA targets, records explicit read outcomes, and writes TSV, JSON, FASTQ, +and HTML artifacts for workflow review. Public claims are scoped to checked +repository evidence and release gates. + +### Core Facility Abstract + +DotMatch helps sequencing cores review known-target assays before results leave +the facility. It supports guide counting, inline barcode demultiplexing, barcode +panel design, and assignment autopsy reports while preserving ambiguity, +unmatched reads, and invalid extraction windows as visible QC signals. + +## 6. Technical Communication Pack + +### Technical Thread + +```text +DotMatch focuses on one layer: assigning fixed read windows to known short DNA +targets. + +Why that matters: +1. unique reads can be counted +2. ambiguous reads stay out of forced calls +3. unmatched reads remain reviewable +4. invalid windows are visible QC failures + +Docs: https://dotmatch.readthedocs.io/ +Homepage: https://dnncha.github.io/dotmatch +``` + +### Forum Prompt + +```text +I am looking for feedback from teams that run known-target sequencing assays: +CRISPR guide counting, inline barcode demultiplexing, feature tags, primers, or +panel starts. DotMatch is scoped to assignment reliability, not downstream +screen statistics or genome alignment. Which workflow wrapper would make review +easiest for your lab: nf-core, MultiQC, Galaxy, Snakemake, or something else? +``` + +### Release Follow-Up + +```text +The latest DotMatch release gates package installability, docs, scientific +validated scope, workflow examples, and public evidence checks before release +tagging. The project is looking for reviewed workflow integrations and scoped +technical feedback. +``` + +## 7. Maintainer Issue Templates + +### nf-core / Workflow Module Opening + +```text +Title: Add DotMatch known-target assignment module + +DotMatch assigns fixed read windows to known short DNA targets and writes TSV, +JSON, FASTQ, and HTML outputs. This module proposal is scoped to assignment +artifacts and explicit unique/ambiguous/unmatched/invalid outcomes. + +Review asks: +- command shape and metadata +- output contract +- container pinning +- fixture coverage +- MultiQC compatibility +``` + +### MultiQC Opening + +```text +Title: Parse DotMatch assignment QC outputs + +DotMatch writes sample QC, summaries, top-unmatched rows, and panel-safety +outputs for known-target sequencing assignments. A MultiQC module should expose +assignment rate, ambiguity rate, unmatched rate, invalid windows, and panel +safety status without implying downstream biological pass/fail calls. +``` + +## 8. Evaluation Scorecard + +Use `docs/pilot-program.md` for intake fields, review steps, and public-use +record requirements. + +| Dimension | Score | Notes | +| --- | --- | --- | +| Install worked from released channel | 0-2 | PyPI, Bioconda, source, or container | +| Input mapping was understandable | 0-2 | Target table, sample table, read window | +| Assignment failures were clearer | 0-2 | Ambiguous, unmatched, invalid, unsafe correction | +| Outputs fit existing workflow | 0-2 | TSV, JSON, FASTQ, HTML, MultiQC, notebook | +| Citation and methods text was usable | 0-2 | Version, command, ambiguity policy | +| Public use record approved | 0-2 | Approved wording and public URL, if applicable | + +Interpretation: + +- 0-4: workflow fit is poor or incomplete. +- 5-8: useful evaluation; record blockers and repeat after fixes. +- 9-12: strong candidate for a documented public workflow example or approved + use record. + +## 9. Integration Tracking Metrics + +Track integration status separately from scientific evidence. + +| KPI | Source | Cadence | +| --- | --- | --- | +| Homepage visits to install clicks | site analytics if enabled | monthly | +| Docs visits to tutorial starts | docs analytics if enabled | monthly | +| External workflow PRs opened | GitHub URLs | weekly during push | +| External workflow PRs merged | accepted public records | release cycle | +| Public use records approved | `docs/adopters/` | release cycle | +| Citation artifacts generated | release or assay outputs | release cycle | +| Distribution channel health | `make distribution-channels` | release cycle | + +Do not combine these metrics with performance or correctness claims. Workflow +interest can increase before external integration is accepted. + +## 10. Release Communications Calendar + +Use this around each release or major integration push. + +| Time | Action | Evidence link | +| --- | --- | --- | +| T-7 days | Confirm validated scope and release notes | `docs/scientific-claims.md` | +| T-5 days | Prepare maintainer issue or PR drafts | `docs/workflow-submissions.md` | +| T-3 days | Prepare short social and forum posts | this page | +| Tag day | Announce only after release workflow artifacts are visible | release URL | +| T+1 day | Verify PyPI, Bioconda, containers, Zenodo as applicable | `docs/distribution-release.json` | +| T+7 days | Follow up with maintainers and pilot contacts | public URLs only | +| T+30 days | Update integration metrics snapshot | public records only | + +## Completion Rule + +These assets are complete only when the homepage, docs index, integration kit, +structured plan, and site guard all reference the same ten items. If the JSON +plan and markdown playbook diverge, treat the playbook as not release-ready. diff --git a/examples/workflows/nf-core/README.md b/examples/workflows/nf-core/README.md index a1515d6b..65f847f0 100644 --- a/examples/workflows/nf-core/README.md +++ b/examples/workflows/nf-core/README.md @@ -52,7 +52,7 @@ pipelines, reproducibility, MultiQC integration, etc.): 3. **Enhance for Upstream (Recommended Polish)** - Use exact bioconda/singularity container hashes from a released version - (update the placeholder `0.1.8--h*` after a tagged release passes Bioconda). + (update the placeholder `0.1.9--h*` after a tagged release passes Bioconda). - The upstream tree already includes: maintainers + license in meta.yml, stub test case, self-contained tests/data/. - Add more nf-test cases (different k, metrics, full vs stub runs) if needed. - Support additional common params via `task.ext` (e.g. `--auto-offset`, diff --git a/examples/workflows/nf-core/modules/local/dotmatch/assay_run/main.nf b/examples/workflows/nf-core/modules/local/dotmatch/assay_run/main.nf index f4b6bf0d..26e539ad 100644 --- a/examples/workflows/nf-core/modules/local/dotmatch/assay_run/main.nf +++ b/examples/workflows/nf-core/modules/local/dotmatch/assay_run/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_ASSAY_RUN { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(assay_spec), path(assay_inputs) diff --git a/examples/workflows/nf-core/modules/local/dotmatch/audit/main.nf b/examples/workflows/nf-core/modules/local/dotmatch/audit/main.nf index 1181f67f..01d4256e 100644 --- a/examples/workflows/nf-core/modules/local/dotmatch/audit/main.nf +++ b/examples/workflows/nf-core/modules/local/dotmatch/audit/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_AUDIT { time { task.ext.time ?: 2.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(targets) diff --git a/examples/workflows/nf-core/modules/local/dotmatch/count/main.nf b/examples/workflows/nf-core/modules/local/dotmatch/count/main.nf index 28794d74..ce27f1b5 100644 --- a/examples/workflows/nf-core/modules/local/dotmatch/count/main.nf +++ b/examples/workflows/nf-core/modules/local/dotmatch/count/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_COUNT { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(reads), path(targets) diff --git a/examples/workflows/nf-core/modules/local/dotmatch/crispr_count/main.nf b/examples/workflows/nf-core/modules/local/dotmatch/crispr_count/main.nf index 6ae89f86..c6382867 100644 --- a/examples/workflows/nf-core/modules/local/dotmatch/crispr_count/main.nf +++ b/examples/workflows/nf-core/modules/local/dotmatch/crispr_count/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_CRISPR_COUNT { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(reads), path(library) diff --git a/examples/workflows/nf-core/modules/local/dotmatch/demux/main.nf b/examples/workflows/nf-core/modules/local/dotmatch/demux/main.nf index e36bfa4a..d1020344 100644 --- a/examples/workflows/nf-core/modules/local/dotmatch/demux/main.nf +++ b/examples/workflows/nf-core/modules/local/dotmatch/demux/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_DEMUX { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(reads), path(barcodes) diff --git a/examples/workflows/nf-core/modules/local/dotmatch/panel_check/main.nf b/examples/workflows/nf-core/modules/local/dotmatch/panel_check/main.nf index a0b25790..ebfae729 100644 --- a/examples/workflows/nf-core/modules/local/dotmatch/panel_check/main.nf +++ b/examples/workflows/nf-core/modules/local/dotmatch/panel_check/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_PANEL_CHECK { time { task.ext.time ?: 2.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(panel) diff --git a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/assay_run/main.nf b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/assay_run/main.nf index f4b6bf0d..26e539ad 100644 --- a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/assay_run/main.nf +++ b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/assay_run/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_ASSAY_RUN { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(assay_spec), path(assay_inputs) diff --git a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/audit/main.nf b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/audit/main.nf index 1181f67f..01d4256e 100644 --- a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/audit/main.nf +++ b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/audit/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_AUDIT { time { task.ext.time ?: 2.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(targets) diff --git a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/count/main.nf b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/count/main.nf index 28794d74..ce27f1b5 100644 --- a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/count/main.nf +++ b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/count/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_COUNT { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(reads), path(targets) diff --git a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/crispr_count/main.nf b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/crispr_count/main.nf index 6ae89f86..c6382867 100644 --- a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/crispr_count/main.nf +++ b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/crispr_count/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_CRISPR_COUNT { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(reads), path(library) diff --git a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/demux/main.nf b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/demux/main.nf index e36bfa4a..d1020344 100644 --- a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/demux/main.nf +++ b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/demux/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_DEMUX { time { task.ext.time ?: 4.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(reads), path(barcodes) diff --git a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/panel_check/main.nf b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/panel_check/main.nf index a0b25790..ebfae729 100644 --- a/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/panel_check/main.nf +++ b/examples/workflows/nf-core/upstream/modules/nf-core/dotmatch/panel_check/main.nf @@ -7,8 +7,8 @@ process DOTMATCH_PANEL_CHECK { time { task.ext.time ?: 2.h } container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.8--h*' : - 'biocontainers/dotmatch:0.1.8--h*' }" + 'https://depot.galaxyproject.org/singularity/dotmatch:0.1.9--h*' : + 'biocontainers/dotmatch:0.1.9--h*' }" input: tuple val(meta), path(panel) diff --git a/include/qdalign.h b/include/qdalign.h index 4440ee27..7b78d4de 100644 --- a/include/qdalign.h +++ b/include/qdalign.h @@ -7,7 +7,7 @@ extern "C" { #endif -#define QDALN_VERSION "0.1.8" +#define QDALN_VERSION "0.1.9" #define QDALN_ALPHABET_POLICY "literal-byte; A/C/G/T/N/IUPAC symbols are ordinary byte symbols; no wildcard expansion" enum qdaln_match_status { diff --git a/package-lock.json b/package-lock.json index 445c324d..2fbfee2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dotmatch-site", - "version": "0.1.8", + "version": "0.1.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dotmatch-site", - "version": "0.1.8", + "version": "0.1.9", "license": "Apache-2.0", "dependencies": { "next": "^16.2.7", diff --git a/package.json b/package.json index 2fdff7cc..53145e46 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dotmatch-site", - "version": "0.1.8", + "version": "0.1.9", "private": true, "scripts": { "dev": "next dev", diff --git a/packaging/bioconda/meta.yaml b/packaging/bioconda/meta.yaml index bddc45f6..3b5e0b2f 100644 --- a/packaging/bioconda/meta.yaml +++ b/packaging/bioconda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "dotmatch" %} -{% set version = "0.1.8" %} +{% set version = "0.1.9" %} {% set sha256 = "REPLACE_WITH_RELEASE_TARBALL_SHA256" %} package: diff --git a/paper/paper.bib b/paper/paper.bib index 01678cfb..4c585529 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -1,7 +1,7 @@ @software{dotmatch_zenodo_017, author = {{O'Toole}, Donncha}, title = {{DotMatch: deterministic known-target short-DNA assignment for sequencing workflows}}, - version = {0.1.8}, + version = {0.1.9}, date = {2026-06-04}, publisher = {Zenodo}, doi = {10.5281/zenodo.20541628}, diff --git a/pyproject.toml b/pyproject.toml index dc4e08f2..aca31133 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dotmatch" -version = "0.1.8" +version = "0.1.9" description = "Deterministic known-target short-DNA assignment for CRISPR guide counting, barcode demultiplexing, and FASTQ workflows" readme = "README.md" requires-python = ">=3.9" diff --git a/python/dotmatch/__init__.py b/python/dotmatch/__init__.py index 940c8488..11749527 100644 --- a/python/dotmatch/__init__.py +++ b/python/dotmatch/__init__.py @@ -50,7 +50,7 @@ def _source_tree_version() -> Optional[str]: try: __version__ = _source_tree_version() or _metadata_version("dotmatch") except PackageNotFoundError: - __version__ = "0.1.8" + __version__ = "0.1.9" __all__ = [ "__version__", diff --git a/python/dotmatch/cli.py b/python/dotmatch/cli.py index 52d76851..6955170c 100644 --- a/python/dotmatch/cli.py +++ b/python/dotmatch/cli.py @@ -2,6 +2,7 @@ import argparse import csv +import glob import gzip import html import json @@ -16,7 +17,7 @@ from typing import Iterable, Iterator, Sequence, TextIO from . import __version__ -from .assayspec import command_assay +from .assayspec import AssaySpecError, command_assay, scaffold_assay_project from .core import ( MATCH_AMBIGUOUS, MATCH_INVALID, @@ -574,6 +575,86 @@ def command_crispr_qc(args: argparse.Namespace) -> int: return 0 +def command_crispr_quickstart(args: argparse.Namespace) -> int: + """Create and optionally run a reviewable CRISPR project from FASTQ paths. + + This is intentionally a thin UX layer over AssaySpec. It keeps the + inference report, staged inputs, preflight checks, QC, and provenance that + make the longer ``assay new`` workflow safe, while removing the temporary + directory and sample-sheet ceremony from a first evaluation. + """ + library = Path(args.library).expanduser().resolve() + if not library.is_file(): + print(f"dotmatch crispr: library does not exist: {library}", file=sys.stderr) + return 2 + + sources: list[Path] = [] + for pattern in args.fastq: + expanded_pattern = os.path.expanduser(pattern) + matches = [Path(value).expanduser().resolve() for value in sorted(glob.glob(expanded_pattern))] + if not matches and Path(expanded_pattern).is_file(): + matches = [Path(expanded_pattern).resolve()] + sources.extend(matches) + # Preserve command order while rejecting duplicate paths produced by an + # overlapping glob. Deterministic ordering makes pilot results reproducible. + unique_sources = list(dict.fromkeys(sources)) + if not unique_sources: + print("dotmatch crispr: --fastq matched no FASTQ files", file=sys.stderr) + return 2 + invalid = [path for path in unique_sources if path.suffix not in {".fastq", ".fq", ".gz"}] + if invalid: + print(f"dotmatch crispr: inputs do not look like FASTQ files: {invalid[0]}", file=sys.stderr) + return 2 + + project = Path(args.out).expanduser().resolve() + staging_parent = project.parent / f".{project.name}.dotmatch-inputs" + if staging_parent.exists(): + shutil.rmtree(staging_parent) + staging_parent.mkdir(parents=True) + try: + # The scaffold API discovers files in a directory. Symlinking only in + # this disposable staging directory lets the scaffold copy the source + # bytes into a self-contained project, so the generated run remains + # valid after this staging directory is removed. + for source in unique_sources: + staged = staging_parent / source.name + if staged.exists(): + raise ValueError(f"duplicate FASTQ basename: {source.name}") + staged.symlink_to(source) + result = scaffold_assay_project( + template="crispr", + project_dir=project, + reads_dir=staging_parent, + targets=library, + link_reads=False, + threads=args.threads, + max_reads=args.max_reads, + max_start=args.max_start, + ) + print(f"Created reviewable CRISPR project: {result['project']}") + print(f"Review inference: {project / 'inference_report.json'}") + print(f"Run: {project / 'run.sh'}") + if args.no_run or not args.accept_inference: + if not args.no_run: + print( + "Draft project created; review inference_report.json, then rerun with " + "--accept-inference or use dotmatch assay start after setting status = \"ready\"." + ) + return 0 + spec_text = (project / "assay.toml").read_text(encoding="utf-8") + if 'status = "draft"' in spec_text: + (project / "assay.toml").write_text( + spec_text.replace('status = "draft"', 'status = "ready"', 1), + encoding="utf-8", + ) + return command_assay(["start", str(project / "assay.toml")]) + except (AssaySpecError, ValueError, OSError) as exc: + print(f"dotmatch crispr: {exc}", file=sys.stderr) + return 2 + finally: + shutil.rmtree(staging_parent, ignore_errors=True) + + def command_crispr_namespace(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser(prog="dotmatch crispr", description="CRISPR guide-count setup, QC, and AssaySpec helpers.") sub = parser.add_subparsers(dest="command", required=True) @@ -600,6 +681,32 @@ def command_crispr_namespace(argv: Sequence[str]) -> int: new.add_argument("--max-reads", type=int, default=50000) new.add_argument("--max-start", type=int, default=32) + quickstart = sub.add_parser( + "quickstart", + help="infer, preflight, and optionally run a CRISPR project from one or more FASTQ paths", + ) + quickstart.add_argument("--library", required=True, help="guide library CSV/TSV") + quickstart.add_argument( + "--fastq", + required=True, + action="append", + help="FASTQ path or glob; repeat for multiple samples", + ) + quickstart.add_argument("--out", required=True, help="new project directory") + quickstart.add_argument("--threads", type=int, default=1) + quickstart.add_argument("--max-reads", type=int, default=50000) + quickstart.add_argument("--max-start", type=int, default=32) + quickstart.add_argument( + "--no-run", + action="store_true", + help="create and infer the project, but leave execution for review", + ) + quickstart.add_argument( + "--accept-inference", + action="store_true", + help="promote a draft inference to ready and run after reviewing the command inputs", + ) + for name, help_text in { "check": "validate a CRISPR AssaySpec and write preflight reliability artifacts", "plan": "print the native commands for a CRISPR AssaySpec", @@ -635,6 +742,8 @@ def command_crispr_namespace(argv: Sequence[str]) -> int: if args.link_reads: assay_args.append("--link-reads") return command_assay(assay_args) + if args.command == "quickstart": + return command_crispr_quickstart(args) if args.command == "infer": assay_args = [ "infer", @@ -2259,7 +2368,9 @@ def print_top_level_help() -> None: panel Design, certify, simulate, lay out, and export barcode panels. crispr - Convenience commands for CRISPR guide-count workflows. + Convenience commands for CRISPR guide-count workflows. Use + `dotmatch crispr quickstart --library guides.csv --fastq '*.fastq.gz' --out run/` + for an inferred, reviewable first run. Diagnostics and validation: audit --targets targets.tsv|targets.csv --k K --out-dir audit_dir diff --git a/python/tests/test_assayspec.py b/python/tests/test_assayspec.py index e654382d..9bcfdb53 100644 --- a/python/tests/test_assayspec.py +++ b/python/tests/test_assayspec.py @@ -871,6 +871,39 @@ def test_crispr_new_scaffold_matches_assay_new(tmp_path: Path) -> None: assert (project / "assay.toml").exists() +def test_crispr_quickstart_creates_self_contained_reviewable_project(tmp_path: Path) -> None: + targets = _write_inference_targets(tmp_path) + reads = _write_inference_reads(tmp_path, prefix="NN", good=True) + source = tmp_path / "sample.fastq" + reads.rename(source) + project = tmp_path / "quickstart" + + rc = _run_cli( + [ + "crispr", + "quickstart", + "--library", + str(targets), + "--fastq", + str(source), + "--out", + str(project), + "--accept-inference", + ] + ) + + assert rc.returncode == 2, rc.stderr + assert (project / "assay.toml").exists() + assert (project / "inference_report.json").exists() + assert (project / "run.sh").exists() + staged = project / "reads" / "sample.fastq" + assert staged.exists() + assert staged.resolve() != source.resolve() + assert staged.read_bytes() == source.read_bytes() + assert "Created reviewable CRISPR project" in rc.stdout + assert (project / "assay_out" / "reliability_report.html").exists() + + def test_assay_new_refuses_non_empty_project_dir(tmp_path: Path) -> None: targets = _write_inference_targets(tmp_path) reads_dir = tmp_path / "fastqs" diff --git a/python/tests/test_packaging_artifacts.py b/python/tests/test_packaging_artifacts.py index dbdd1e22..5ba3cd00 100644 --- a/python/tests/test_packaging_artifacts.py +++ b/python/tests/test_packaging_artifacts.py @@ -291,4 +291,4 @@ def test_distribution_docs_include_biocontainers_runtime_verification() -> None: assert '"docker", "run", "--rm", image, "dotmatch", "leq", "1", "ACGT", "AGGT"' in checker assert "quay.io/biocontainers/dotmatch:--" in packaging assert "BioContainers images for DotMatch are generated from the accepted Bioconda" in packaging - assert "quay.io/biocontainers/dotmatch:0.1.8--" in readme + assert "quay.io/biocontainers/dotmatch:0.1.9--" in readme diff --git a/python/tests/test_public_language_guardrail.py b/python/tests/test_public_language_guardrail.py new file mode 100644 index 00000000..c2b47df9 --- /dev/null +++ b/python/tests/test_public_language_guardrail.py @@ -0,0 +1,57 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + +PUBLIC_PATHS = [ + ROOT / "README.md", + ROOT / "docs", + ROOT / "app", + ROOT / "public", + ROOT / ".github", +] + +PUBLIC_SUFFIXES = {".css", ".html", ".json", ".md", ".mdx", ".ts", ".tsx", ".yaml", ".yml"} +SKIP_PARTS = {"_build", "node_modules", "__pycache__"} + +FORBIDDEN_PHRASES = [ + "adoption " + "evidence", + "adoption " + "trust", + "ai " + "slop", + "big " + "wins", + "evidence-" + "bounded", + "industry " + "exposure", + "massive industry " + "impact", + "next " + "wins", + "pilot " + "conversations", + "private " + "feedback", + "quote-" + "approved", + "turning private evaluation into public adoption " + "evidence", + "without turning private feedback into public " + "evidence", +] + + +def _public_files(): + for path in PUBLIC_PATHS: + if path.is_file(): + yield path + continue + if not path.exists(): + continue + for candidate in path.rglob("*"): + if not candidate.is_file() or candidate.suffix.lower() not in PUBLIC_SUFFIXES: + continue + if SKIP_PARTS.intersection(candidate.relative_to(ROOT).parts): + continue + yield candidate + + +def test_public_language_avoids_internal_process_phrasing(): + failures = [] + for path in _public_files(): + text = path.read_text(encoding="utf-8").lower() + for phrase in FORBIDDEN_PHRASES: + if phrase in text: + failures.append(f"{path.relative_to(ROOT)} contains forbidden phrase: {phrase}") + + assert failures == [] diff --git a/scripts/check_adoption_metrics.py b/scripts/check_adoption_metrics.py new file mode 100644 index 00000000..1eba4617 --- /dev/null +++ b/scripts/check_adoption_metrics.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Validate the adoption measurement contract without collecting user data.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from check_common import AuditResult, is_https_url, read_json + + +MANIFEST = Path("docs/adoption-metrics.json") +REQUIRED_METRIC_IDS = { + "anaconda_downloads_6m", + "pypi_downloads_30d", + "completed_external_evaluations_30d", + "repeat_workflows_90d", + "accepted_external_integrations", + "docs_to_install_intent", +} +VALID_ROLES = {"distribution_health", "north_star", "retention", "ecosystem", "funnel"} + + +def audit(root: Path) -> AuditResult: + result = AuditResult() + path = root / MANIFEST + try: + data = read_json(path) + except Exception as exc: + result.failures.append(f"{MANIFEST.as_posix()} could not be read: {exc}") + return result + + if data.get("schema_version") != 1: + result.failures.append("adoption metrics must declare schema_version 1") + if data.get("status") != "instrumentation_ready": + result.failures.append("adoption metrics must remain instrumentation_ready until a live dashboard exists") + + north_star = data.get("north_star") + if not isinstance(north_star, dict): + result.failures.append("adoption metrics must declare a north_star object") + else: + for field in ["id", "definition", "target_next_90_days", "why"]: + if not str(north_star.get(field) or "").strip() and north_star.get(field) != 0: + result.failures.append(f"north_star missing {field}") + if north_star.get("id") != "completed_external_evaluations_30d": + result.failures.append("north_star must be completed_external_evaluations_30d") + + metrics = data.get("metrics") + if not isinstance(metrics, list) or len(metrics) != len(REQUIRED_METRIC_IDS): + result.failures.append(f"adoption metrics must contain exactly {len(REQUIRED_METRIC_IDS)} metrics") + metrics = [] + seen: set[str] = set() + for metric in metrics: + if not isinstance(metric, dict): + result.failures.append("adoption metrics entries must be objects") + continue + metric_id = str(metric.get("id") or "").strip() + if metric_id in seen: + result.failures.append(f"duplicate adoption metric id: {metric_id}") + seen.add(metric_id) + for field in ["id", "name", "definition", "source", "cadence", "target", "limitation"]: + if not str(metric.get(field) or "").strip(): + result.failures.append(f"{metric_id or ''} missing {field}") + role = str(metric.get("role") or "").strip() + if role not in VALID_ROLES: + result.failures.append(f"{metric_id or ''} has unsupported role: {role}") + source = str(metric.get("source") or "").strip() + if source.startswith("http") and not is_https_url(source): + result.failures.append(f"{metric_id or ''} source must use https") + missing = REQUIRED_METRIC_IDS - seen + extra = seen - REQUIRED_METRIC_IDS + if missing: + result.failures.append(f"adoption metrics missing ids: {sorted(missing)}") + if extra: + result.failures.append(f"adoption metrics has unexpected ids: {sorted(extra)}") + + rules = data.get("decision_rules") + if not isinstance(rules, list) or len(rules) < 3: + result.failures.append("adoption metrics must declare at least three decision_rules") + joined = " ".join(str(rule) for rule in rules or []).lower() + for phrase in ["download count", "unique-user", "public use"]: + if phrase not in joined: + result.failures.append(f"adoption metrics decision_rules must mention {phrase}") + if not result.failures: + result.passed.append("adoption measurement contract is explicit and non-identifying") + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", default=".", help="repository root") + args = parser.parse_args() + result = audit(Path(args.root)) + for item in result.passed: + print(f"PASS: {item}") + for item in result.failures: + print(f"FAIL: {item}") + if result.ok: + print("ADOPTION METRICS: PASS") + return 0 + print("ADOPTION METRICS: FAIL") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_reviewer_readiness_assets.py b/scripts/check_reviewer_readiness_assets.py new file mode 100644 index 00000000..834d1c0f --- /dev/null +++ b/scripts/check_reviewer_readiness_assets.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +"""Verify DotMatch reviewer materials are concrete, scoped, and public-facing.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from check_common import AuditResult, is_https_url, read_json, uses_placeholder_host + + +ROOT = Path(".") +READINESS = Path("docs/reviewer-readiness.json") +WORKFLOW_PLAN = Path("docs/workflow-integration-plan.json") +INTEGRATIONS = Path("docs/integration-targets.json") +BIOTOOLS = Path("docs/registries/biotools.yml") +ADOPTION_METRICS = Path("docs/adoption-metrics.json") + +EXPECTED_READINESS_IDS = { + "bioinformatics_evaluation_packet", + "external_review_packet", + "integration_target_tracker", + "biotools_registry_draft", + "evaluation_protocol", + "public_use_record_template", + "workflow_integration_issue_template", + "evaluation_feedback_issue_template", + "pull_request_language_checklist", + "reviewer_readiness_gate", +} + +EXPECTED_WORKFLOW_PLAN_IDS = { + "reviewer_decision_tree", + "persona_one_pagers", + "integration_tracker", + "reviewer_packet", + "conference_abstracts", + "technical_communication_pack", + "maintainer_issue_templates", + "evaluation_scorecard", + "integration_tracking_metrics", + "release_communications_calendar", +} + +EXPECTED_INTEGRATION_IDS = { + "nf_core_modules", + "multiqc_module", + "galaxy_iuc", + "snakemake_wrapper", + "bio_tools_record", +} + +ALLOWED_INTEGRATION_STATUSES = { + "local_payload_ready", + "local_parser_ready", + "local_wrapper_ready", + "local_workflow_ready", + "draft_metadata_ready", +} + +PUBLIC_LANGUAGE_FILES = [ + Path("README.md"), + Path("app/page.tsx"), + Path("app/layout.tsx"), + Path("docs/bioinformatics-evaluation.md"), + Path("docs/external-review-packet.md"), + Path("docs/workflow-integration-kit.md"), + Path("docs/workflow-integration-roadmap.md"), + Path("docs/pilot-program.md"), + Path("docs/adopters/README.md"), + Path("docs/adopters/record-template.md"), + Path("docs/adoption-metrics.md"), +] + +REQUIRED_FILES = [ + READINESS, + WORKFLOW_PLAN, + INTEGRATIONS, + BIOTOOLS, + ADOPTION_METRICS, + Path("docs/bioinformatics-evaluation.md"), + Path("docs/external-review-packet.md"), + Path("docs/workflow-integration-kit.md"), + Path("docs/workflow-integration-roadmap.md"), + Path("docs/pilot-program.md"), + Path("docs/adopters/record-template.md"), + Path("docs/adoption-metrics.md"), + Path(".github/ISSUE_TEMPLATE/workflow_integration.yml"), + Path(".github/ISSUE_TEMPLATE/pilot_feedback.yml"), + Path("scripts/check_reviewer_readiness_assets.py"), +] + +HYPE_PHRASES = [ + "massive industry impact", + "ai slop", + "game-changing", + "revolutionary", + "world-class", + "best-in-class", + "enterprise-grade", + "production-ready", + "just works", + "magic", +] + +INTERNAL_PROCESS_PHRASES = [ + "adoption evidence", + "adoption trust", + "evidence-bounded", + "industry exposure", + "next wins", + "pilot conversations", + "private feedback", + "private pilot", + "quote-approved", + "turning private", + "without turning", +] + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _require_file(path: Path, result: AuditResult) -> None: + if not path.exists(): + result.failures.append(f"missing reviewer readiness asset: {path.as_posix()}") + + +def _check_item_plan( + result: AuditResult, + path: Path, + expected_ids: set[str], + label: str, +) -> None: + data = read_json(path) + if data.get("schema_version") != 1: + result.failures.append(f"{label} must declare schema_version 1") + if data.get("status") != "ready_to_execute": + result.failures.append(f"{label} must stay ready_to_execute while public materials are complete") + items = data.get("items") + if not isinstance(items, list) or len(items) != len(expected_ids): + result.failures.append(f"{label} must contain exactly {len(expected_ids)} items") + return + seen: set[str] = set() + for item in items: + if not isinstance(item, dict): + result.failures.append(f"{label} items must be objects") + continue + item_id = str(item.get("id") or "").strip() + asset = Path(str(item.get("asset") or "").split("#", 1)[0].strip()) + if item_id in seen: + result.failures.append(f"duplicate {label} item id: {item_id}") + seen.add(item_id) + for field in ["id", "title", "asset", "done_when"]: + if not str(item.get(field) or "").strip(): + result.failures.append(f"{label} item {item_id or ''} missing {field}") + if asset and not asset.exists(): + result.failures.append(f"{label} item asset missing: {asset.as_posix()}") + missing = expected_ids - seen + extra = seen - expected_ids + if missing: + result.failures.append(f"{label} missing ids: {sorted(missing)}") + if extra: + result.failures.append(f"{label} has unexpected ids: {sorted(extra)}") + + +def _check_integrations(result: AuditResult) -> None: + data = read_json(INTEGRATIONS) + if data.get("schema_version") != 1: + result.failures.append("integration target tracker must declare schema_version 1") + if data.get("status") != "planned": + result.failures.append("integration target tracker must remain planned until external records exist") + if data.get("adoption_record") != "docs/workflow-adoption.json": + result.failures.append("integration tracker must point at docs/workflow-adoption.json") + targets = data.get("targets") + if not isinstance(targets, list) or len(targets) != 5: + result.failures.append("integration target tracker must contain five targets") + return + seen: set[str] = set() + for target in targets: + if not isinstance(target, dict): + result.failures.append("integration targets must be objects") + continue + target_id = str(target.get("id") or "").strip() + seen.add(target_id) + status = str(target.get("status") or "").strip() + if status not in ALLOWED_INTEGRATION_STATUSES: + result.failures.append(f"{target_id} has unsupported planned status: {status}") + url = str(target.get("external_target") or "").strip() + if not is_https_url(url) or uses_placeholder_host(url): + result.failures.append(f"{target_id} must declare a real https external_target") + if target.get("public_record_required") is not True: + result.failures.append(f"{target_id} must require a public record before integration claims") + if not str(target.get("next_action") or "").strip(): + result.failures.append(f"{target_id} must declare next_action") + for asset in target.get("source_assets") or []: + asset_path = Path(str(asset)) + if not asset_path.exists(): + result.failures.append(f"{target_id} source asset missing: {asset_path.as_posix()}") + missing = EXPECTED_INTEGRATION_IDS - seen + extra = seen - EXPECTED_INTEGRATION_IDS + if missing: + result.failures.append(f"integration tracker missing ids: {sorted(missing)}") + if extra: + result.failures.append(f"integration tracker has unexpected ids: {sorted(extra)}") + + +def _check_biotools(result: AuditResult) -> None: + text = _read(BIOTOOLS) + for phrase in [ + "name: DotMatch", + "biotoolsID: dotmatch", + "documentation: https://dotmatch.readthedocs.io/", + "license: Apache-2.0", + "known-target short-DNA assignment", + "Draft registry metadata", + ]: + if phrase not in text: + result.failures.append(f"bio.tools draft missing: {phrase}") + if "accepted bio.tools record" in text and "Do not describe" not in text: + result.failures.append("bio.tools draft must not imply accepted registry status") + + +def _check_templates(result: AuditResult) -> None: + workflow_issue = _read(Path(".github/ISSUE_TEMPLATE/workflow_integration.yml")) + evaluation_issue = _read(Path(".github/ISSUE_TEMPLATE/pilot_feedback.yml")) + pr_template = _read(Path(".github/PULL_REQUEST_TEMPLATE.md")) + for phrase in ["Workflow Manager", "Expected Outputs", "Validated Scope"]: + if phrase not in workflow_issue: + result.failures.append(f"workflow integration template missing: {phrase}") + for phrase in ["Assay Context", "Outputs Reviewed", "Public Use Permission"]: + if phrase not in evaluation_issue: + result.failures.append(f"evaluation feedback template missing: {phrase}") + for phrase in [ + "Public Language and Reviewer Readiness", + "This PR does not imply accepted external workflow integration", + "Broad replacement wording or launch copy was removed or avoided", + ]: + if phrase not in pr_template: + result.failures.append(f"PR template missing reviewer readiness checkbox: {phrase}") + + +def _check_docs_wiring(result: AuditResult) -> None: + readme = _read(Path("README.md")) + index = _read(Path("docs/index.md")) + page = _read(Path("app/page.tsx")) + public_text = readme + index + page + for path in [ + "docs/external-review-packet.md", + "docs/integration-targets.json", + "docs/pilot-program.md", + "docs/reviewer-readiness.json", + "docs/workflow-integration-kit.md", + "docs/workflow-integration-roadmap.md", + "docs/adoption-metrics.json", + ]: + if path not in public_text: + result.failures.append(f"public docs do not link {path}") + + +def _check_public_language(result: AuditResult) -> None: + forbidden = HYPE_PHRASES + INTERNAL_PROCESS_PHRASES + for path in PUBLIC_LANGUAGE_FILES: + text = _read(path).lower() + for phrase in forbidden: + if phrase in text: + result.failures.append(f"{path.as_posix()} contains public-facing internal or hype phrase: {phrase}") + + +def audit(root: Path) -> AuditResult: + result = AuditResult() + for path in REQUIRED_FILES: + _require_file(path, result) + if result.failures: + return result + _check_item_plan(result, READINESS, EXPECTED_READINESS_IDS, "reviewer readiness record") + _check_item_plan(result, WORKFLOW_PLAN, EXPECTED_WORKFLOW_PLAN_IDS, "workflow integration plan") + _check_integrations(result) + _check_biotools(result) + _check_templates(result) + _check_docs_wiring(result) + _check_public_language(result) + if result.ok: + result.passed.append("reviewer materials are concrete, scoped, and public-facing") + return result + + +def main() -> int: + result = audit(ROOT) + for item in result.passed: + print(f"PASS: {item}") + for item in result.failures: + print(f"FAIL: {item}") + if result.ok: + print("REVIEWER READINESS: PASS") + return 0 + print("REVIEWER READINESS: FAIL") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_site_assets.mjs b/scripts/check_site_assets.mjs index 5b306dcc..df85f633 100644 --- a/scripts/check_site_assets.mjs +++ b/scripts/check_site_assets.mjs @@ -91,7 +91,7 @@ for (const phrase of [ "CRISPR screen teams", "Workflow maintainers", "Assay developers", - "Industry exposure kit", + "Outreach and integration kit", "Workflow submission pack", "Next 10 exposure wins", "Decision tree", @@ -186,7 +186,7 @@ if (!sitemap.includes("MetadataRoute.Sitemap") || !sitemap.includes("changeFrequ } for (const phrase of [ - "The Big 5 Wins", + "The Big 5 Distribution Moves", "Next 10 Wins", "Workflow distribution handoff", "Citation and methods flywheel", diff --git a/src/qda.c b/src/qda.c index 5bce7552..f1985296 100644 --- a/src/qda.c +++ b/src/qda.c @@ -1,3 +1,6 @@ +Warning: truncated output (original token count: 100844) +Total output lines: 9730 + #ifndef _POSIX_C_SOURCE #define _POSIX_C_SOURCE 200809L #endif @@ -29,7 +32,7 @@ #include #ifndef DOTMATCH_VERSION -#define DOTMATCH_VERSION "0.1.8" +#define DOTMATCH_VERSION "0.1.9" #endif #define MAX_AUTO_OFFSET 1024 @@ -4830,83 +4833,7 @@ static int run_count(const char *argv0, int argc, char **argv) { size_t indel_window = 0; size_t auto_offset = 0; size_t auto_offset_sample = 1000; - offset_mode offsets_mode = OFFSET_MODE_BEST; - double offset_min_fraction = 0.005; - size_t threads = 0; - int max_correction_qual = -1; - int k = -1; - count_backend_mode backend_mode = COUNT_BACKEND_AUTO; - string_list reads = {0}; - string_list labels = {0}; - int show_progress = isatty(STDERR_FILENO); - size_t progress_interval_reads = 250000; - char default_sample_qc_path[PATH_MAX]; - count_progress *progress_by_sample = NULL; - int metal_validate = env_truthy(getenv("DOTMATCH_METAL_VALIDATE")); - const char *metal_validation_status = NULL; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if ((strcmp(arg, "--targets") == 0 || strcmp(arg, "--library") == 0) && i < argc) { - targets_path = argv[i++]; - } else if (strcmp(arg, "--samples") == 0 && i < argc) { - samples_path = argv[i++]; - } else if (strcmp(arg, "--reads") == 0 && i < argc) { - if (push_string(&reads, argv[i++]) != 0) { - fprintf(stderr, "out of memory\n"); - goto fail_args; - } - } else if (strcmp(arg, "--sample-label") == 0 && i < argc) { - if (split_string_list(&labels, argv[i++], ',') != 0) { - fprintf(stderr, "out of memory\n"); - goto fail_args; - } - } else if ((strcmp(arg, "--target-start") == 0 || strcmp(arg, "--guide-start") == 0) && i < argc) { - if (parse_size_value(argv[i++], &target_start) != 0) { - usage(argv0); - goto fail_args; - } - } else if ((strcmp(arg, "--target-length") == 0 || strcmp(arg, "--guide-length") == 0) && i < argc) { - if (parse_size_value(argv[i++], &target_len) != 0 || target_len == 0) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 3) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--metric") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "hamming") == 0) { - metric = COUNT_METRIC_HAMMING; - } else if (strcmp(value, "levenshtein") == 0) { - metric = COUNT_METRIC_LEVENSHTEIN; - } else { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--hamming-index") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "auto") == 0) { - hamming_strategy = HAMMING_INDEX_AUTO; - } else if (strcmp(value, "query") == 0) { - hamming_strategy = HAMMING_INDEX_QUERY; - } else if (strcmp(value, "precompute") == 0) { - hamming_strategy = HAMMING_INDEX_PRECOMPUTE; - } else { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--indel-window") == 0 && i < argc) { - if (parse_size_value(argv[i++], &indel_window) != 0 || indel_window > 1) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--auto-offset") == 0 && i < argc) { - if (parse_size_value(argv[i++], &auto_offset) != 0) { - usage(argv0); + offset_mod…844 tokens truncated… usage(argv0); goto fail_args; } } else if (strcmp(arg, "--auto-offset-sample") == 0 && i < argc) { From 3d718c281bcd4b3a7a77fd4813a3e67ca401efee Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Mon, 13 Jul 2026 15:04:37 +0100 Subject: [PATCH 02/93] Preserve complete native CLI source in release branch --- src/qda.c | 8419 +---------------------------------------------------- 1 file changed, 1 insertion(+), 8418 deletions(-) diff --git a/src/qda.c b/src/qda.c index f1985296..802d5640 100644 --- a/src/qda.c +++ b/src/qda.c @@ -1,6 +1,3 @@ -Warning: truncated output (original token count: 100844) -Total output lines: 9730 - #ifndef _POSIX_C_SOURCE #define _POSIX_C_SOURCE 200809L #endif @@ -1240,8418 +1237,4 @@ static void free_offset_list(offset_list *list) { static int offset_list_contains(const offset_list *list, size_t offset) { for (size_t i = 0; i < list->count; ++i) { - if (list->items[i] == offset) return 1; - } - return 0; -} - -static int push_offset_unique(offset_list *list, size_t offset) { - if (offset_list_contains(list, offset)) return 0; - if (list->count == list->cap) { - size_t next_cap = list->cap == 0 ? 8 : list->cap * 2; - size_t *next = (size_t *)realloc(list->items, next_cap * sizeof(size_t)); - if (next == NULL) return -1; - list->items = next; - list->cap = next_cap; - } - list->items[list->count++] = offset; - return 0; -} - -static size_t first_selected_offset(const offset_list *list, size_t fallback) { - return list != NULL && list->count != 0 ? list->items[0] : fallback; -} - -typedef struct hamming_lookup_entry { - uint64_t code; - int target_index; - int match_count; -} hamming_lookup_entry; - -typedef struct hamming_seed_entry { - uint64_t code; - int target_index; - int next; - unsigned char seed_id; -} hamming_seed_entry; - -typedef struct hamming_lookup { - hamming_lookup_entry *exact; - hamming_lookup_entry *mismatch; - hamming_seed_entry *seeds; - int *seed_heads; - uint64_t *target_codes; - size_t exact_cap; - size_t mismatch_cap; - size_t seed_hash_cap; - size_t n_seeds; - size_t target_len; - size_t seed0_len; - int ready; - int seed_ready; -} hamming_lookup; - -typedef struct levenshtein1_lookup { - hamming_lookup_entry *exact; - hamming_lookup_entry *substitution; - hamming_lookup_entry *target_deletion; - hamming_lookup_entry *target_insertion; - size_t exact_cap; - size_t substitution_cap; - size_t target_deletion_cap; - size_t target_insertion_cap; - size_t target_len; - int ready; -} levenshtein1_lookup; - -static const char *hamming_lookup_kind(const hamming_lookup *lookup) { - if (lookup == NULL || !lookup->ready) return "query"; - if (lookup->mismatch != NULL && lookup->mismatch_cap != 0) return "precompute"; - if (lookup->seed_ready) return "seed"; - return "exact"; -} - -static size_t next_pow2_local(size_t n) { - size_t p = 1; - while (p < n && p <= (SIZE_MAX >> 1)) p <<= 1; - return p < n ? n : p; -} - -static inline size_t code_hash_local(uint64_t code, size_t len, size_t cap) { - uint64_t x = code ^ ((uint64_t)len * 0x9e3779b97f4a7c15ULL); - x *= 0x9e3779b97f4a7c15ULL; - x ^= x >> 32; - return (size_t)x & (cap - 1); -} - -static inline size_t seed_hash_local(uint64_t code, size_t len, unsigned char seed_id, size_t cap) { - return code_hash_local(code ^ ((uint64_t)seed_id * 0x517cc1b727220a95ULL), len + seed_id * 37U, cap); -} - -static uint64_t code_low_mask_local(size_t len) { - if (len == 0) return 0; - if (len >= 32) return UINT64_MAX; - return (1ULL << (2 * len)) - 1ULL; -} - -static uint64_t code_segment_local(uint64_t code, size_t start, size_t len) { - return (code >> (2 * start)) & code_low_mask_local(len); -} - -static int hamming_code_distance_local(uint64_t a, uint64_t b, size_t len) { - uint64_t diff = a ^ b; - diff |= diff >> 1; - diff &= code_low_mask_local(len); - diff &= 0x5555555555555555ULL; -#if defined(__GNUC__) || defined(__clang__) - return __builtin_popcountll(diff); -#else - int d = 0; - while (diff != 0) { - d += (int)(diff & 1ULL); - diff >>= 2; - } - return d; -#endif -} - -static int dna2_code_local(const char *s, size_t len, uint64_t *code_out) { - if (s == NULL && len != 0) return 0; - if (len > 32) return 0; - uint64_t code = 0; - for (size_t i = 0; i < len; ++i) { - uint64_t v; - switch (s[i]) { - case 'A': - v = 0; - break; - case 'C': - v = 1; - break; - case 'G': - v = 2; - break; - case 'T': - v = 3; - break; - default: - return 0; - } - code |= v << (2 * i); - } - *code_out = code; - return 1; -} - -static int dna2_code_local_fold(const char *s, size_t len, uint64_t *code_out) { - if (s == NULL && len != 0) return 0; - if (len > 32) return 0; - uint64_t code = 0; - for (size_t i = 0; i < len; ++i) { - uint64_t v; - switch (s[i]) { - case 'A': - case 'a': - v = 0; - break; - case 'C': - case 'c': - v = 1; - break; - case 'G': - case 'g': - v = 2; - break; - case 'T': - case 't': - v = 3; - break; - default: - return 0; - } - code |= v << (2 * i); - } - *code_out = code; - return 1; -} - -static int dna2_base_fold_value(char c, uint64_t *value_out) { - switch (c) { - case 'A': - case 'a': - *value_out = 0; - return 1; - case 'C': - case 'c': - *value_out = 1; - return 1; - case 'G': - case 'g': - *value_out = 2; - return 1; - case 'T': - case 't': - *value_out = 3; - return 1; - default: - *value_out = 0; - return 0; - } -} - -static void copy_upper_ascii_window(char *dst, size_t dst_cap, const char *src, size_t len) { - if (dst_cap == 0) return; - if (len >= dst_cap) len = dst_cap - 1; - for (size_t i = 0; i < len; ++i) { - unsigned char c = (unsigned char)src[i]; - dst[i] = (char)(c >= 'a' && c <= 'z' ? c - 32 : c); - } - dst[len] = '\0'; -} - -static char uppercase_ascii_char(char c) { - unsigned char uc = (unsigned char)c; - return (char)(uc >= 'a' && uc <= 'z' ? uc - 32 : uc); -} - -static int phred33_quality(char c) { - unsigned char uc = (unsigned char)c; - return uc < 33 ? 0 : (int)uc - 33; -} - -static int window_matches_observed_folded(const char *seq, size_t seq_len, size_t start, - const char *observed, size_t observed_len) { - if (start > seq_len || observed_len > seq_len - start) return 0; - for (size_t i = 0; i < observed_len; ++i) { - if (uppercase_ascii_char(seq[start + i]) != observed[i]) return 0; - } - return 1; -} - -static int observed_base_qualities_within_edit(const char *observed, size_t observed_len, - const char *target, size_t target_len, - const char *qual, size_t qual_start, - int max_correction_qual) { - if (observed_len == target_len) { - int saw_mismatch = 0; - for (size_t i = 0; i < observed_len; ++i) { - if (observed[i] == target[i]) continue; - saw_mismatch = 1; - if (phred33_quality(qual[qual_start + i]) > max_correction_qual) return 0; - } - return saw_mismatch; - } - - if (observed_len == target_len + 1) { - for (size_t drop = 0; drop < observed_len; ++drop) { - size_t ti = 0; - int matches = 1; - for (size_t oi = 0; oi < observed_len; ++oi) { - if (oi == drop) continue; - if (observed[oi] != target[ti++]) { - matches = 0; - break; - } - } - if (matches && phred33_quality(qual[qual_start + drop]) <= max_correction_qual) return 1; - } - return 0; - } - - if (observed_len + 1 == target_len) { - return 1; - } - - return 0; -} - -static int hamming_lookup_insert(hamming_lookup_entry *table, size_t cap, uint64_t code, int target_index) { - size_t slot = code_hash_local(code, 0, cap); - for (;;) { - hamming_lookup_entry *entry = &table[slot]; - if (entry->target_index < 0) { - entry->code = code; - entry->target_index = target_index; - entry->match_count = 1; - return 0; - } - if (entry->code == code) { - if (entry->target_index != target_index) { - if (target_index < entry->target_index) entry->target_index = target_index; - ++entry->match_count; - } - return 0; - } - slot = (slot + 1) & (cap - 1); - } -} - -static const hamming_lookup_entry *hamming_lookup_find(const hamming_lookup_entry *table, size_t cap, uint64_t code) { - if (table == NULL || cap == 0) return NULL; - size_t slot = code_hash_local(code, 0, cap); - for (;;) { - const hamming_lookup_entry *entry = &table[slot]; - if (entry->target_index < 0) return NULL; - if (entry->code == code) return entry; - slot = (slot + 1) & (cap - 1); - } -} - -static int levenshtein1_lookup_insert(hamming_lookup_entry *table, size_t cap, uint64_t code, size_t len, - int target_index) { - size_t slot = code_hash_local(code, len, cap); - for (;;) { - hamming_lookup_entry *entry = &table[slot]; - if (entry->target_index < 0) { - entry->code = code; - entry->target_index = target_index; - entry->match_count = 1; - return 0; - } - if (entry->code == code) { - if (entry->target_index != target_index) { - if (target_index < entry->target_index) entry->target_index = target_index; - ++entry->match_count; - } - return 0; - } - slot = (slot + 1) & (cap - 1); - } -} - -static const hamming_lookup_entry *levenshtein1_lookup_find(const hamming_lookup_entry *table, size_t cap, - uint64_t code, size_t len) { - if (table == NULL || cap == 0) return NULL; - size_t slot = code_hash_local(code, len, cap); - for (;;) { - const hamming_lookup_entry *entry = &table[slot]; - if (entry->target_index < 0) return NULL; - if (entry->code == code) return entry; - slot = (slot + 1) & (cap - 1); - } -} - -static hamming_lookup_entry *alloc_hamming_table(size_t cap) { - hamming_lookup_entry *table = (hamming_lookup_entry *)malloc(cap * sizeof(hamming_lookup_entry)); - if (table == NULL) return NULL; - for (size_t i = 0; i < cap; ++i) { - table[i].code = 0; - table[i].target_index = -1; - table[i].match_count = 0; - } - return table; -} - -static int hamming_seed_insert(hamming_lookup *lookup, unsigned char seed_id, uint64_t code, int target_index) { - if (lookup->n_seeds > (size_t)INT32_MAX) return -1; - size_t seed_len = seed_id == 0 ? lookup->seed0_len : lookup->target_len - lookup->seed0_len; - size_t slot = seed_hash_local(code, seed_len, seed_id, lookup->seed_hash_cap); - size_t e = lookup->n_seeds++; - lookup->seeds[e].code = code; - lookup->seeds[e].target_index = target_index; - lookup->seeds[e].seed_id = seed_id; - lookup->seeds[e].next = lookup->seed_heads[slot]; - lookup->seed_heads[slot] = (int)e; - return 0; -} - -static void free_hamming_lookup(hamming_lookup *lookup) { - if (lookup == NULL) return; - free(lookup->exact); - free(lookup->mismatch); - free(lookup->seeds); - free(lookup->seed_heads); - free(lookup->target_codes); - lookup->exact = NULL; - lookup->mismatch = NULL; - lookup->seeds = NULL; - lookup->seed_heads = NULL; - lookup->target_codes = NULL; - lookup->exact_cap = 0; - lookup->mismatch_cap = 0; - lookup->seed_hash_cap = 0; - lookup->n_seeds = 0; - lookup->target_len = 0; - lookup->seed0_len = 0; - lookup->ready = 0; - lookup->seed_ready = 0; -} - -static void free_levenshtein1_lookup(levenshtein1_lookup *lookup) { - if (lookup == NULL) return; - free(lookup->exact); - free(lookup->substitution); - free(lookup->target_deletion); - free(lookup->target_insertion); - memset(lookup, 0, sizeof(*lookup)); -} - -static uint64_t code_remove_base_local(uint64_t code, size_t pos, size_t len) { - uint64_t low = code & code_low_mask_local(pos); - uint64_t high = code >> (2 * (pos + 1)); - (void)len; - return low | (high << (2 * pos)); -} - -static uint64_t code_insert_base_local(uint64_t code, size_t pos, size_t len, uint64_t base) { - uint64_t low = code & code_low_mask_local(pos); - uint64_t high = (code >> (2 * pos)) & code_low_mask_local(len - pos); - return low | (base << (2 * pos)) | (high << (2 * (pos + 1))); -} - -static int build_levenshtein1_lookup(const seq_table *targets, size_t target_len, levenshtein1_lookup *lookup) { - memset(lookup, 0, sizeof(*lookup)); - if (target_len == 0 || target_len > 31) return 0; - for (size_t i = 0; i < targets->count; ++i) { - uint64_t code = 0; - if (targets->records[i].len != target_len || - !dna2_code_local(targets->records[i].seq, target_len, &code)) { - return 0; - } - } - - lookup->exact_cap = next_pow2_local(targets->count * 2 + 16); - lookup->substitution_cap = next_pow2_local(targets->count * target_len * 3 + 16); - lookup->target_deletion_cap = next_pow2_local(targets->count * target_len + 16); - lookup->target_insertion_cap = next_pow2_local(targets->count * (target_len + 1) * 4 + 16); - lookup->exact = alloc_hamming_table(lookup->exact_cap); - lookup->substitution = alloc_hamming_table(lookup->substitution_cap); - lookup->target_deletion = alloc_hamming_table(lookup->target_deletion_cap); - lookup->target_insertion = alloc_hamming_table(lookup->target_insertion_cap); - if (lookup->exact == NULL || lookup->substitution == NULL || lookup->target_deletion == NULL || - lookup->target_insertion == NULL) { - free_levenshtein1_lookup(lookup); - return -1; - } - lookup->target_len = target_len; - - for (size_t i = 0; i < targets->count; ++i) { - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { - free_levenshtein1_lookup(lookup); - return 0; - } - if (levenshtein1_lookup_insert(lookup->exact, lookup->exact_cap, code, target_len, (int)i) != 0) { - free_levenshtein1_lookup(lookup); - return -1; - } - for (size_t pos = 0; pos < target_len; ++pos) { - uint64_t shift = (uint64_t)2 * pos; - uint64_t old_base = (code >> shift) & 3ULL; - uint64_t mask = 3ULL << shift; - for (uint64_t b = 0; b < 4; ++b) { - if (b == old_base) continue; - uint64_t mutated = (code & ~mask) | (b << shift); - if (levenshtein1_lookup_insert(lookup->substitution, lookup->substitution_cap, mutated, - target_len, (int)i) != 0) { - free_levenshtein1_lookup(lookup); - return -1; - } - } - uint64_t deleted = code_remove_base_local(code, pos, target_len); - if (levenshtein1_lookup_insert(lookup->target_deletion, lookup->target_deletion_cap, deleted, - target_len - 1, (int)i) != 0) { - free_levenshtein1_lookup(lookup); - return -1; - } - } - for (size_t pos = 0; pos <= target_len; ++pos) { - for (uint64_t b = 0; b < 4; ++b) { - uint64_t inserted = code_insert_base_local(code, pos, target_len, b); - if (levenshtein1_lookup_insert(lookup->target_insertion, lookup->target_insertion_cap, inserted, - target_len + 1, (int)i) != 0) { - free_levenshtein1_lookup(lookup); - return -1; - } - } - } - } - lookup->ready = 1; - return 0; -} - -static int build_hamming_lookup(const seq_table *targets, size_t target_len, hamming_lookup *lookup) { - memset(lookup, 0, sizeof(*lookup)); - if (target_len == 0 || target_len > 32) return 0; - for (size_t i = 0; i < targets->count; ++i) { - if (targets->records[i].len != target_len) return 0; - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) return 0; - } - - size_t exact_need = targets->count * 2 + 16; - /* Tuned *3 (not *4) since exactly 3 mutations per position; smaller mismatch table - improves cache residency for hamming_lookup precompute k=1 guide/barcode counting. */ - size_t mismatch_need = targets->count * target_len * 3 + 16; - lookup->exact_cap = next_pow2_local(exact_need); - lookup->mismatch_cap = next_pow2_local(mismatch_need); - lookup->exact = alloc_hamming_table(lookup->exact_cap); - lookup->mismatch = alloc_hamming_table(lookup->mismatch_cap); - if (lookup->exact == NULL || lookup->mismatch == NULL) { - free_hamming_lookup(lookup); - return -1; - } - lookup->target_len = target_len; - - for (size_t i = 0; i < targets->count; ++i) { - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { - free_hamming_lookup(lookup); - return 0; - } - if (hamming_lookup_insert(lookup->exact, lookup->exact_cap, code, (int)i) != 0) { - free_hamming_lookup(lookup); - return -1; - } - for (size_t pos = 0; pos < target_len; ++pos) { - uint64_t shift = (uint64_t)2 * pos; - uint64_t old_base = (code >> shift) & 3ULL; - uint64_t mask = 3ULL << shift; - for (uint64_t b = 0; b < 4; ++b) { - if (b == old_base) continue; - uint64_t mutated = (code & ~mask) | (b << shift); - if (hamming_lookup_insert(lookup->mismatch, lookup->mismatch_cap, mutated, (int)i) != 0) { - free_hamming_lookup(lookup); - return -1; - } - } - } - } - lookup->ready = 1; - return 0; -} - -static int build_hamming_exact_lookup(const seq_table *targets, size_t target_len, hamming_lookup *lookup) { - memset(lookup, 0, sizeof(*lookup)); - if (target_len == 0 || target_len > 32) return 0; - for (size_t i = 0; i < targets->count; ++i) { - if (targets->records[i].len != target_len) return 0; - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) return 0; - } - - size_t exact_need = targets->count * 2 + 16; - lookup->exact_cap = next_pow2_local(exact_need); - lookup->exact = alloc_hamming_table(lookup->exact_cap); - if (lookup->exact == NULL) { - free_hamming_lookup(lookup); - return -1; - } - lookup->target_len = target_len; - - for (size_t i = 0; i < targets->count; ++i) { - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { - free_hamming_lookup(lookup); - return 0; - } - if (hamming_lookup_insert(lookup->exact, lookup->exact_cap, code, (int)i) != 0) { - free_hamming_lookup(lookup); - return -1; - } - } - lookup->ready = 1; - return 0; -} - -static int build_hamming_seed_lookup(const seq_table *targets, size_t target_len, hamming_lookup *lookup) { - memset(lookup, 0, sizeof(*lookup)); - if (target_len < 2 || target_len > 32) return build_hamming_lookup(targets, target_len, lookup); - for (size_t i = 0; i < targets->count; ++i) { - if (targets->records[i].len != target_len) return 0; - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) return 0; - } - - size_t exact_need = targets->count * 2 + 16; - size_t seed_need = targets->count * 2 + 16; - lookup->exact_cap = next_pow2_local(exact_need); - lookup->seed_hash_cap = next_pow2_local(seed_need * 2 + 1); - lookup->exact = alloc_hamming_table(lookup->exact_cap); - lookup->seeds = (hamming_seed_entry *)malloc(seed_need * sizeof(hamming_seed_entry)); - lookup->seed_heads = (int *)malloc(lookup->seed_hash_cap * sizeof(int)); - lookup->target_codes = (uint64_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(uint64_t)); - if (lookup->exact == NULL || lookup->seeds == NULL || lookup->seed_heads == NULL || lookup->target_codes == NULL) { - free_hamming_lookup(lookup); - return -1; - } - for (size_t i = 0; i < lookup->seed_hash_cap; ++i) lookup->seed_heads[i] = -1; - lookup->target_len = target_len; - lookup->seed0_len = target_len / 2; - - for (size_t i = 0; i < targets->count; ++i) { - uint64_t code = 0; - if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { - free_hamming_lookup(lookup); - return 0; - } - lookup->target_codes[i] = code; - if (hamming_lookup_insert(lookup->exact, lookup->exact_cap, code, (int)i) != 0) { - free_hamming_lookup(lookup); - return -1; - } - uint64_t seed0 = code_segment_local(code, 0, lookup->seed0_len); - uint64_t seed1 = code_segment_local(code, lookup->seed0_len, target_len - lookup->seed0_len); - if (hamming_seed_insert(lookup, 0, seed0, (int)i) != 0 || - hamming_seed_insert(lookup, 1, seed1, (int)i) != 0) { - free_hamming_lookup(lookup); - return -1; - } - } - lookup->ready = 1; - lookup->seed_ready = 1; - return 0; -} - -static int cmp_ull_desc(const void *a, const void *b) { - unsigned long long aa = *(const unsigned long long *)a; - unsigned long long bb = *(const unsigned long long *)b; - return aa < bb ? 1 : (aa > bb ? -1 : 0); -} - -static int cmp_ull_asc(const void *a, const void *b) { - unsigned long long aa = *(const unsigned long long *)a; - unsigned long long bb = *(const unsigned long long *)b; - return aa > bb ? 1 : (aa < bb ? -1 : 0); -} - -static double gini_from_counts(const unsigned long long *values, size_t n) { - if (n == 0) return 0.0; - unsigned long long *tmp = (unsigned long long *)malloc(n * sizeof(unsigned long long)); - if (tmp == NULL) return 0.0; - unsigned long long sum = 0; - for (size_t i = 0; i < n; ++i) { - tmp[i] = values[i]; - sum += values[i]; - } - if (sum == 0) { - free(tmp); - return 0.0; - } - qsort(tmp, n, sizeof(unsigned long long), cmp_ull_asc); - long double weighted = 0.0; - for (size_t i = 0; i < n; ++i) weighted += (long double)(i + 1) * (long double)tmp[i]; - free(tmp); - long double gini = (2.0L * weighted / ((long double)n * (long double)sum)) - - (((long double)n + 1.0L) / (long double)n); - if (gini < 0.0L) return 0.0; - if (gini > 1.0L) return 1.0; - return (double)gini; -} - -static double top_fraction_from_counts(const unsigned long long *values, size_t n, double fraction) { - if (n == 0) return 0.0; - unsigned long long *tmp = (unsigned long long *)malloc(n * sizeof(unsigned long long)); - if (tmp == NULL) return 0.0; - unsigned long long sum = 0; - for (size_t i = 0; i < n; ++i) { - tmp[i] = values[i]; - sum += values[i]; - } - if (sum == 0) { - free(tmp); - return 0.0; - } - qsort(tmp, n, sizeof(unsigned long long), cmp_ull_desc); - size_t top_n = (size_t)((double)n * fraction); - if (top_n == 0) top_n = 1; - if (top_n > n) top_n = n; - unsigned long long top_sum = 0; - for (size_t i = 0; i < top_n; ++i) top_sum += tmp[i]; - free(tmp); - return (double)top_sum / (double)sum; -} - -static int compute_sample_qc_metrics(const seq_table *targets, const unsigned long long *counts, size_t sample_index, - const count_stats *stats, sample_qc_metrics *metrics_out) { - if (targets == NULL || counts == NULL || stats == NULL || metrics_out == NULL) return -1; - unsigned long long *target_totals = - (unsigned long long *)calloc(targets->count == 0 ? 1 : targets->count, sizeof(unsigned long long)); - if (target_totals == NULL) return -1; - unsigned long long observed_targets = 0; - for (size_t t = 0; t < targets->count; ++t) { - for (size_t kind = 0; kind < 5; ++kind) { - target_totals[t] += counts[((sample_index * targets->count + t) * 5) + kind]; - } - if (target_totals[t] != 0) ++observed_targets; - } - unsigned long long valid = stats->total >= stats->invalid ? stats->total - stats->invalid : 0; - double valid_denom = valid == 0 ? 1.0 : (double)valid; - metrics_out->assignment_rate = (double)stats->unique / valid_denom; - metrics_out->ambiguous_rate = (double)stats->ambiguous / valid_denom; - metrics_out->no_match_rate = (double)stats->unmatched / valid_denom; - metrics_out->invalid_rate = stats->total == 0 ? 0.0 : (double)stats->invalid / (double)stats->total; - metrics_out->coverage_fraction = - targets->count == 0 ? 0.0 : (double)observed_targets / (double)targets->count; - metrics_out->zero_count_fraction = - targets->count == 0 ? 0.0 : (double)(targets->count - observed_targets) / (double)targets->count; - metrics_out->gini_index = gini_from_counts(target_totals, targets->count); - metrics_out->top_1pct_fraction = top_fraction_from_counts(target_totals, targets->count, 0.01); - free(target_totals); - return 0; -} - -static void emit_sample_qc_review_warnings(const string_list *labels, const sample_qc_metrics *metrics, size_t count) { - if (labels == NULL || metrics == NULL || count == 0) return; - int any = 0; - for (size_t sample = 0; sample < count; ++sample) { - const sample_qc_metrics *m = &metrics[sample]; - int sample_warn = 0; - if (m->assignment_rate < 0.80) sample_warn = 1; - if (m->ambiguous_rate > 0.05) sample_warn = 1; - if (m->no_match_rate > 0.15) sample_warn = 1; - if (m->invalid_rate > 0.02) sample_warn = 1; - if (m->coverage_fraction < 0.90) sample_warn = 1; - if (m->zero_count_fraction > 0.10) sample_warn = 1; - if (m->gini_index > 0.50) sample_warn = 1; - if (m->top_1pct_fraction > 0.30) sample_warn = 1; - if (!sample_warn) continue; - any = 1; - fprintf(stderr, "dotmatch: QC review recommended for sample %s:", labels->items[sample]); - if (m->assignment_rate < 0.80) fprintf(stderr, " assignment_rate=%.1f%%", 100.0 * m->assignment_rate); - if (m->ambiguous_rate > 0.05) fprintf(stderr, " ambiguous_rate=%.1f%%", 100.0 * m->ambiguous_rate); - if (m->no_match_rate > 0.15) fprintf(stderr, " no_match_rate=%.1f%%", 100.0 * m->no_match_rate); - if (m->invalid_rate > 0.02) fprintf(stderr, " invalid_rate=%.1f%%", 100.0 * m->invalid_rate); - if (m->coverage_fraction < 0.90) fprintf(stderr, " coverage=%.1f%%", 100.0 * m->coverage_fraction); - if (m->zero_count_fraction > 0.10) fprintf(stderr, " zero_count_guides=%.1f%%", 100.0 * m->zero_count_fraction); - if (m->gini_index > 0.50) fprintf(stderr, " gini=%.2f", m->gini_index); - if (m->top_1pct_fraction > 0.30) fprintf(stderr, " top_1pct_fraction=%.1f%%", 100.0 * m->top_1pct_fraction); - fprintf(stderr, "\n"); - } - if (any) { - fprintf(stderr, - "dotmatch: review sample_qc.tsv and summary.json before downstream MAGeCK/BAGEL analysis; " - "thresholds are conservative diagnostics, not biological pass/fail rules\n"); - } -} - -typedef enum ambiguity_policy { - AMBIGUITY_POLICY_BEST = 0, - AMBIGUITY_POLICY_RADIUS = 1 -} ambiguity_policy; - -static const char *ambiguity_policy_name(ambiguity_policy policy) { - return policy == AMBIGUITY_POLICY_RADIUS ? "radius" : "best"; -} - -static int apply_ambiguity_policy(qdaln_match_result *result, ambiguity_policy policy) { - if (policy == AMBIGUITY_POLICY_RADIUS && result->status == QDALN_MATCH_UNIQUE && result->match_count > 1) { - result->status = QDALN_MATCH_AMBIGUOUS; - } - return 0; -} - -static void html_escape(FILE *out, const char *s); - -static void write_tsv_preview_table(FILE *out, const char *title, const char *path, size_t max_rows) { - FILE *in = fopen(path, "r"); - if (in == NULL) return; - fprintf(out, "

      "); - html_escape(out, title); - fprintf(out, "

      \n"); - char line[16384]; - size_t row = 0; - while (row <= max_rows && fgets(line, sizeof(line), in) != NULL) { - trim_line(line); - fprintf(out, ""); - char *fields[128]; - size_t n = split_fields(line, '\t', fields, 128); - for (size_t i = 0; i < n; ++i) { - fprintf(out, row == 0 ? ""); - } - fprintf(out, "\n"); - ++row; - } - fprintf(out, "
      " : ""); - html_escape(out, fields[i]); - fprintf(out, row == 0 ? "" : "
      \n"); - fclose(in); -} - -static int write_count_html_report(const char *path, const seq_table *targets, const string_list *reads, - const string_list *labels, const unsigned long long *counts, - const count_stats *stats_by_sample, const offset_list *selected_offsets, - int k, count_metric metric, ambiguity_policy policy, size_t target_len, - const char *audit_dir, const char *unmatched_report_path) { - FILE *out = open_output_file(path); - if (out == NULL) return -1; - - int needs_review = 0; - for (size_t sample = 0; sample < reads->count; ++sample) { - const count_stats *s = &stats_by_sample[sample]; - unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; - double denom = valid == 0 ? 1.0 : (double)valid; - if ((double)s->ambiguous / denom > 0.01 || (double)s->unmatched / denom > 0.10) needs_review = 1; - } - - fprintf(out, - "\nDotMatch Report" - "
      \n"); - fprintf(out, "

      DotMatch Report

      \n"); - fprintf(out, "

      Known-target assignment report for %zu target%s and %zu sample%s. Ambiguous reads are not silently counted.

      \n", - targets->count, targets->count == 1 ? "" : "s", reads->count, reads->count == 1 ? "" : "s"); - fprintf(out, "

      Run Status

      " - "
      Status
      %s
      " - "
      Targets
      %zu
      " - "
      Samples
      %zu
      " - "
      Target length
      %zu
      \n", - needs_review ? "warn" : "ok", needs_review ? "Needs Review" : "Ready", - targets->count, reads->count, target_len); - fprintf(out, "

      Inputs and Configuration

      k
      %d
      " - "
      Metric
      %s
      " - "
      Ambiguity policy
      %s
      " - "
      Assignment
      Known target
      \n", - k, metric_name(metric), ambiguity_policy_name(policy)); - - fprintf(out, "

      Target Assignment QC

      " - "" - "\n"); - for (size_t sample = 0; sample < reads->count; ++sample) { - const count_stats *s = &stats_by_sample[sample]; - unsigned long long covered = 0; - for (size_t t = 0; t < targets->count; ++t) { - unsigned long long total = 0; - for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * targets->count + t) * 5) + kind]; - if (total != 0) ++covered; - } - unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; - double denom = valid == 0 ? 1.0 : (double)valid; - fprintf(out, "" - "\n", - s->total, valid, 100.0 * (double)s->unique / denom, 100.0 * (double)s->exact / denom, - 100.0 * (double)s->corrected / denom, 100.0 * (double)s->ambiguous / denom, - 100.0 * (double)s->unmatched / denom, - targets->count == 0 ? 0.0 : 100.0 * (double)covered / (double)targets->count, - s->candidates_verified); - } - fprintf(out, "
      SampleTotal readsValid windowsAssignment rateExact rateRescue rateAmbiguous rateNo-match rateLibrary coverageCandidates verified
      "); - html_escape(out, labels->items[sample]); - fprintf(out, "%llu%llu%.2f%%%.2f%%%.2f%%%.2f%%%.2f%%%.2f%%%llu
      \n"); - - fprintf(out, "

      Warnings

        \n"); - if (!needs_review) { - fprintf(out, "
      • No high ambiguous or no-match warning thresholds were crossed.
        • \n"); - } - for (size_t sample = 0; sample < reads->count; ++sample) { - const count_stats *s = &stats_by_sample[sample]; - unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; - double denom = valid == 0 ? 1.0 : (double)valid; - if ((double)s->ambiguous / denom > 0.01) { - fprintf(out, "
      • Sample "); - html_escape(out, labels->items[sample]); - fprintf(out, " has ambiguous assignments above 1%% of valid extracted windows.
        • \n"); - } - if ((double)s->unmatched / denom > 0.10) { - fprintf(out, "
      • Sample "); - html_escape(out, labels->items[sample]); - fprintf(out, " has no-match reads above 10%% of valid extracted windows.
        • \n"); - } - } - fprintf(out, "
      • Ambiguous reads are not silently counted.
      \n"); - - fprintf(out, "

      Input Files

      \n"); - for (size_t sample = 0; sample < reads->count; ++sample) { - fprintf(out, "\n"); - } - fprintf(out, "
      SampleFASTQSelected start(s)
      "); - html_escape(out, labels->items[sample]); - fprintf(out, ""); - html_escape(out, reads->items[sample]); - fprintf(out, ""); - for (size_t i = 0; i < selected_offsets[sample].count; ++i) { - if (i != 0) fprintf(out, ", "); - fprintf(out, "%zu", selected_offsets[sample].items[i]); - } - fprintf(out, "
      \n"); - - if (audit_dir != NULL) { - char audit_path[4096]; - int n = snprintf(audit_path, sizeof(audit_path), "%s/%s", audit_dir, "audit_summary.tsv"); - if (n >= 0 && (size_t)n < sizeof(audit_path)) { - write_tsv_preview_table(out, "Library Audit", audit_path, 40); - } - } - if (unmatched_report_path != NULL) { - write_tsv_preview_table(out, "Top Unmatched", unmatched_report_path, 25); - } - - fprintf(out, "
      \n"); - fclose(out); - return 0; -} - -typedef struct count_dirty_slot { - size_t slot; - unsigned long long count; -} count_dirty_slot; - -typedef struct count_dirty_slots { - count_dirty_slot *items; - size_t count; - size_t cap; - size_t *table; - size_t table_cap; -} count_dirty_slots; - -typedef struct count_sample_job { - const qdaln_index *index; - const hamming_lookup *hlookup; - const levenshtein1_lookup *levlookup; - const seq_table *targets; - const char **target_ptrs; - const size_t *target_lens; - const char *reads_path; - const char *sample_label; - size_t sample_index; - offset_list *selected_offsets; - size_t target_len; - int k; - count_metric metric; - size_t indel_window; - unsigned long long *counts; - count_stats *stats; - FILE *assignments; - FILE *ambiguous_out; - FILE *unmatched_out; - const char *ambiguous_policy; - ambiguity_policy assignment_policy; - int direct_hamming_counts; - int metal_hamming_counts; - const uint64_t *metal_target_codes; - int fused_offset_detection; - size_t target_start; - size_t auto_offset; - size_t auto_offset_sample; - offset_mode offsets_mode; - double offset_min_fraction; - size_t read_threads; - int max_correction_qual; - int rc; - count_dirty_slots *dirty_slots; - count_progress *progress; -} count_sample_job; - -static void write_assignment_like_row(FILE *out, const seq_table *targets, const char *sample, const char *read_id, - const char *observed, qdaln_match_result r, const char *correction) { - const char *target_id = ""; - const char *target_seq = ""; - if (r.target_index >= 0) { - target_id = targets->records[r.target_index].id; - target_seq = targets->records[r.target_index].seq; - } - fprintf(out, "%s\t%s\t%s\t%d\t%s\t%s\t%d\t%d\t%d\t%s\t%s\n", - sample, read_id, observed, r.target_index, target_id, target_seq, r.best_distance, - r.second_best_distance, r.match_count, status_name(r.status), correction); -} - -static int find_observed_quality_window(const char *seq, size_t seq_len, const offset_list *offsets, - size_t fallback_offset, size_t target_len, count_metric metric, - size_t indel_window, int k, const char *observed, - size_t observed_len, size_t *start_out) { - size_t min_len = target_len; - size_t max_len = target_len; - if (metric == COUNT_METRIC_LEVENSHTEIN && indel_window != 0 && k == 1) { - min_len = target_len > indel_window ? target_len - indel_window : 0; - max_len = target_len + indel_window; - } - if (observed_len < min_len || observed_len > max_len) return 0; - - size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; - if (window_matches_observed_folded(seq, seq_len, offset, observed, observed_len)) { - *start_out = offset; - return 1; - } - } - return 0; -} - -static int quality_allows_unique_correction(const char *seq, size_t seq_len, const char *qual, - const offset_list *offsets, size_t fallback_offset, - size_t target_len, count_metric metric, size_t indel_window, - int k, const char *observed, const seq_record *target, - qdaln_match_result result, int max_correction_qual) { - if (max_correction_qual < 0 || qual == NULL) return 1; - if (result.status != QDALN_MATCH_UNIQUE || result.best_distance <= 0 || result.target_index < 0) return 1; - - size_t observed_len = strlen(observed); - size_t qual_start = 0; - if (!find_observed_quality_window(seq, seq_len, offsets, fallback_offset, target_len, metric, indel_window, - k, observed, observed_len, &qual_start)) { - return 0; - } - return observed_base_qualities_within_edit(observed, observed_len, target->seq, target->len, qual, - qual_start, max_correction_qual); -} - -static void html_escape(FILE *out, const char *s) { - for (; s != NULL && *s != '\0'; ++s) { - switch (*s) { - case '&': - fputs("&", out); - break; - case '<': - fputs("<", out); - break; - case '>': - fputs(">", out); - break; - case '"': - fputs(""", out); - break; - default: - fputc(*s, out); - break; - } - } -} - -static int build_target_arrays(const seq_table *targets, const char ***target_ptrs_out, size_t **target_lens_out) { - const char **target_ptrs = (const char **)malloc(targets->count * sizeof(char *)); - size_t *target_lens = (size_t *)malloc(targets->count * sizeof(size_t)); - if (targets->count != 0 && (target_ptrs == NULL || target_lens == NULL)) { - free(target_ptrs); - free(target_lens); - return -1; - } - for (size_t i = 0; i < targets->count; ++i) { - target_ptrs[i] = targets->records[i].seq; - target_lens[i] = targets->records[i].len; - } - *target_ptrs_out = target_ptrs; - *target_lens_out = target_lens; - return 0; -} - -static int all_targets_have_length(const seq_table *targets, size_t len) { - for (size_t i = 0; i < targets->count; ++i) { - if (targets->records[i].len != len) return 0; - } - return 1; -} - -static int cmp_size_asc(const void *a, const void *b) { - size_t aa = *(const size_t *)a; - size_t bb = *(const size_t *)b; - return aa > bb ? 1 : (aa < bb ? -1 : 0); -} - -static int collect_target_lengths(const seq_table *targets, size_t **lengths_out, size_t *count_out) { - size_t *lengths = (size_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(size_t)); - if (lengths == NULL) return -1; - size_t count = 0; - for (size_t i = 0; i < targets->count; ++i) { - size_t len = targets->records[i].len; - int seen = 0; - for (size_t j = 0; j < count; ++j) { - if (lengths[j] == len) { - seen = 1; - break; - } - } - if (!seen) lengths[count++] = len; - } - qsort(lengths, count, sizeof(size_t), cmp_size_asc); - *lengths_out = lengths; - *count_out = count; - return 0; -} - -typedef struct match_merge_hit { - int target_index; - int distance; -} match_merge_hit; - -typedef struct match_merge_state { - match_merge_hit inline_hits[16]; - match_merge_hit *hits; - size_t count; - size_t cap; - int next_synthetic_target; - int saw_none; -} match_merge_state; - -static void merge_state_init(match_merge_state *state) { - state->hits = state->inline_hits; - state->count = 0; - state->cap = sizeof(state->inline_hits) / sizeof(state->inline_hits[0]); - state->next_synthetic_target = -2; - state->saw_none = 0; -} - -static void merge_state_free(match_merge_state *state) { - if (state->hits != state->inline_hits) free(state->hits); - merge_state_init(state); -} - -static void copy_merge_observed(char *dst, size_t dst_cap, const char *src) { - if (dst_cap == 0) return; - size_t n = 0; - while (n + 1 < dst_cap && src[n] != '\0') ++n; - memcpy(dst, src, n); - dst[n] = '\0'; -} - -static int merge_state_grow(match_merge_state *state) { - size_t next_cap = state->cap * 2; - match_merge_hit *next = (match_merge_hit *)malloc(next_cap * sizeof(match_merge_hit)); - if (next == NULL) return -1; - memcpy(next, state->hits, state->count * sizeof(match_merge_hit)); - if (state->hits != state->inline_hits) free(state->hits); - state->hits = next; - state->cap = next_cap; - return 0; -} - -static int merge_state_add_hit(match_merge_state *state, int target_index, int distance) { - if (target_index >= 0) { - for (size_t i = 0; i < state->count; ++i) { - if (state->hits[i].target_index == target_index) { - if (distance < state->hits[i].distance) state->hits[i].distance = distance; - return 0; - } - } - } - if (state->count == state->cap && merge_state_grow(state) != 0) return -1; - state->hits[state->count].target_index = target_index; - state->hits[state->count].distance = distance; - ++state->count; - return 0; -} - -static int merge_state_best_distance(const match_merge_state *state) { - int best = -1; - for (size_t i = 0; i < state->count; ++i) { - int d = state->hits[i].distance; - if (best < 0 || d < best) best = d; - } - return best; -} - -static int merge_state_add_result(match_merge_state *state, char *best_observed, size_t best_observed_cap, - const char *observed, qdaln_match_result r) { - if (r.status == QDALN_MATCH_INVALID) return 0; - if (r.match_count == 0) { - if (state->count == 0 && !state->saw_none && best_observed_cap != 0) { - copy_merge_observed(best_observed, best_observed_cap, observed); - } - state->saw_none = 1; - return 0; - } - - int prior_best = merge_state_best_distance(state); - if ((prior_best < 0 || r.best_distance < prior_best) && best_observed_cap != 0) { - copy_merge_observed(best_observed, best_observed_cap, observed); - } - - if (r.target_index >= 0 && merge_state_add_hit(state, r.target_index, r.best_distance) != 0) return -1; - int remaining = r.match_count - (r.target_index >= 0 ? 1 : 0); - if (remaining <= 0) return 0; - - int synthetic_distance = r.best_distance; - if (r.status != QDALN_MATCH_AMBIGUOUS && r.second_best_distance >= 0) { - synthetic_distance = r.second_best_distance; - } - for (int i = 0; i < remaining; ++i) { - if (merge_state_add_hit(state, state->next_synthetic_target--, synthetic_distance) != 0) return -1; - } - return 0; -} - -static void merge_state_finish(const match_merge_state *state, qdaln_match_result *result) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (state->count == 0) { - result->status = state->saw_none ? QDALN_MATCH_NONE : QDALN_MATCH_INVALID; - return; - } - - int best_ties = 0; - for (size_t i = 0; i < state->count; ++i) { - int d = state->hits[i].distance; - ++result->match_count; - if (result->best_distance < 0 || d < result->best_distance) { - result->second_best_distance = result->best_distance; - result->best_distance = d; - result->target_index = state->hits[i].target_index >= 0 ? state->hits[i].target_index : -1; - best_ties = 1; - } else if (d == result->best_distance) { - if (state->hits[i].target_index >= 0 && - (result->target_index < 0 || state->hits[i].target_index < result->target_index)) { - result->target_index = state->hits[i].target_index; - } - ++best_ties; - } else if (result->second_best_distance < 0 || d < result->second_best_distance) { - result->second_best_distance = d; - } - } - result->status = best_ties > 1 ? QDALN_MATCH_AMBIGUOUS : QDALN_MATCH_UNIQUE; -} - -static void merge_summary_result(qdaln_match_result *best, char *best_observed, size_t best_observed_cap, - const char *observed, qdaln_match_result r) { - if (r.status == QDALN_MATCH_INVALID) return; - if (r.match_count == 0) { - if (best->status == QDALN_MATCH_INVALID) { - *best = r; - copy_merge_observed(best_observed, best_observed_cap, observed); - } - return; - } - if (best->match_count == 0 || best->best_distance < 0 || r.best_distance < best->best_distance) { - *best = r; - copy_merge_observed(best_observed, best_observed_cap, observed); - return; - } - if (r.best_distance == best->best_distance) { - if (r.target_index != best->target_index || r.status == QDALN_MATCH_AMBIGUOUS || - best->status == QDALN_MATCH_AMBIGUOUS) { - if (r.target_index >= 0 && (best->target_index < 0 || r.target_index < best->target_index)) { - best->target_index = r.target_index; - } - best->status = QDALN_MATCH_AMBIGUOUS; - best->match_count += r.match_count; - } - } else if (best->second_best_distance < 0 || r.best_distance < best->second_best_distance) { - best->second_best_distance = r.best_distance; - best->match_count += r.match_count; - } -} - -static int assign_count_window(const qdaln_index *index, const char *seq, size_t seq_len, size_t target_start, - size_t target_len, int k, count_metric metric, size_t indel_window, - qdaln_match_result *result, qdaln_index_stats *stats, char *observed, - size_t observed_cap, int best_exact_shortcut) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (stats != NULL) { - stats->candidates_considered = 0; - stats->candidates_verified = 0; - } - if (observed_cap != 0) observed[0] = '\0'; - - match_merge_state merge; - merge_state_init(&merge); - int rc = 0; - size_t min_len = target_len; - size_t max_len = target_len; - if (metric == COUNT_METRIC_LEVENSHTEIN && indel_window != 0 && k == 1) { - min_len = target_len > indel_window ? target_len - indel_window : 0; - max_len = target_len + indel_window; - } - - for (size_t len = min_len; len <= max_len; ++len) { - if (len >= observed_cap) continue; - if (target_start > seq_len || len > seq_len - target_start) continue; - - if (metric == COUNT_METRIC_HAMMING && k == 0) { - qdaln_match_result r; - qdaln_index_stats s = {0, 0}; - int exact_rc = qdaln_index_lookup_exact_ascii_stats(index, seq + target_start, len, &r, &s); - if (exact_rc != 0) { - rc = -1; - goto done; - } - if (stats != NULL) { - stats->candidates_considered += s.candidates_considered; - stats->candidates_verified += s.candidates_verified; - } - char candidate[8192]; - if (len >= sizeof(candidate)) continue; - memcpy(candidate, seq + target_start, len); - candidate[len] = '\0'; - uppercase_ascii(candidate); - if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { - rc = -1; - goto done; - } - continue; - } - - char candidate[8192]; - if (len >= sizeof(candidate)) continue; - memcpy(candidate, seq + target_start, len); - candidate[len] = '\0'; - uppercase_ascii(candidate); - - const char *read_ptr = candidate; - size_t read_len = len; - qdaln_match_result r; - qdaln_index_stats s = {0, 0}; - if (best_exact_shortcut && metric == COUNT_METRIC_HAMMING && k == 1) { - int exact_rc = qdaln_index_lookup_exact_stats(index, read_ptr, read_len, &r, &s); - if (exact_rc != 0) { - rc = -1; - goto done; - } - if (stats != NULL) { - stats->candidates_considered += s.candidates_considered; - stats->candidates_verified += s.candidates_verified; - } - if (r.status == QDALN_MATCH_UNIQUE || r.status == QDALN_MATCH_AMBIGUOUS || r.status == QDALN_MATCH_INVALID) { - if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { - rc = -1; - goto done; - } - continue; - } - s.candidates_considered = 0; - s.candidates_verified = 0; - } - int assign_rc = metric == COUNT_METRIC_HAMMING - ? qdaln_index_assign_hamming_stats(index, &read_ptr, &read_len, 1, k, &r, &s) - : best_exact_shortcut - ? qdaln_index_assign_status_stats(index, &read_ptr, &read_len, 1, k, &r, &s) - : qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &r, &s); - if (assign_rc != 0) { - rc = -1; - goto done; - } - if (stats != NULL) { - stats->candidates_considered += s.candidates_considered; - stats->candidates_verified += s.candidates_verified; - } - if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { - rc = -1; - goto done; - } - } - -done: - merge_state_finish(&merge, result); - merge_state_free(&merge); - return rc; -} - -static int assign_count_length_set(const qdaln_index *index, const char *seq, size_t seq_len, size_t target_start, - const size_t *lengths, size_t n_lengths, int k, count_metric metric, - size_t indel_window, qdaln_match_result *result, qdaln_index_stats *stats, - char *observed, size_t observed_cap, int best_exact_shortcut) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (stats != NULL) { - stats->candidates_considered = 0; - stats->candidates_verified = 0; - } - if (observed_cap != 0) observed[0] = '\0'; - for (size_t i = 0; i < n_lengths; ++i) { - qdaln_match_result r = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_index_stats s = {0, 0}; - char candidate[8192]; - if (assign_count_window(index, seq, seq_len, target_start, lengths[i], k, metric, indel_window, - &r, &s, candidate, sizeof(candidate), best_exact_shortcut) != 0) { - return -1; - } - if (stats != NULL) { - stats->candidates_considered += s.candidates_considered; - stats->candidates_verified += s.candidates_verified; - } - merge_summary_result(result, observed, observed_cap, candidate, r); - } - return 0; -} - -static int assign_count_offsets(const qdaln_index *index, const char *seq, size_t seq_len, - const offset_list *offsets, size_t fallback_offset, size_t target_len, - int k, count_metric metric, size_t indel_window, - qdaln_match_result *result, qdaln_index_stats *stats, - char *observed, size_t observed_cap, int best_exact_shortcut) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (stats != NULL) { - stats->candidates_considered = 0; - stats->candidates_verified = 0; - } - if (observed_cap != 0) observed[0] = '\0'; - - size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; - if (best_exact_shortcut && k == 1 && target_len < observed_cap) { - match_merge_state exact_merge; - merge_state_init(&exact_merge); - qdaln_match_result exact_result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_index_stats exact_stats_total = {0, 0}; - char exact_observed[8192]; - int exact_rc_total = 0; - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; - if (offset > seq_len || target_len > seq_len - offset || target_len >= sizeof(exact_observed)) continue; - memcpy(exact_observed, seq + offset, target_len); - exact_observed[target_len] = '\0'; - uppercase_ascii(exact_observed); - const char *read_ptr = exact_observed; - size_t read_len = target_len; - qdaln_match_result exact_one; - qdaln_index_stats exact_stats = {0, 0}; - if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, 0, &exact_one, &exact_stats) != 0) { - merge_state_free(&exact_merge); - return -1; - } - exact_stats_total.candidates_considered += exact_stats.candidates_considered; - exact_stats_total.candidates_verified += exact_stats.candidates_verified; - if (merge_state_add_result(&exact_merge, observed, observed_cap, exact_observed, exact_one) != 0) { - exact_rc_total = -1; - break; - } - } - merge_state_finish(&exact_merge, &exact_result); - merge_state_free(&exact_merge); - if (exact_rc_total != 0) return -1; - if (stats != NULL) { - stats->candidates_considered += exact_stats_total.candidates_considered; - stats->candidates_verified += exact_stats_total.candidates_verified; - } - if (exact_result.match_count > 0) { - *result = exact_result; - return 0; - } - } - match_merge_state merge; - merge_state_init(&merge); - int rc = 0; - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; - qdaln_match_result candidate = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_index_stats local_stats = {0, 0}; - char local_observed[8192]; - if (assign_count_window(index, seq, seq_len, offset, target_len, k, metric, indel_window, - &candidate, &local_stats, local_observed, sizeof(local_observed), - best_exact_shortcut) != 0) { - rc = -1; - break; - } - if (stats != NULL) { - stats->candidates_considered += local_stats.candidates_considered; - stats->candidates_verified += local_stats.candidates_verified; - } - if (merge_state_add_result(&merge, observed, observed_cap, local_observed, candidate) != 0) { - rc = -1; - break; - } - } - - merge_state_finish(&merge, result); - merge_state_free(&merge); - return rc; -} - -static int hamming_distance_within_k_cli(const char *a, size_t a_len, const char *b, size_t b_len, int k) { - if (a_len != b_len) return -1; - int d = 0; - for (size_t i = 0; i < a_len; ++i) { - if (a[i] != b[i] && ++d > k) return -1; - } - return d; -} - -static int hamming_distance_cli(const char *a, size_t a_len, const char *b, size_t b_len) { - if (a_len != b_len) return -1; - int d = 0; - for (size_t i = 0; i < a_len; ++i) { - if (a[i] != b[i]) ++d; - } - return d; -} - -static int scan_assign_metric(const char *read, size_t read_len, const char *const *targets, - const size_t *target_lens, size_t n_targets, int k, - count_metric metric, qdaln_match_result *result) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; - int best_ties = 0; - for (size_t i = 0; i < n_targets; ++i) { - int d = metric == COUNT_METRIC_HAMMING - ? hamming_distance_within_k_cli(read, read_len, targets[i], target_lens[i], k) - : qdaln_edit_distance_leq(read, read_len, targets[i], target_lens[i], k) > 0 - ? qdaln_edit_distance(read, read_len, targets[i], target_lens[i]) - : -1; - if (d < 0 || d > k) continue; - ++result->match_count; - if (result->best_distance < 0 || d < result->best_distance) { - result->second_best_distance = result->best_distance; - result->best_distance = d; - result->target_index = (int)i; - best_ties = 1; - } else if (d == result->best_distance) { - if (result->target_index < 0 || (int)i < result->target_index) result->target_index = (int)i; - ++best_ties; - } else if (result->second_best_distance < 0 || d < result->second_best_distance) { - result->second_best_distance = d; - } - } - if (result->match_count == 0) { - result->status = QDALN_MATCH_NONE; - } else if (best_ties > 1) { - result->status = QDALN_MATCH_AMBIGUOUS; - } else { - result->status = QDALN_MATCH_UNIQUE; - } - return 0; -} - -static int scan_count_window(const char *const *targets, const size_t *target_lens, size_t n_targets, - const char *seq, size_t seq_len, size_t target_start, size_t target_len, - int k, count_metric metric, size_t indel_window, - qdaln_match_result *result, char *observed, size_t observed_cap) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (observed_cap != 0) observed[0] = '\0'; - - match_merge_state merge; - merge_state_init(&merge); - int rc = 0; - size_t min_len = target_len; - size_t max_len = target_len; - if (metric == COUNT_METRIC_LEVENSHTEIN && indel_window != 0 && k == 1) { - min_len = target_len > indel_window ? target_len - indel_window : 0; - max_len = target_len + indel_window; - } - - for (size_t len = min_len; len <= max_len; ++len) { - if (len >= observed_cap) continue; - if (target_start > seq_len || len > seq_len - target_start) continue; - char candidate[8192]; - if (len >= sizeof(candidate)) continue; - memcpy(candidate, seq + target_start, len); - candidate[len] = '\0'; - uppercase_ascii(candidate); - qdaln_match_result r; - if (scan_assign_metric(candidate, len, targets, target_lens, n_targets, k, metric, &r) != 0) { - rc = -1; - break; - } - if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { - rc = -1; - break; - } - } - merge_state_finish(&merge, result); - merge_state_free(&merge); - return rc; -} - -static int scan_count_offsets(const char *const *targets, const size_t *target_lens, size_t n_targets, - const char *seq, size_t seq_len, const offset_list *offsets, - size_t fallback_offset, size_t target_len, int k, count_metric metric, - size_t indel_window, qdaln_match_result *result, - char *observed, size_t observed_cap) { - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (observed_cap != 0) observed[0] = '\0'; - size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; - match_merge_state merge; - merge_state_init(&merge); - int rc = 0; - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; - qdaln_match_result candidate = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - char local_observed[8192]; - if (scan_count_window(targets, target_lens, n_targets, seq, seq_len, offset, target_len, k, metric, - indel_window, &candidate, local_observed, sizeof(local_observed)) != 0) { - rc = -1; - break; - } - if (merge_state_add_result(&merge, observed, observed_cap, local_observed, candidate) != 0) { - rc = -1; - break; - } - } - merge_state_finish(&merge, result); - merge_state_free(&merge); - return rc; -} - -static void hamming_lookup_result_from_entry(const hamming_lookup_entry *entry, int distance, - qdaln_match_result *result) { - result->target_index = entry->target_index; - result->best_distance = distance; - result->second_best_distance = -1; - result->match_count = entry->match_count; - result->status = entry->match_count > 1 ? QDALN_MATCH_AMBIGUOUS : QDALN_MATCH_UNIQUE; -} - -static void levenshtein1_lookup_result_from_entry(const hamming_lookup_entry *entry, int distance, - qdaln_match_result *result) { - result->target_index = entry->target_index; - result->best_distance = distance; - result->second_best_distance = -1; - result->match_count = entry->match_count; - result->status = entry->match_count > 1 ? QDALN_MATCH_AMBIGUOUS : QDALN_MATCH_UNIQUE; -} - -static int assign_levenshtein1_lookup_offset(const levenshtein1_lookup *lookup, const char *seq, size_t seq_len, - size_t offset, qdaln_match_result *result, qdaln_index_stats *stats, - char *observed, size_t observed_cap) { - if (lookup == NULL || !lookup->ready || lookup->target_len == 0 || lookup->target_len > 31) return 0; - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (stats != NULL) { - stats->candidates_considered = 0; - stats->candidates_verified = 0; - } - if (observed_cap != 0) observed[0] = '\0'; - - size_t target_len = lookup->target_len; - if (offset > seq_len || target_len - 1 > seq_len - offset) return 1; - - uint64_t code = 0; - const hamming_lookup_entry *entry = NULL; - qdaln_match_result local = {-1, -1, -1, 0, QDALN_MATCH_NONE}; - int have_len_l = 0; - uint64_t code_len_l = 0; - if (target_len < observed_cap && target_len <= seq_len - offset && - dna2_code_local_fold(seq + offset, target_len, &code_len_l)) { - have_len_l = 1; - code = code_len_l; - entry = levenshtein1_lookup_find(lookup->exact, lookup->exact_cap, code, target_len); - if (entry != NULL) { - levenshtein1_lookup_result_from_entry(entry, 0, &local); - if (stats != NULL) { - stats->candidates_considered += (size_t)entry->match_count; - stats->candidates_verified += (size_t)entry->match_count; - } - copy_upper_ascii_window(observed, observed_cap, seq + offset, target_len); - *result = local; - return 1; - } - } - - match_merge_state merge; - merge_state_init(&merge); - int rc = 0; - char candidate_observed[128]; - - if (have_len_l) { - entry = levenshtein1_lookup_find(lookup->substitution, lookup->substitution_cap, code_len_l, target_len); - if (entry != NULL) { - levenshtein1_lookup_result_from_entry(entry, 1, &local); - if (stats != NULL) { - stats->candidates_considered += (size_t)entry->match_count; - stats->candidates_verified += (size_t)entry->match_count; - } - copy_upper_ascii_window(candidate_observed, sizeof(candidate_observed), seq + offset, target_len); - rc = merge_state_add_result(&merge, observed, observed_cap, candidate_observed, local); - } - } - - if (rc == 0 && target_len - 1 < observed_cap && dna2_code_local_fold(seq + offset, target_len - 1, &code)) { - entry = levenshtein1_lookup_find(lookup->target_deletion, lookup->target_deletion_cap, code, target_len - 1); - if (entry != NULL) { - levenshtein1_lookup_result_from_entry(entry, 1, &local); - if (stats != NULL) { - stats->candidates_considered += (size_t)entry->match_count; - stats->candidates_verified += (size_t)entry->match_count; - } - copy_upper_ascii_window(candidate_observed, sizeof(candidate_observed), seq + offset, target_len - 1); - rc = merge_state_add_result(&merge, observed, observed_cap, candidate_observed, local); - } - } - - if (rc == 0 && target_len + 1 < observed_cap && target_len + 1 <= seq_len - offset && - dna2_code_local_fold(seq + offset, target_len + 1, &code)) { - entry = levenshtein1_lookup_find(lookup->target_insertion, lookup->target_insertion_cap, code, target_len + 1); - if (entry != NULL) { - levenshtein1_lookup_result_from_entry(entry, 1, &local); - if (stats != NULL) { - stats->candidates_considered += (size_t)entry->match_count; - stats->candidates_verified += (size_t)entry->match_count; - } - copy_upper_ascii_window(candidate_observed, sizeof(candidate_observed), seq + offset, target_len + 1); - rc = merge_state_add_result(&merge, observed, observed_cap, candidate_observed, local); - } - } - - if (rc != 0) { - merge_state_free(&merge); - return -1; - } - merge_state_finish(&merge, result); - merge_state_free(&merge); - if (result->match_count > 0) { - return 1; - } - - if (target_len <= seq_len - offset && target_len < observed_cap) { - copy_upper_ascii_window(observed, observed_cap, seq + offset, target_len); - } else if (target_len - 1 < observed_cap) { - copy_upper_ascii_window(observed, observed_cap, seq + offset, target_len - 1); - } - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; - return 1; -} - -static int hamming_lookup_counts_eligible(int count_only, int max_correction_qual, count_metric metric, - size_t indel_window, int k, size_t target_len, - hamming_index_strategy hamming_strategy) { - if (!count_only || max_correction_qual >= 0) return 0; - if (metric != COUNT_METRIC_HAMMING || indel_window != 0) return 0; - if (k != 0 && k != 1) return 0; - if (target_len > 32) return 0; - return hamming_strategy == HAMMING_INDEX_PRECOMPUTE || hamming_strategy == HAMMING_INDEX_AUTO; -} - -static int levenshtein1_lookup_counts_eligible(int count_only, int max_correction_qual, count_metric metric, - size_t indel_window, int k, size_t target_len, - size_t max_selected_offsets, FILE *assignments, - FILE *ambiguous_out, FILE *unmatched_out, - ambiguity_policy assignment_policy) { - if (!count_only || max_correction_qual >= 0) return 0; - if (assignments != NULL || ambiguous_out != NULL || unmatched_out != NULL) return 0; - if (assignment_policy != AMBIGUITY_POLICY_BEST) return 0; - if (metric != COUNT_METRIC_LEVENSHTEIN || indel_window != 1 || k != 1) return 0; - if (max_selected_offsets > 1 || target_len == 0 || target_len > 31) return 0; - return 1; -} - -static int hamming_direct_worker_eligible(int lookup_eligible, ambiguity_policy assignment_policy, int k) { - if (!lookup_eligible) return 0; - if (k == 0) return 1; - return assignment_policy == AMBIGUITY_POLICY_BEST || assignment_policy == AMBIGUITY_POLICY_RADIUS; -} - -static const char *count_backend_mode_name(count_backend_mode mode) { - switch (mode) { - case COUNT_BACKEND_CPU: - return "cpu"; - case COUNT_BACKEND_METAL: - return "gpu-metal-experimental"; - case COUNT_BACKEND_AUTO: - default: - return "auto"; - } -} - -static int parse_count_backend_mode(const char *value, count_backend_mode *mode_out) { - if (strcmp(value, "auto") == 0) { - *mode_out = COUNT_BACKEND_AUTO; - return 0; - } - if (strcmp(value, "cpu") == 0) { - *mode_out = COUNT_BACKEND_CPU; - return 0; - } - if (strcmp(value, "gpu-metal-experimental") == 0 || strcmp(value, "metal") == 0) { - *mode_out = COUNT_BACKEND_METAL; - return 0; - } - return -1; -} - -static int metal_hamming_count_eligible(count_backend_mode backend, int hamming_lookup_eligible, - ambiguity_policy assignment_policy, int k, size_t max_selected_offsets, - int fused_offset_detection) { - if (backend != COUNT_BACKEND_METAL) return 0; - if (!hamming_lookup_eligible || !qdmetal_available()) return 0; - if (fused_offset_detection) return 0; - if (max_selected_offsets > 1) return 0; - if (k == 1 && assignment_policy != AMBIGUITY_POLICY_BEST) return 0; - return 1; -} - -static int build_packed_target_codes(const seq_table *targets, size_t target_len, uint64_t **codes_out) { - if (targets == NULL || codes_out == NULL || target_len == 0 || target_len > 32) return 0; - uint64_t *codes = (uint64_t *)calloc(targets->count == 0 ? 1 : targets->count, sizeof(uint64_t)); - if (codes == NULL) return -1; - for (size_t i = 0; i < targets->count; ++i) { - if (targets->records[i].len != target_len || !dna2_code_local(targets->records[i].seq, target_len, &codes[i])) { - free(codes); - return 0; - } - } - *codes_out = codes; - return 1; -} - -static const char *metal_count_engine_name(size_t n_targets) { - return n_targets >= 1024 ? "hamming_metal_seed_index" : "hamming_metal_brute_force"; -} - -static int direct_hamming_merge_lookup_entry(match_merge_state *merge, const hamming_lookup_entry *entry, - int distance) { - if (entry == NULL) return 0; - qdaln_match_result r; - hamming_lookup_result_from_entry(entry, distance, &r); - return merge_state_add_result(merge, NULL, 0, "", r); -} - -static int direct_hamming_collect_seed_hits(count_sample_job *job, match_merge_state *merge, unsigned char seed_id, - uint64_t seed_code, uint64_t read_code) { - const hamming_lookup *lookup = job->hlookup; - size_t seed_len = seed_id == 0 ? lookup->seed0_len : lookup->target_len - lookup->seed0_len; - size_t slot = seed_hash_local(seed_code, seed_len, seed_id, lookup->seed_hash_cap); - for (int e = lookup->seed_heads[slot]; e >= 0; e = lookup->seeds[e].next) { - const hamming_seed_entry *entry = &lookup->seeds[e]; - if (entry->seed_id != seed_id || entry->code != seed_code || entry->target_index < 0) continue; - job->stats->candidates_considered += 1; - job->stats->candidates_verified += 1; - if (hamming_code_distance_local(read_code, lookup->target_codes[entry->target_index], lookup->target_len) > 1) { - continue; - } - qdaln_match_result r = {entry->target_index, 1, -1, 1, QDALN_MATCH_UNIQUE}; - if (merge_state_add_result(merge, NULL, 0, "", r) != 0) return -1; - } - return 0; -} - -static int assign_hamming_lookup_offsets(const hamming_lookup *lookup, const char *seq, size_t seq_len, - const offset_list *offsets, size_t fallback_offset, int k, - qdaln_match_result *result, qdaln_index_stats *stats, - char *observed, size_t observed_cap, int exact_merge) { - if (lookup == NULL || !lookup->ready || lookup->target_len >= observed_cap || (k != 0 && k != 1)) return 0; - *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (stats != NULL) { - stats->candidates_considered = 0; - stats->candidates_verified = 0; - } - if (observed_cap != 0) observed[0] = '\0'; - - size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; - match_merge_state merge; - merge_state_init(&merge); - qdaln_match_result fast_result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - int fast_saw_window = 0; - int rc = 1; - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; - if (offset > seq_len || lookup->target_len > seq_len - offset) continue; - if (!exact_merge) fast_saw_window = 1; - char candidate[8192]; - uint64_t code = 0; - const char *window = seq + offset; - if (!dna2_code_local_fold(window, lookup->target_len, &code)) { - rc = 0; - break; - } - - const hamming_lookup_entry *entry = hamming_lookup_find(lookup->exact, lookup->exact_cap, code); - qdaln_match_result r = {-1, -1, -1, 0, QDALN_MATCH_NONE}; - if (entry != NULL) { - hamming_lookup_result_from_entry(entry, 0, &r); - if (stats != NULL) { - stats->candidates_considered += (size_t)entry->match_count; - stats->candidates_verified += (size_t)entry->match_count; - } - copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); - if (exact_merge ? merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0 - : (merge_summary_result(&fast_result, observed, observed_cap, candidate, r), 0)) { - rc = -1; - break; - } - if (!(exact_merge && k == 1)) continue; - } - if (k == 1) { - if (!exact_merge && fast_result.best_distance == 0) continue; - entry = hamming_lookup_find(lookup->mismatch, lookup->mismatch_cap, code); - if (entry != NULL) { - hamming_lookup_result_from_entry(entry, 1, &r); - if (stats != NULL) { - stats->candidates_considered += (size_t)entry->match_count; - stats->candidates_verified += (size_t)entry->match_count; - } - copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); - if (exact_merge ? merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0 - : (merge_summary_result(&fast_result, observed, observed_cap, candidate, r), 0)) { - rc = -1; - break; - } - } else { - if (!exact_merge) continue; - copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); - if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { - rc = -1; - break; - } - } - } else { - if (!exact_merge) continue; - copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); - if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { - rc = -1; - break; - } - } - } - - if (exact_merge) merge_state_finish(&merge, result); - else { - if (fast_result.status == QDALN_MATCH_INVALID && fast_saw_window) { - fast_result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; - } - *result = fast_result; - } - merge_state_free(&merge); - return rc; -} - -typedef struct seq_buffer { - char **items; - size_t *lens; - size_t count; - size_t cap; - char *fixed_items; - size_t fixed_len; - size_t fixed_cap; - int fixed_active; -} seq_buffer; - -static int seq_buffer_ptr_in_fixed(const seq_buffer *buffer, const char *ptr) { - if (buffer->fixed_items == NULL || buffer->fixed_cap == 0) return 0; - uintptr_t p = (uintptr_t)ptr; - uintptr_t start = (uintptr_t)buffer->fixed_items; - uintptr_t end = start + buffer->fixed_cap * (buffer->fixed_len + 1); - return p >= start && p < end; -} - -static void free_seq_buffer(seq_buffer *buffer) { - if (buffer == NULL) return; - for (size_t i = 0; i < buffer->count; ++i) { - if (!seq_buffer_ptr_in_fixed(buffer, buffer->items[i])) free(buffer->items[i]); - } - free(buffer->fixed_items); - free(buffer->items); - free(buffer->lens); - buffer->items = NULL; - buffer->lens = NULL; - buffer->fixed_items = NULL; - buffer->count = 0; - buffer->cap = 0; - buffer->fixed_len = 0; - buffer->fixed_cap = 0; - buffer->fixed_active = 0; -} - -/* reset_seq_buffer reclaims per-sequence mallocs (for variable-length batches) but - * retains the items/lens arrays and fixed block (when active for uniform lengths) - * so subsequent batches reuse the same allocations without per-batch malloc/free. - * This is used in the high-throughput batched reader paths for large FASTQ. - */ -static void reset_seq_buffer(seq_buffer *buffer) { - if (buffer == NULL) return; - for (size_t i = 0; i < buffer->count; ++i) { - if (!seq_buffer_ptr_in_fixed(buffer, buffer->items[i])) free(buffer->items[i]); - } - if (!buffer->fixed_active) { - free(buffer->fixed_items); - buffer->fixed_items = NULL; - buffer->fixed_len = 0; - buffer->fixed_cap = 0; - } - /* keep items/lens (and fixed block + active if still set) and cap for reuse */ - buffer->count = 0; -} - -static int grow_seq_buffer(seq_buffer *buffer) { - size_t old_cap = buffer->cap; - size_t next_cap = buffer->cap == 0 ? 1024 : buffer->cap * 2; - char **next_items = (char **)realloc(buffer->items, next_cap * sizeof(char *)); - if (next_items == NULL) return -1; - buffer->items = next_items; - size_t *next_lens = (size_t *)realloc(buffer->lens, next_cap * sizeof(size_t)); - if (next_lens == NULL) return -1; - buffer->lens = next_lens; - buffer->cap = next_cap; - if (buffer->fixed_active) { - char *next_fixed = (char *)realloc(buffer->fixed_items, next_cap * (buffer->fixed_len + 1)); - if (next_fixed == NULL) { - buffer->cap = old_cap; - return -1; - } - buffer->fixed_items = next_fixed; - buffer->fixed_cap = next_cap; - for (size_t i = 0; i < buffer->count; ++i) { - buffer->items[i] = buffer->fixed_items + i * (buffer->fixed_len + 1); - } - } - return 0; -} - -static int reserve_seq_buffer(seq_buffer *buffer, size_t requested_cap) { - if (requested_cap <= buffer->cap) return 0; - char **next_items = (char **)realloc(buffer->items, requested_cap * sizeof(char *)); - if (next_items == NULL) return -1; - buffer->items = next_items; - size_t *next_lens = (size_t *)realloc(buffer->lens, requested_cap * sizeof(size_t)); - if (next_lens == NULL) return -1; - buffer->lens = next_lens; - buffer->cap = requested_cap; - if (buffer->fixed_active) { - char *next_fixed = (char *)realloc(buffer->fixed_items, requested_cap * (buffer->fixed_len + 1)); - if (next_fixed == NULL) return -1; - buffer->fixed_items = next_fixed; - buffer->fixed_cap = requested_cap; - for (size_t i = 0; i < buffer->count; ++i) { - buffer->items[i] = buffer->fixed_items + i * (buffer->fixed_len + 1); - } - } - return 0; -} - -static int push_seq_buffer(seq_buffer *buffer, const char *seq, size_t len) { - if (buffer->count == buffer->cap && grow_seq_buffer(buffer) != 0) return -1; - if (buffer->count == 0 && buffer->fixed_items == NULL && len <= 8191) { - buffer->fixed_items = (char *)malloc(buffer->cap * (len + 1)); - if (buffer->fixed_items != NULL) { - buffer->fixed_len = len; - buffer->fixed_cap = buffer->cap; - buffer->fixed_active = 1; - } - } - if (buffer->fixed_active && len == buffer->fixed_len) { - char *dst = buffer->fixed_items + buffer->count * (buffer->fixed_len + 1); - memcpy(dst, seq, len); - dst[len] = '\0'; - buffer->items[buffer->count] = dst; - } else { - if (buffer->fixed_active && len != buffer->fixed_len) buffer->fixed_active = 0; - buffer->items[buffer->count] = xstrndup(seq, len); - if (buffer->items[buffer->count] == NULL) return -1; - } - buffer->lens[buffer->count] = len; - ++buffer->count; - return 0; -} - -static void direct_hamming_record_hit(int target_index, int match_count, int *best_target, int *ambiguous) { - if (match_count > 1) *ambiguous = 1; - if (*best_target < 0) { - *best_target = target_index; - } else if (target_index != *best_target) { - if (target_index >= 0 && target_index < *best_target) *best_target = target_index; - *ambiguous = 1; - } -} - -static void merge_count_stats(count_stats *dst, const count_stats *src) { - dst->total += src->total; - dst->unique += src->unique; - dst->exact += src->exact; - dst->corrected += src->corrected; - dst->ambiguous += src->ambiguous; - dst->unmatched += src->unmatched; - dst->invalid += src->invalid; - dst->candidates_considered += src->candidates_considered; - dst->candidates_verified += src->candidates_verified; -} - -static void free_count_dirty_slots(count_dirty_slots *dirty) { - if (dirty == NULL) return; - free(dirty->items); - free(dirty->table); - dirty->items = NULL; - dirty->table = NULL; - dirty->count = 0; - dirty->cap = 0; - dirty->table_cap = 0; -} - -static size_t count_dirty_slot_hash(size_t slot) { - uint64_t x = (uint64_t)slot; - x ^= x >> 30; - x *= UINT64_C(0xbf58476d1ce4e5b9); - x ^= x >> 27; - x *= UINT64_C(0x94d049bb133111eb); - x ^= x >> 31; - return (size_t)x; -} - -static int count_dirty_slots_rehash(count_dirty_slots *dirty, size_t min_cap) { - size_t cap = 32; - while (cap < min_cap) cap *= 2; - size_t *table = (size_t *)calloc(cap, sizeof(size_t)); - if (table == NULL) return -1; - - for (size_t i = 0; i < dirty->count; ++i) { - size_t mask = cap - 1; - size_t pos = count_dirty_slot_hash(dirty->items[i].slot) & mask; - while (table[pos] != 0) pos = (pos + 1) & mask; - table[pos] = i + 1; - } - - free(dirty->table); - dirty->table = table; - dirty->table_cap = cap; - return 0; -} - -static int mark_count_dirty_slot(count_dirty_slots *dirty, size_t slot) { - if (dirty == NULL) return 0; - - if (dirty->table_cap == 0 || ((dirty->count + 1) * 2) > dirty->table_cap) { - if (count_dirty_slots_rehash(dirty, (dirty->count + 1) * 4) != 0) return -1; - } - - size_t mask = dirty->table_cap - 1; - size_t pos = count_dirty_slot_hash(slot) & mask; - while (dirty->table[pos] != 0) { - count_dirty_slot *item = &dirty->items[dirty->table[pos] - 1]; - if (item->slot == slot) { - ++item->count; - return 0; - } - pos = (pos + 1) & mask; - } - - if (dirty->count == dirty->cap) { - size_t next_cap = dirty->cap == 0 ? 16 : dirty->cap * 2; - count_dirty_slot *next = (count_dirty_slot *)realloc(dirty->items, next_cap * sizeof(count_dirty_slot)); - if (next == NULL) return -1; - dirty->items = next; - dirty->cap = next_cap; - } - dirty->items[dirty->count].slot = slot; - dirty->items[dirty->count].count = 1; - dirty->table[pos] = dirty->count + 1; - ++dirty->count; - return 0; -} - -static int increment_count_slot(count_sample_job *job, size_t slot) { - if (job->dirty_slots != NULL) return mark_count_dirty_slot(job->dirty_slots, slot); - ++job->counts[slot]; - return 0; -} - -static int direct_hamming_apply_match_result(count_sample_job *job, qdaln_match_result result, int saw_window) { - apply_ambiguity_policy(&result, job->assignment_policy); - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { - int kind = result.best_distance == 0 ? 0 : 1; - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + (size_t)kind) != 0) { - return -1; - } - ++job->stats->unique; - if (result.best_distance == 0) ++job->stats->exact; - else ++job->stats->corrected; - } else if (result.status == QDALN_MATCH_AMBIGUOUS) { - ++job->stats->ambiguous; - } else if (result.status == QDALN_MATCH_NONE) { - ++job->stats->unmatched; - } else if (saw_window) { - ++job->stats->unmatched; - } else { - ++job->stats->invalid; - } - return 0; -} - -static void direct_hamming_visit_seed(const count_sample_job *job, unsigned char seed_id, uint64_t seed_code, - uint64_t read_code, int *best_target, int *ambiguous) { - const hamming_lookup *lookup = job->hlookup; - size_t seed_len = seed_id == 0 ? lookup->seed0_len : lookup->target_len - lookup->seed0_len; - size_t slot = seed_hash_local(seed_code, seed_len, seed_id, lookup->seed_hash_cap); - for (int e = lookup->seed_heads[slot]; e >= 0; ) { - int nexte = lookup->seeds[e].next; - const hamming_seed_entry *entry = &lookup->seeds[e]; - if (entry->seed_id != seed_id || entry->code != seed_code) { - e = nexte; - continue; - } - int target_index = entry->target_index; - if (target_index < 0) { - e = nexte; - continue; - } - job->stats->candidates_considered += 1; - job->stats->candidates_verified += 1; - if (hamming_code_distance_local(read_code, lookup->target_codes[target_index], lookup->target_len) > 1) { - e = nexte; - continue; - } - direct_hamming_record_hit(target_index, 1, best_target, ambiguous); - e = nexte; - } -} - -static size_t selected_offset_at(const offset_list *offsets, size_t fallback_offset, size_t i) { - return offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; -} - -static int selected_offsets_are_sorted(const offset_list *offsets) { - if (offsets == NULL || offsets->count < 2) return 1; - for (size_t i = 1; i < offsets->count; ++i) { - if (offsets->items[i] < offsets->items[i - 1]) return 0; - } - return 1; -} - -static void fill_direct_hamming_codes(const char *seq, size_t seq_len, const offset_list *offsets, - size_t fallback_offset, size_t target_len, uint64_t *codes, - unsigned char *valid, unsigned char *invalid_counts, - unsigned char *bad_positions, size_t n_offsets, - int *saw_window, int *saw_non_acgt_window) { - *saw_window = 0; - *saw_non_acgt_window = 0; - memset(valid, 0, n_offsets); - memset(invalid_counts, 0, n_offsets); - memset(bad_positions, 0, n_offsets); - if (target_len == 0 || target_len > 32) return; - - if (!selected_offsets_are_sorted(offsets)) { - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = selected_offset_at(offsets, fallback_offset, i); - if (offset > seq_len || target_len > seq_len - offset) continue; - *saw_window = 1; - uint64_t code = 0; - unsigned char n_bad = 0; - unsigned char bad_pos = 0; - for (size_t j = 0; j < target_len; ++j) { - uint64_t value = 0; - if (!dna2_base_fold_value(seq[offset + j], &value)) { - if (n_bad < 255) ++n_bad; - bad_pos = (unsigned char)j; - } - code |= value << (2 * j); - } - if (n_bad != 0) { - *saw_non_acgt_window = 1; - codes[i] = code; - invalid_counts[i] = n_bad; - bad_positions[i] = bad_pos; - continue; - } - valid[i] = 1; - codes[i] = code; - } - return; - } - - uint64_t code = 0; - size_t invalid_count = 0; - size_t current_offset = 0; - int have_window = 0; - for (size_t i = 0; i < n_offsets; ++i) { - size_t offset = selected_offset_at(offsets, fallback_offset, i); - if (offset > seq_len || target_len > seq_len - offset) break; - - if (!have_window) { - code = 0; - invalid_count = 0; - size_t last_bad = 0; - for (size_t j = 0; j < target_len; ++j) { - uint64_t value = 0; - if (!dna2_base_fold_value(seq[offset + j], &value)) { - ++invalid_count; - last_bad = j; - } - code |= value << (2 * j); - } - current_offset = offset; - have_window = 1; - if (invalid_count == 1) bad_positions[i] = (unsigned char)last_bad; - } else { - while (current_offset < offset) { - uint64_t outgoing = 0; - if (!dna2_base_fold_value(seq[current_offset], &outgoing) && invalid_count != 0) --invalid_count; - (void)outgoing; - code >>= 2; - uint64_t incoming = 0; - if (!dna2_base_fold_value(seq[current_offset + target_len], &incoming)) ++invalid_count; - code |= incoming << (2 * (target_len - 1)); - ++current_offset; - } - if (invalid_count == 1) { - for (size_t j = 0; j < target_len; ++j) { - uint64_t value = 0; - if (!dna2_base_fold_value(seq[current_offset + j], &value)) { - bad_positions[i] = (unsigned char)j; - break; - } - } - } - } - - *saw_window = 1; - if (invalid_count == 0) { - valid[i] = 1; - codes[i] = code; - } else { - *saw_non_acgt_window = 1; - codes[i] = code; - invalid_counts[i] = invalid_count > 255 ? 255 : (unsigned char)invalid_count; - } - } -} - -static int direct_hamming_count_seq(count_sample_job *job, const char *seq, size_t seq_len) { - if (job->hlookup == NULL || !job->hlookup->ready) return 0; - ++job->stats->total; - count_progress_tick(job->progress); - - size_t n_offsets = job->selected_offsets == NULL || job->selected_offsets->count == 0 - ? 1 : job->selected_offsets->count; - if (n_offsets == 1) { - size_t offset = job->selected_offsets == NULL || job->selected_offsets->count == 0 - ? job->target_start : job->selected_offsets->items[0]; - if (offset > seq_len || job->hlookup->target_len > seq_len - offset) { - ++job->stats->invalid; - return 0; - } - - uint64_t code = 0; - unsigned char invalid_count = 0; - unsigned char bad_position = 0; - for (size_t j = 0; j < job->hlookup->target_len; ++j) { - uint64_t value = 0; - if (!dna2_base_fold_value(seq[offset + j], &value)) { - if (invalid_count < 255) ++invalid_count; - bad_position = (unsigned char)j; - } - code |= value << (2 * j); - } - - int exact_target = -1; - int exact_ambiguous = 0; - if (invalid_count == 0) { - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, code); - if (entry != NULL) { - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - if (job->k == 0) { - if (entry->match_count > 1) { - ++job->stats->ambiguous; - } else { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + - (size_t)entry->target_index) * 5) + 0) != 0) { - return -1; - } - ++job->stats->unique; - ++job->stats->exact; - } - return 0; - } - direct_hamming_record_hit(entry->target_index, entry->match_count, &exact_target, - &exact_ambiguous); - } - } - - if (job->k == 0) { - ++job->stats->unmatched; - return 0; - } - - if (exact_target >= 0) { - if (exact_ambiguous) { - ++job->stats->ambiguous; - } else { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)exact_target) * 5) + 0) != 0) { - return -1; - } - ++job->stats->unique; - ++job->stats->exact; - } - return 0; - } - - int mismatch_target = -1; - int mismatch_ambiguous = 0; - if (job->k == 1) { - if (invalid_count == 0) { - if (job->hlookup->seed_ready) { - uint64_t seed0 = code_segment_local(code, 0, job->hlookup->seed0_len); - uint64_t seed1 = code_segment_local(code, job->hlookup->seed0_len, - job->hlookup->target_len - job->hlookup->seed0_len); - direct_hamming_visit_seed(job, 0, seed0, code, &mismatch_target, &mismatch_ambiguous); - direct_hamming_visit_seed(job, 1, seed1, code, &mismatch_target, &mismatch_ambiguous); - } else { - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->mismatch, job->hlookup->mismatch_cap, code); - if (entry != NULL) { - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, - &mismatch_ambiguous); - } - } - } else if (invalid_count == 1) { - uint64_t shift = (uint64_t)2 * bad_position; - uint64_t mask = 3ULL << shift; - for (uint64_t b = 0; b < 4; ++b) { - uint64_t patched = (code & ~mask) | (b << shift); - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, patched); - if (entry == NULL) continue; - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, - &mismatch_ambiguous); - } - } - } - - if (mismatch_target >= 0) { - if (mismatch_ambiguous) { - ++job->stats->ambiguous; - } else { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)mismatch_target) * 5) + 1) != 0) { - return -1; - } - ++job->stats->unique; - ++job->stats->corrected; - } - } else { - ++job->stats->unmatched; - } - return 0; - } - - uint64_t inline_codes[64]; - unsigned char inline_valid[64]; - unsigned char inline_invalid_counts[64]; - unsigned char inline_bad_positions[64]; - uint64_t *codes = inline_codes; - unsigned char *valid = inline_valid; - unsigned char *invalid_counts = inline_invalid_counts; - unsigned char *bad_positions = inline_bad_positions; - if (n_offsets > sizeof(inline_codes) / sizeof(inline_codes[0])) { - codes = (uint64_t *)malloc(n_offsets * sizeof(uint64_t)); - valid = (unsigned char *)malloc(n_offsets); - invalid_counts = (unsigned char *)malloc(n_offsets); - bad_positions = (unsigned char *)malloc(n_offsets); - if (codes == NULL || valid == NULL || invalid_counts == NULL || bad_positions == NULL) { - free(codes); - free(valid); - free(invalid_counts); - free(bad_positions); - ++job->stats->invalid; - return -1; - } - } - - int saw_window = 0; - int saw_non_acgt_window = 0; - fill_direct_hamming_codes(seq, seq_len, job->selected_offsets, job->target_start, - job->hlookup->target_len, codes, valid, invalid_counts, - bad_positions, n_offsets, - &saw_window, &saw_non_acgt_window); - (void)saw_non_acgt_window; - - if (job->k == 1 && job->assignment_policy == AMBIGUITY_POLICY_RADIUS) { - match_merge_state merge; - merge_state_init(&merge); - int rc = 0; - for (size_t i = 0; i < n_offsets && rc == 0; ++i) { - if (valid[i]) { - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, codes[i]); - if (entry != NULL) { - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - rc = direct_hamming_merge_lookup_entry(&merge, entry, 0); - } - if (rc == 0) { - if (job->hlookup->seed_ready) { - uint64_t seed0 = code_segment_local(codes[i], 0, job->hlookup->seed0_len); - uint64_t seed1 = code_segment_local(codes[i], job->hlookup->seed0_len, - job->hlookup->target_len - job->hlookup->seed0_len); - if (direct_hamming_collect_seed_hits(job, &merge, 0, seed0, codes[i]) != 0 || - direct_hamming_collect_seed_hits(job, &merge, 1, seed1, codes[i]) != 0) { - rc = -1; - } - } else { - entry = hamming_lookup_find(job->hlookup->mismatch, job->hlookup->mismatch_cap, codes[i]); - if (entry != NULL) { - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - rc = direct_hamming_merge_lookup_entry(&merge, entry, 1); - } - } - } - } else if (invalid_counts[i] == 1) { - uint64_t shift = (uint64_t)2 * bad_positions[i]; - uint64_t mask = 3ULL << shift; - for (uint64_t b = 0; b < 4 && rc == 0; ++b) { - uint64_t patched = (codes[i] & ~mask) | (b << shift); - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, patched); - if (entry == NULL) continue; - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - rc = direct_hamming_merge_lookup_entry(&merge, entry, 1); - } - } - } - qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - if (rc == 0) { - merge_state_finish(&merge, &result); - if (result.status == QDALN_MATCH_INVALID && saw_window) { - result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; - } - rc = direct_hamming_apply_match_result(job, result, saw_window); - } - merge_state_free(&merge); - if (codes != inline_codes) { - free(codes); - free(valid); - free(invalid_counts); - free(bad_positions); - } - return rc; - } - - int exact_target = -1; - int exact_ambiguous = 0; - for (size_t i = 0; i < n_offsets; ++i) { - if (!valid[i]) continue; - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, codes[i]); - if (entry == NULL) continue; - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - direct_hamming_record_hit(entry->target_index, entry->match_count, &exact_target, &exact_ambiguous); - } - - if (exact_target >= 0) { - if (exact_ambiguous) { - ++job->stats->ambiguous; - } else { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)exact_target) * 5) + 0) != 0) { - if (codes != inline_codes) { - free(codes); - free(valid); - free(invalid_counts); - free(bad_positions); - } - return -1; - } - ++job->stats->unique; - ++job->stats->exact; - } - if (codes != inline_codes) { - free(codes); - free(valid); - free(invalid_counts); - free(bad_positions); - } - return 0; - } - - int mismatch_target = -1; - int mismatch_ambiguous = 0; - if (job->k == 1) { - for (size_t i = 0; i < n_offsets; ++i) { - if (valid[i]) { - if (job->hlookup->seed_ready) { - uint64_t seed0 = code_segment_local(codes[i], 0, job->hlookup->seed0_len); - uint64_t seed1 = code_segment_local(codes[i], job->hlookup->seed0_len, - job->hlookup->target_len - job->hlookup->seed0_len); - direct_hamming_visit_seed(job, 0, seed0, codes[i], &mismatch_target, &mismatch_ambiguous); - direct_hamming_visit_seed(job, 1, seed1, codes[i], &mismatch_target, &mismatch_ambiguous); - } else { - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->mismatch, job->hlookup->mismatch_cap, codes[i]); - if (entry == NULL) continue; - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, - &mismatch_ambiguous); - } - } else if (invalid_counts[i] == 1) { - uint64_t shift = (uint64_t)2 * bad_positions[i]; - uint64_t mask = 3ULL << shift; - for (uint64_t b = 0; b < 4; ++b) { - uint64_t patched = (codes[i] & ~mask) | (b << shift); - const hamming_lookup_entry *entry = - hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, patched); - if (entry == NULL) continue; - job->stats->candidates_considered += (unsigned long long)entry->match_count; - job->stats->candidates_verified += (unsigned long long)entry->match_count; - direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, - &mismatch_ambiguous); - } - } - } - } - - if (mismatch_target >= 0) { - if (mismatch_ambiguous) { - ++job->stats->ambiguous; - } else { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)mismatch_target) * 5) + 1) != 0) { - if (codes != inline_codes) { - free(codes); - free(valid); - free(invalid_counts); - free(bad_positions); - } - return -1; - } - ++job->stats->unique; - ++job->stats->corrected; - } - } else if (saw_window) { - ++job->stats->unmatched; - } else { - ++job->stats->invalid; - } - - if (codes != inline_codes) { - free(codes); - free(valid); - free(invalid_counts); - free(bad_positions); - } - return 0; -} - -typedef struct direct_hamming_batch_job { - count_sample_job job; - char **items; - size_t *lens; - size_t start; - size_t end; - count_dirty_slots dirty_slots; - count_stats local_stats; - int rc; -} direct_hamming_batch_job; - -static void *direct_hamming_batch_worker(void *arg) { - direct_hamming_batch_job *batch = (direct_hamming_batch_job *)arg; - batch->job.sample_index = 0; - batch->job.stats = &batch->local_stats; - batch->job.dirty_slots = &batch->dirty_slots; - batch->rc = 0; - for (size_t i = batch->start; i < batch->end; ++i) { - if (direct_hamming_count_seq(&batch->job, batch->items[i], batch->lens[i]) != 0) { - batch->rc = 1; - break; - } - } - return NULL; -} - -static int process_direct_hamming_buffer(count_sample_job *job, const seq_buffer *buffer) { - if (buffer->count == 0) return 0; - if (job->k == 0 && job->index != NULL && job->hlookup != NULL && job->hlookup->ready && - job->hlookup->target_len == job->target_len && (job->selected_offsets == NULL || job->selected_offsets->count <= 1)) { - qdaln_match_result *results = (qdaln_match_result *)calloc(buffer->count, sizeof(qdaln_match_result)); - if (results == NULL) return 1; - qdaln_index_stats batch_stats = {0, 0}; - if (qdaln_index_lookup_exact_ascii_many_stats(job->index, (const char *const *)buffer->items, buffer->lens, - buffer->count, results, &batch_stats) != 0) { - free(results); - return 1; - } - job->stats->candidates_considered += batch_stats.candidates_considered; - job->stats->candidates_verified += batch_stats.candidates_verified; - for (size_t i = 0; i < buffer->count; ++i) { - qdaln_match_result result = results[i]; - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + 0) != 0) { - free(results); - return 1; - } - ++job->stats->unique; - ++job->stats->exact; - } else if (result.status == QDALN_MATCH_AMBIGUOUS) { - ++job->stats->ambiguous; - } else if (result.status == QDALN_MATCH_NONE) { - ++job->stats->unmatched; - } else { - ++job->stats->invalid; - } - } - free(results); - return 0; - } - size_t read_threads = job->read_threads; - if (read_threads <= 1 || buffer->count < 1024) { - for (size_t i = 0; i < buffer->count; ++i) { - if (direct_hamming_count_seq(job, buffer->items[i], buffer->lens[i]) != 0) return 1; - } - return 0; - } - if (read_threads > buffer->count) read_threads = buffer->count; - - size_t target_slots = job->targets->count * 5; - pthread_t *thread_ids = (pthread_t *)calloc(read_threads, sizeof(pthread_t)); - direct_hamming_batch_job *jobs = (direct_hamming_batch_job *)calloc(read_threads, sizeof(direct_hamming_batch_job)); - if (thread_ids == NULL || jobs == NULL) { - free(thread_ids); - free(jobs); - return 1; - } - - size_t launched = 0; - int rc = 0; - for (size_t t = 0; t < read_threads; ++t) { - size_t start = (buffer->count * t) / read_threads; - size_t end = (buffer->count * (t + 1)) / read_threads; - jobs[t].job = *job; - jobs[t].items = buffer->items; - jobs[t].lens = buffer->lens; - jobs[t].start = start; - jobs[t].end = end; - if (pthread_create(&thread_ids[t], NULL, direct_hamming_batch_worker, &jobs[t]) != 0) { - rc = 1; - break; - } - ++launched; - } - - for (size_t t = 0; t < launched; ++t) { - pthread_join(thread_ids[t], NULL); - if (jobs[t].rc != 0) rc = 1; - } - if (rc == 0) { - size_t dst_offset = job->sample_index * target_slots; - for (size_t t = 0; t < launched; ++t) { - merge_count_stats(job->stats, &jobs[t].local_stats); - for (size_t i = 0; i < jobs[t].dirty_slots.count; ++i) { - size_t slot = jobs[t].dirty_slots.items[i].slot; - job->counts[dst_offset + slot] += jobs[t].dirty_slots.items[i].count; - } - } - } - - for (size_t t = 0; t < read_threads; ++t) free_count_dirty_slots(&jobs[t].dirty_slots); - free(thread_ids); - free(jobs); - return rc; -} - -static void score_offsets_for_seq(const hamming_lookup *lookup, const char *seq, size_t seq_len, - size_t target_start, size_t target_len, size_t range, - unsigned long long *scores) { - if (lookup == NULL || !lookup->ready || lookup->target_len != target_len) return; - size_t n_offsets = 0; - if (offset_count_for_range(range, &n_offsets) != 0) return; - for (size_t oi = 0; oi < n_offsets; ++oi) { - long delta = (long)oi - (long)range; - if (delta < 0 && target_start < (size_t)(-delta)) continue; - size_t offset = delta < 0 ? target_start - (size_t)(-delta) : target_start + (size_t)delta; - if (offset > seq_len || target_len > seq_len - offset) continue; - uint64_t code = 0; - if (!dna2_code_local_fold(seq + offset, target_len, &code)) continue; - const hamming_lookup_entry *entry = hamming_lookup_find(lookup->exact, lookup->exact_cap, code); - if (entry != NULL && entry->match_count == 1) ++scores[oi]; - } -} - -static int select_offsets_from_scores(size_t target_start, size_t range, const unsigned long long *scores, - size_t checked, offset_mode mode, double min_fraction, - offset_list *selected_offsets); - -static int count_sample_worker_direct_hamming(count_sample_job *job) { - fastq_reader reader = {0}; - if (fastq_reader_open(&reader, job->reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - return 1; - } - - char seq[8192]; - int got = 0; - - if (job->fused_offset_detection) { - size_t n_offsets = 0; - if (offset_count_for_range(job->auto_offset, &n_offsets) != 0) { - fastq_reader_close(&reader); - return 1; - } - unsigned long long *scores = (unsigned long long *)calloc(n_offsets, sizeof(unsigned long long)); - if (scores == NULL) { - fastq_reader_close(&reader); - return 1; - } - seq_buffer buffered = {0}; - if (reserve_seq_buffer(&buffered, job->auto_offset_sample) != 0) { - free_seq_buffer(&buffered); - free(scores); - fastq_reader_close(&reader); - return 1; - } - size_t checked = 0; - size_t seq_len = 0; - while (checked < job->auto_offset_sample && - (got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { - score_offsets_for_seq(job->hlookup, seq, seq_len, job->target_start, job->target_len, - job->auto_offset, scores); - if (push_seq_buffer(&buffered, seq, seq_len) != 0) { - free_seq_buffer(&buffered); - free(scores); - fastq_reader_close(&reader); - return 1; - } - ++checked; - } - if (got < 0 || - select_offsets_from_scores(job->target_start, job->auto_offset, scores, checked, job->offsets_mode, - job->offset_min_fraction, job->selected_offsets) != 0) { - free_seq_buffer(&buffered); - free(scores); - fastq_reader_close(&reader); - return 1; - } - free(scores); - if (process_direct_hamming_buffer(job, &buffered) != 0) { - free_seq_buffer(&buffered); - fastq_reader_close(&reader); - return 1; - } - free_seq_buffer(&buffered); - } - - size_t seq_len = 0; - if (job->read_threads <= 1) { - while ((got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { - if (direct_hamming_count_seq(job, seq, seq_len) != 0) { - fastq_reader_close(&reader); - return 1; - } - } - } else { - const size_t batch_reads = 1048576; - seq_buffer batch = {0}; - if (reserve_seq_buffer(&batch, batch_reads) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - return 1; - } - while ((got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { - if (push_seq_buffer(&batch, seq, seq_len) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - return 1; - } - if (batch.count == batch_reads) { - if (process_direct_hamming_buffer(job, &batch) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - return 1; - } - reset_seq_buffer(&batch); - if (reserve_seq_buffer(&batch, batch_reads) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - return 1; - } - } - } - if (got >= 0 && batch.count != 0 && process_direct_hamming_buffer(job, &batch) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - return 1; - } - free_seq_buffer(&batch); - } - fastq_reader_close(&reader); - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - return 1; - } - return 0; -} - -static int pack_read_window_code(const char *seq, size_t seq_len, size_t offset, size_t target_len, uint64_t *code_out) { - if (offset > seq_len || target_len > seq_len - offset) return 0; - return dna2_code_local(seq + offset, target_len, code_out); -} - -static int apply_metal_match_to_counts(count_sample_job *job, const qdmetal_match_result *metal_result) { - qdaln_match_result result = {metal_result->target_index, metal_result->best_distance, - metal_result->second_best_distance, metal_result->match_count, - metal_result->status}; - return direct_hamming_apply_match_result(job, result, 1); -} - -static int count_sample_worker_metal_hamming(count_sample_job *job) { - if (job->metal_target_codes == NULL) return 1; - fastq_reader reader = {0}; - if (fastq_reader_open(&reader, job->reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - return 1; - } - - size_t window_offset = job->selected_offsets == NULL || job->selected_offsets->count == 0 - ? job->target_start - : job->selected_offsets->items[0]; - const size_t batch_cap = 262144; - uint64_t *read_codes = (uint64_t *)malloc(batch_cap * sizeof(uint64_t)); - qdmetal_match_result *metal_results = - (qdmetal_match_result *)malloc(batch_cap * sizeof(qdmetal_match_result)); - if (read_codes == NULL || metal_results == NULL) { - free(read_codes); - free(metal_results); - fastq_reader_close(&reader); - return 1; - } - - char seq[8192]; - size_t seq_len = 0; - size_t batch_count = 0; - int got = 0; - int rc = 0; - - while (rc == 0 && (got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { - ++job->stats->total; - count_progress_tick(job->progress); - uint64_t code = 0; - if (!pack_read_window_code(seq, seq_len, window_offset, job->target_len, &code)) { - if (window_offset > seq_len || job->target_len > seq_len - window_offset) ++job->stats->invalid; - else ++job->stats->unmatched; - continue; - } - read_codes[batch_count++] = code; - if (batch_count == batch_cap) { - qdmetal_assign_stats mstats = {0, 0, NULL, NULL}; - if (qdmetal_hamming_assign(read_codes, batch_count, job->metal_target_codes, job->targets->count, - job->target_len, job->k, metal_results, &mstats) != 0) { - rc = 1; - break; - } - job->stats->candidates_considered += (unsigned long long)mstats.candidates_considered; - job->stats->candidates_verified += (unsigned long long)mstats.candidates_verified; - for (size_t i = 0; i < batch_count; ++i) { - if (apply_metal_match_to_counts(job, &metal_results[i]) != 0) { - rc = 1; - break; - } - } - batch_count = 0; - } - } - - if (rc == 0 && got >= 0 && batch_count != 0) { - qdmetal_assign_stats mstats = {0, 0, NULL, NULL}; - if (qdmetal_hamming_assign(read_codes, batch_count, job->metal_target_codes, job->targets->count, - job->target_len, job->k, metal_results, &mstats) != 0) { - rc = 1; - } else { - job->stats->candidates_considered += (unsigned long long)mstats.candidates_considered; - job->stats->candidates_verified += (unsigned long long)mstats.candidates_verified; - for (size_t i = 0; i < batch_count; ++i) { - if (apply_metal_match_to_counts(job, &metal_results[i]) != 0) { - rc = 1; - break; - } - } - } - } - - free(read_codes); - free(metal_results); - fastq_reader_close(&reader); - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - return 1; - } - return rc; -} - -static int count_sample_sequence(count_sample_job *job, const char *seq, size_t seq_len, const char *qual, - const char *read_id) { - char observed[8192]; - qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_index_stats istats = {0, 0}; - observed[0] = '\0'; - ++job->stats->total; - count_progress_tick(job->progress); - int best_exact_shortcut = job->k == 1 && - job->assignment_policy == AMBIGUITY_POLICY_BEST && job->assignments == NULL && - job->ambiguous_out == NULL && job->unmatched_out == NULL; - int handled = 0; - if (job->metric == COUNT_METRIC_LEVENSHTEIN && job->k == 1 && job->indel_window == 1 && - job->levlookup != NULL && job->levlookup->ready && - (job->selected_offsets == NULL || job->selected_offsets->count <= 1)) { - size_t offset = job->selected_offsets == NULL || job->selected_offsets->count == 0 - ? job->target_start : job->selected_offsets->items[0]; - int lookup_rc = assign_levenshtein1_lookup_offset(job->levlookup, seq, seq_len, offset, &result, - &istats, observed, sizeof(observed)); - if (lookup_rc < 0) return -1; - handled = lookup_rc; - } - if (job->metric == COUNT_METRIC_HAMMING && job->indel_window == 0 && job->hlookup != NULL && job->hlookup->ready) { - int exact_merge = job->assignment_policy == AMBIGUITY_POLICY_RADIUS || - job->assignments != NULL || job->ambiguous_out != NULL || - job->unmatched_out != NULL; - int lookup_rc = assign_hamming_lookup_offsets(job->hlookup, seq, seq_len, job->selected_offsets, 0, - job->k, &result, &istats, observed, sizeof(observed), - exact_merge); - if (lookup_rc < 0) return -1; - handled = lookup_rc; - } - if (!handled && - assign_count_offsets(job->index, seq, seq_len, job->selected_offsets, 0, job->target_len, job->k, - job->metric, job->indel_window, &result, &istats, observed, - sizeof(observed), best_exact_shortcut) != 0) { - return -1; - } - apply_ambiguity_policy(&result, job->assignment_policy); - if (result.status != QDALN_MATCH_INVALID) { - job->stats->candidates_considered += (unsigned long long)istats.candidates_considered; - job->stats->candidates_verified += (unsigned long long)istats.candidates_verified; - } - - const char *correction = "invalid"; - int quality_rejected = 0; - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0 && result.best_distance > 0) { - seq_record *target = &job->targets->records[result.target_index]; - if (!quality_allows_unique_correction(seq, seq_len, qual, job->selected_offsets, 0, job->target_len, - job->metric, job->indel_window, job->k, observed, target, result, - job->max_correction_qual)) { - result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; - quality_rejected = 1; - } - } - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { - seq_record *target = &job->targets->records[result.target_index]; - int kind = correction_kind(observed, strlen(observed), target->seq, target->len, result.best_distance); - size_t slot = ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + (size_t)kind; - if (increment_count_slot(job, slot) != 0) return -1; - ++job->stats->unique; - if (result.best_distance == 0) ++job->stats->exact; - else ++job->stats->corrected; - correction = correction_name(kind); - } else if (result.status == QDALN_MATCH_AMBIGUOUS) { - ++job->stats->ambiguous; - correction = "ambiguous"; - } else if (result.status == QDALN_MATCH_NONE) { - ++job->stats->unmatched; - correction = quality_rejected ? "quality_rejected" : "none"; - } else { - ++job->stats->invalid; - } - - const char *id = read_id == NULL ? "" : read_id; - if (job->assignments != NULL && - (result.status != QDALN_MATCH_AMBIGUOUS || strcmp(job->ambiguous_policy, "report") == 0)) { - write_assignment_like_row(job->assignments, job->targets, job->sample_label, id, observed, result, - correction); - } - if (job->ambiguous_out != NULL && result.status == QDALN_MATCH_AMBIGUOUS) { - write_assignment_like_row(job->ambiguous_out, job->targets, job->sample_label, id, observed, result, - correction); - } - if (job->unmatched_out != NULL && - (result.status == QDALN_MATCH_NONE || result.status == QDALN_MATCH_INVALID)) { - write_assignment_like_row(job->unmatched_out, job->targets, job->sample_label, id, observed, result, - correction); - } - return 0; -} - -typedef struct count_batch_job { - count_sample_job job; - char **items; - size_t *lens; - size_t start; - size_t end; - count_dirty_slots dirty_slots; - count_stats local_stats; - int rc; -} count_batch_job; - -static void *count_batch_worker(void *arg) { - count_batch_job *batch = (count_batch_job *)arg; - batch->job.sample_index = 0; - batch->job.stats = &batch->local_stats; - batch->job.dirty_slots = &batch->dirty_slots; - batch->job.assignments = NULL; - batch->job.ambiguous_out = NULL; - batch->job.unmatched_out = NULL; - batch->rc = 0; - for (size_t i = batch->start; i < batch->end; ++i) { - if (count_sample_sequence(&batch->job, batch->items[i], batch->lens[i], NULL, NULL) != 0) { - batch->rc = 1; - break; - } - } - return NULL; -} - -static int process_count_buffer(count_sample_job *job, const seq_buffer *buffer) { - if (buffer->count == 0) return 0; - if (job->k == 0 && job->metric == COUNT_METRIC_HAMMING && job->indel_window == 0 && job->index != NULL && - job->hlookup != NULL && job->hlookup->ready && job->hlookup->target_len == job->target_len && - job->assignments == NULL && job->ambiguous_out == NULL && job->unmatched_out == NULL && - job->max_correction_qual < 0 && job->assignment_policy == AMBIGUITY_POLICY_BEST) { - qdaln_match_result *results = (qdaln_match_result *)calloc(buffer->count, sizeof(qdaln_match_result)); - if (results == NULL) return 1; - qdaln_index_stats batch_stats = {0, 0}; - if (qdaln_index_lookup_exact_ascii_many_stats(job->index, (const char *const *)buffer->items, buffer->lens, - buffer->count, results, &batch_stats) != 0) { - free(results); - return 1; - } - job->stats->candidates_considered += batch_stats.candidates_considered; - job->stats->candidates_verified += batch_stats.candidates_verified; - for (size_t i = 0; i < buffer->count; ++i) { - qdaln_match_result result = results[i]; - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { - if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + 0) != 0) { - free(results); - return 1; - } - ++job->stats->unique; - ++job->stats->exact; - } else if (result.status == QDALN_MATCH_AMBIGUOUS) { - ++job->stats->ambiguous; - } else if (result.status == QDALN_MATCH_NONE) { - ++job->stats->unmatched; - } else { - ++job->stats->invalid; - } - } - free(results); - return 0; - } - size_t read_threads = job->read_threads; - if (read_threads <= 1 || buffer->count < 1024) { - for (size_t i = 0; i < buffer->count; ++i) { - if (count_sample_sequence(job, buffer->items[i], buffer->lens[i], NULL, NULL) != 0) return 1; - } - return 0; - } - if (read_threads > buffer->count) read_threads = buffer->count; - - size_t target_slots = job->targets->count * 5; - pthread_t *thread_ids = (pthread_t *)calloc(read_threads, sizeof(pthread_t)); - count_batch_job *jobs = (count_batch_job *)calloc(read_threads, sizeof(count_batch_job)); - if (thread_ids == NULL || jobs == NULL) { - free(thread_ids); - free(jobs); - return 1; - } - - size_t launched = 0; - int rc = 0; - for (size_t t = 0; t < read_threads; ++t) { - size_t start = (buffer->count * t) / read_threads; - size_t end = (buffer->count * (t + 1)) / read_threads; - jobs[t].job = *job; - jobs[t].items = buffer->items; - jobs[t].lens = buffer->lens; - jobs[t].start = start; - jobs[t].end = end; - if (pthread_create(&thread_ids[t], NULL, count_batch_worker, &jobs[t]) != 0) { - rc = 1; - break; - } - ++launched; - } - - for (size_t t = 0; t < launched; ++t) { - pthread_join(thread_ids[t], NULL); - if (jobs[t].rc != 0) rc = 1; - } - if (rc == 0) { - size_t dst_offset = job->sample_index * target_slots; - for (size_t t = 0; t < launched; ++t) { - merge_count_stats(job->stats, &jobs[t].local_stats); - for (size_t i = 0; i < jobs[t].dirty_slots.count; ++i) { - size_t slot = jobs[t].dirty_slots.items[i].slot; - job->counts[dst_offset + slot] += jobs[t].dirty_slots.items[i].count; - } - } - } - - for (size_t t = 0; t < read_threads; ++t) free_count_dirty_slots(&jobs[t].dirty_slots); - free(thread_ids); - free(jobs); - return rc; -} - -static void *count_sample_worker(void *arg); - -typedef struct count_samples_args { - const qdaln_index *index; - const hamming_lookup *hlookup; - const levenshtein1_lookup *levlookup; - const seq_table *targets; - const char **target_ptrs; - const size_t *target_lens; - const string_list *reads; - const string_list *labels; - offset_list *selected_offsets; - size_t target_len; - int k; - count_metric metric; - size_t indel_window; - unsigned long long *counts; - count_stats *stats_by_sample; - FILE *assignments; - FILE *ambiguous_out; - FILE *unmatched_out; - const char *ambiguous_policy; - ambiguity_policy assignment_policy; - int direct_hamming_counts; - int metal_hamming_counts; - const uint64_t *metal_target_codes; - int fused_offset_detection; - size_t target_start; - size_t auto_offset; - size_t auto_offset_sample; - offset_mode offsets_mode; - double offset_min_fraction; - size_t effective_read_threads; - int max_correction_qual; - size_t sample_threads; - count_progress *progress_by_sample; -} count_samples_args; - -static int run_count_samples_phase(const count_samples_args *args) { - if (args == NULL || args->reads == NULL || args->labels == NULL || args->counts == NULL || - args->stats_by_sample == NULL) { - return -1; - } - size_t sample_threads = args->sample_threads; - if (sample_threads <= 1 || args->reads->count <= 1) { - for (size_t sample = 0; sample < args->reads->count; ++sample) { - count_sample_job job = { - args->index, args->hlookup, args->levlookup, args->targets, args->target_ptrs, args->target_lens, - args->reads->items[sample], args->labels->items[sample], sample, &args->selected_offsets[sample], - args->target_len, args->k, args->metric, args->indel_window, args->counts, &args->stats_by_sample[sample], - args->assignments, args->ambiguous_out, args->unmatched_out, args->ambiguous_policy, - args->assignment_policy, args->direct_hamming_counts, args->metal_hamming_counts, - args->metal_target_codes, args->fused_offset_detection, args->target_start, args->auto_offset, - args->auto_offset_sample, args->offsets_mode, args->offset_min_fraction, args->effective_read_threads, - args->max_correction_qual, 1, NULL, - args->progress_by_sample != NULL ? &args->progress_by_sample[sample] : NULL}; - count_sample_worker(&job); - if (job.rc != 0) return job.rc; - } - return 0; - } - - pthread_t *thread_ids = (pthread_t *)calloc(sample_threads, sizeof(pthread_t)); - count_sample_job *jobs = (count_sample_job *)calloc(args->reads->count, sizeof(count_sample_job)); - if (thread_ids == NULL || jobs == NULL) { - free(thread_ids); - free(jobs); - fprintf(stderr, "out of memory\n"); - return -1; - } - int rc = 0; - size_t next_sample = 0; - while (next_sample < args->reads->count && rc == 0) { - size_t batch = args->reads->count - next_sample; - if (batch > sample_threads) batch = sample_threads; - for (size_t i = 0; i < batch; ++i) { - size_t sample = next_sample + i; - jobs[sample] = (count_sample_job){ - args->index, args->hlookup, args->levlookup, args->targets, args->target_ptrs, args->target_lens, - args->reads->items[sample], args->labels->items[sample], sample, &args->selected_offsets[sample], - args->target_len, args->k, args->metric, args->indel_window, args->counts, &args->stats_by_sample[sample], - NULL, NULL, NULL, args->ambiguous_policy, args->assignment_policy, args->direct_hamming_counts, - args->metal_hamming_counts, args->metal_target_codes, args->fused_offset_detection, args->target_start, - args->auto_offset, args->auto_offset_sample, args->offsets_mode, args->offset_min_fraction, 1, - args->max_correction_qual, 1, NULL, - args->progress_by_sample != NULL ? &args->progress_by_sample[sample] : NULL}; - if (pthread_create(&thread_ids[i], NULL, count_sample_worker, &jobs[sample]) != 0) { - fprintf(stderr, "failed to create worker thread\n"); - batch = i; - rc = -1; - break; - } - } - for (size_t i = 0; i < batch; ++i) { - pthread_join(thread_ids[i], NULL); - if (rc == 0 && jobs[next_sample + i].rc != 0) rc = jobs[next_sample + i].rc; - } - next_sample += batch; - } - free(thread_ids); - free(jobs); - return rc; -} - -static unsigned long long count_matrix_cell_total(const unsigned long long *counts, size_t n_targets, size_t sample, - size_t target) { - unsigned long long total = 0; - for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * n_targets + target) * 5) + kind]; - return total; -} - -static int validate_metal_counts_against_cpu(const unsigned long long *metal_counts, const unsigned long long *cpu_counts, - size_t n_samples, size_t n_targets, size_t *diff_guides_out, - long long *delta_reads_out, char *example_guide, size_t example_cap) { - if (metal_counts == NULL || cpu_counts == NULL || diff_guides_out == NULL || delta_reads_out == NULL) return -1; - *diff_guides_out = 0; - *delta_reads_out = 0; - if (example_guide != NULL && example_cap > 0) example_guide[0] = '\0'; - for (size_t t = 0; t < n_targets; ++t) { - int guide_diff = 0; - for (size_t s = 0; s < n_samples; ++s) { - unsigned long long metal_total = count_matrix_cell_total(metal_counts, n_targets, s, t); - unsigned long long cpu_total = count_matrix_cell_total(cpu_counts, n_targets, s, t); - if (metal_total != cpu_total) { - guide_diff = 1; - *delta_reads_out += (long long)metal_total - (long long)cpu_total; - if (example_guide != NULL && example_guide[0] == '\0' && example_cap > 0) { - snprintf(example_guide, example_cap, "sample_index=%zu target_index=%zu metal=%llu cpu=%llu", s, t, - metal_total, cpu_total); - } - } - } - if (guide_diff) ++*diff_guides_out; - } - return *diff_guides_out == 0 ? 0 : 1; -} - -static int env_truthy(const char *value) { - if (value == NULL || value[0] == '\0') return 0; - if (strcmp(value, "0") == 0) return 0; - if (strcmp(value, "false") == 0 || strcmp(value, "FALSE") == 0) return 0; - if (strcmp(value, "no") == 0 || strcmp(value, "NO") == 0) return 0; - return 1; -} - -static void *count_sample_worker(void *arg) { - count_sample_job *job = (count_sample_job *)arg; - if (job->metal_hamming_counts) { - job->rc = count_sample_worker_metal_hamming(job); - return NULL; - } - if (job->direct_hamming_counts) { - job->rc = count_sample_worker_direct_hamming(job); - return NULL; - } - - fastq_reader reader = {0}; - if (fastq_reader_open(&reader, job->reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - job->rc = 1; - return NULL; - } - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - char read_id[8192]; - int got = 0; - int need_read_id = job->assignments != NULL || job->ambiguous_out != NULL || job->unmatched_out != NULL; - int need_quality = job->max_correction_qual >= 0; - int need_full_record = need_read_id || need_quality; - size_t seq_len = 0; - if (job->read_threads > 1 && !need_full_record) { - const size_t batch_reads = 1048576; - seq_buffer batch = {0}; - if (reserve_seq_buffer(&batch, batch_reads) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - job->rc = 1; - return NULL; - } - while ((got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { - if (push_seq_buffer(&batch, seq, seq_len) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - job->rc = 1; - return NULL; - } - if (batch.count == batch_reads) { - if (process_count_buffer(job, &batch) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - fprintf(stderr, "FASTQ assignment failed\n"); - job->rc = 1; - return NULL; - } - reset_seq_buffer(&batch); - if (reserve_seq_buffer(&batch, batch_reads) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - job->rc = 1; - return NULL; - } - } - } - if (got >= 0 && batch.count != 0 && process_count_buffer(job, &batch) != 0) { - free_seq_buffer(&batch); - fastq_reader_close(&reader); - fprintf(stderr, "FASTQ assignment failed\n"); - job->rc = 1; - return NULL; - } - free_seq_buffer(&batch); - } else { - while ((got = need_full_record - ? fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len) - : fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { - read_id[0] = '\0'; - if (need_read_id) fastq_read_id(header, read_id, sizeof(read_id)); - if (count_sample_sequence(job, seq, seq_len, job->max_correction_qual >= 0 ? qual : NULL, - read_id) != 0) { - fastq_reader_close(&reader); - fprintf(stderr, "FASTQ assignment failed\n"); - job->rc = 1; - return NULL; - } - } - } - fastq_reader_close(&reader); - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - job->rc = 1; - return NULL; - } - job->rc = 0; - return NULL; -} - -static int select_offsets_from_scores(size_t target_start, size_t range, const unsigned long long *scores, - size_t checked, offset_mode mode, double min_fraction, - offset_list *selected_offsets) { - free_offset_list(selected_offsets); - if (range == 0) return push_offset_unique(selected_offsets, target_start); - - size_t n_offsets = 0; - if (offset_count_for_range(range, &n_offsets) != 0) return -1; - - size_t best_i = range; - for (size_t oi = 0; oi < n_offsets; ++oi) { - size_t best_dist = best_i > range ? best_i - range : range - best_i; - size_t this_dist = oi > range ? oi - range : range - oi; - if (scores[oi] > scores[best_i] || (scores[oi] == scores[best_i] && this_dist < best_dist)) { - best_i = oi; - } - } - - int rc = 0; - if (mode == OFFSET_MODE_MULTI && checked != 0) { - for (size_t oi = 0; oi < n_offsets; ++oi) { - double fraction = (double)scores[oi] / (double)checked; - if (fraction + 1e-12 < min_fraction) continue; - long delta = (long)oi - (long)range; - if (delta < 0 && target_start < (size_t)(-delta)) continue; - size_t offset = delta < 0 ? target_start - (size_t)(-delta) : target_start + (size_t)delta; - if (push_offset_unique(selected_offsets, offset) != 0) { - rc = -1; - break; - } - } - } - if (rc == 0 && selected_offsets->count == 0 && scores[best_i] != 0) { - long best_delta = (long)best_i - (long)range; - size_t offset = best_delta < 0 ? target_start - (size_t)(-best_delta) : target_start + (size_t)best_delta; - rc = push_offset_unique(selected_offsets, offset); - } - if (rc == 0 && selected_offsets->count == 0) { - rc = push_offset_unique(selected_offsets, target_start); - } - return rc; -} - -static int detect_offsets(const qdaln_index *index, const hamming_lookup *exact_lookup, const char *reads_path, size_t target_start, - size_t target_len, size_t range, size_t sample_limit, offset_mode mode, - double min_fraction, offset_list *selected_offsets) { - if (range == 0) { - free_offset_list(selected_offsets); - return push_offset_unique(selected_offsets, target_start); - } - - size_t n_offsets = 0; - if (offset_count_for_range(range, &n_offsets) != 0) return -1; - unsigned long long *scores = (unsigned long long *)calloc(n_offsets, sizeof(unsigned long long)); - if (scores == NULL) return -1; - - fastq_reader reader = {0}; - if (fastq_reader_open(&reader, reads_path) != 0) { - free(scores); - return -1; - } - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - size_t checked = 0; - int got = 0; - size_t seq_len = 0; - while (checked < sample_limit && - (got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - if (exact_lookup != NULL && exact_lookup->ready && exact_lookup->target_len == target_len) { - score_offsets_for_seq(exact_lookup, seq, seq_len, target_start, target_len, range, scores); - } else { - for (size_t oi = 0; oi < n_offsets; ++oi) { - long delta = (long)oi - (long)range; - if (delta < 0 && target_start < (size_t)(-delta)) continue; - size_t offset = delta < 0 ? target_start - (size_t)(-delta) : target_start + (size_t)delta; - if (offset > seq_len || target_len > seq_len - offset) continue; - char observed[8192]; - if (target_len >= sizeof(observed)) continue; - memcpy(observed, seq + offset, target_len); - observed[target_len] = '\0'; - uppercase_ascii(observed); - const char *read_ptr = observed; - size_t read_len = target_len; - qdaln_match_result r; - if (qdaln_index_assign(index, &read_ptr, &read_len, 1, 0, &r) != 0) { - fastq_reader_close(&reader); - free(scores); - return -1; - } - if (r.status == QDALN_MATCH_UNIQUE) ++scores[oi]; - } - } - ++checked; - } - - fastq_reader_close(&reader); - if (got < 0) { - free(scores); - return -1; - } - - int rc = select_offsets_from_scores(target_start, range, scores, checked, mode, min_fraction, selected_offsets); - free(scores); - return rc; -} - -typedef struct offset_detect_job { - const qdaln_index *index; - const hamming_lookup *exact_lookup; - const char *reads_path; - size_t target_start; - size_t target_len; - size_t range; - size_t sample_limit; - offset_mode mode; - double min_fraction; - offset_list *selected_offsets; - int rc; -} offset_detect_job; - -static void *detect_offsets_worker(void *arg) { - offset_detect_job *job = (offset_detect_job *)arg; - job->rc = detect_offsets(job->index, job->exact_lookup, job->reads_path, job->target_start, job->target_len, - job->range, job->sample_limit, job->mode, job->min_fraction, job->selected_offsets); - return NULL; -} - -static int detect_offsets_for_samples(const qdaln_index *index, const hamming_lookup *exact_lookup, - const string_list *reads, size_t target_start, size_t target_len, - size_t range, size_t sample_limit, offset_mode mode, double min_fraction, - offset_list *selected_offsets, size_t threads) { - if (reads->count == 0) return 0; - if (threads > reads->count) threads = reads->count; - if (threads <= 1 || reads->count <= 1) { - for (size_t sample = 0; sample < reads->count; ++sample) { - if (detect_offsets(index, exact_lookup, reads->items[sample], target_start, target_len, range, - sample_limit, mode, min_fraction, &selected_offsets[sample]) != 0) { - return -1; - } - } - return 0; - } - - pthread_t *thread_ids = (pthread_t *)calloc(threads, sizeof(pthread_t)); - offset_detect_job *jobs = (offset_detect_job *)calloc(reads->count, sizeof(offset_detect_job)); - if (thread_ids == NULL || jobs == NULL) { - free(thread_ids); - free(jobs); - return -1; - } - - int rc = 0; - size_t next_sample = 0; - while (next_sample < reads->count && rc == 0) { - size_t batch = reads->count - next_sample; - if (batch > threads) batch = threads; - for (size_t i = 0; i < batch; ++i) { - size_t sample = next_sample + i; - jobs[sample] = (offset_detect_job){index, exact_lookup, reads->items[sample], target_start, target_len, - range, sample_limit, mode, min_fraction, - &selected_offsets[sample], 0}; - if (pthread_create(&thread_ids[i], NULL, detect_offsets_worker, &jobs[sample]) != 0) { - batch = i; - rc = -1; - break; - } - } - for (size_t i = 0; i < batch; ++i) { - pthread_join(thread_ids[i], NULL); - if (jobs[next_sample + i].rc != 0) rc = -1; - } - next_sample += batch; - } - - free(thread_ids); - free(jobs); - return rc; -} - -static int run_count(const char *argv0, int argc, char **argv) { - const char *targets_path = NULL; - const char *samples_path = NULL; - const char *out_path = NULL; - const char *assignments_path = NULL; - const char *summary_path = NULL; - const char *report_path = NULL; - const char *report_audit_dir = NULL; - const char *report_unmatched_path = NULL; - const char *ambiguous_path = NULL; - const char *unmatched_path = NULL; - const char *sample_qc_path = NULL; - const char *target_counts_long_path = NULL; - const int crispr_mode = strcmp(argv[1], "crispr-count") == 0; - const char *format = crispr_mode ? "mageck" : "dotmatch"; - const char *ambiguous_policy = "discard"; - ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; - count_metric metric = COUNT_METRIC_LEVENSHTEIN; - hamming_index_strategy hamming_strategy = HAMMING_INDEX_AUTO; - size_t target_start = 0; - size_t target_len = 0; - size_t indel_window = 0; - size_t auto_offset = 0; - size_t auto_offset_sample = 1000; - offset_mod…844 tokens truncated… usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--auto-offset-sample") == 0 && i < argc) { - if (parse_size_value(argv[i++], &auto_offset_sample) != 0 || auto_offset_sample == 0) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--offset-mode") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "best") == 0) { - offsets_mode = OFFSET_MODE_BEST; - } else if (strcmp(value, "multi") == 0) { - offsets_mode = OFFSET_MODE_MULTI; - } else { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--offset-min-fraction") == 0 && i < argc) { - if (parse_double_value(argv[i++], &offset_min_fraction) != 0 || - offset_min_fraction < 0.0 || offset_min_fraction > 1.0) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--backend") == 0 && i < argc) { - if (parse_count_backend_mode(argv[i++], &backend_mode) != 0) { - fprintf(stderr, "--backend must be auto, cpu, or gpu-metal-experimental\n"); - goto fail_args; - } - } else if (strcmp(arg, "--progress") == 0) { - show_progress = 1; - } else if (strcmp(arg, "--no-progress") == 0) { - show_progress = 0; - } else if (strcmp(arg, "--progress-interval") == 0 && i < argc) { - if (parse_size_value(argv[i++], &progress_interval_reads) != 0 || progress_interval_reads == 0) { - fprintf(stderr, "--progress-interval must be a positive integer\n"); - goto fail_args; - } - } else if (strcmp(arg, "--metal-validate") == 0) { - metal_validate = 1; - } else if (strcmp(arg, "--threads") == 0 && i < argc) { - if (parse_size_value(argv[i++], &threads) != 0) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--max-correction-qual") == 0 && i < argc) { - if (parse_int_value(argv[i++], &max_correction_qual) != 0 || - max_correction_qual < 0 || max_correction_qual > 93) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--out") == 0 && i < argc) { - out_path = argv[i++]; - } else if (strcmp(arg, "--assignments") == 0 && i < argc) { - assignments_path = argv[i++]; - } else if (strcmp(arg, "--summary") == 0 && i < argc) { - summary_path = argv[i++]; - } else if (strcmp(arg, "--report") == 0 && i < argc) { - report_path = argv[i++]; - } else if (strcmp(arg, "--report-audit-dir") == 0 && i < argc) { - report_audit_dir = argv[i++]; - } else if (strcmp(arg, "--report-unmatched") == 0 && i < argc) { - report_unmatched_path = argv[i++]; - } else if (strcmp(arg, "--qc") == 0 && i < argc) { - sample_qc_path = argv[i++]; - } else if (strcmp(arg, "--sample-qc") == 0 && i < argc) { - sample_qc_path = argv[i++]; - } else if (strcmp(arg, "--target-counts-long") == 0 && i < argc) { - target_counts_long_path = argv[i++]; - } else if (strcmp(arg, "--ambiguous-out") == 0 && i < argc) { - ambiguous_path = argv[i++]; - } else if (strcmp(arg, "--unmatched-out") == 0 && i < argc) { - unmatched_path = argv[i++]; - } else if (strcmp(arg, "--ambiguous") == 0 && i < argc) { - ambiguous_policy = argv[i++]; - if (strcmp(ambiguous_policy, "discard") != 0 && strcmp(ambiguous_policy, "report") != 0) { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "best") == 0) { - assignment_policy = AMBIGUITY_POLICY_BEST; - } else if (strcmp(value, "radius") == 0) { - assignment_policy = AMBIGUITY_POLICY_RADIUS; - } else { - usage(argv0); - goto fail_args; - } - } else if (strcmp(arg, "--format") == 0 && i < argc) { - format = argv[i++]; - if (strcmp(format, "dotmatch") != 0 && strcmp(format, "mageck") != 0) { - usage(argv0); - goto fail_args; - } - } else { - usage(argv0); - goto fail_args; - } - } - - if (samples_path != NULL && read_samples_file(samples_path, &labels, &reads) != 0) { - fprintf(stderr, "failed to read samples file\n"); - goto fail_args; - } - if (targets_path == NULL || reads.count == 0 || out_path == NULL || target_len == 0 || k < 0) { - usage(argv0); - goto fail_args; - } - if (crispr_mode && sample_qc_path == NULL && - derive_output_sibling_path(out_path, "sample_qc.tsv", default_sample_qc_path, - sizeof(default_sample_qc_path)) == 0) { - sample_qc_path = default_sample_qc_path; - } - if (auto_offset > MAX_AUTO_OFFSET) { - fprintf(stderr, "--auto-offset must be <= %d\n", MAX_AUTO_OFFSET); - goto fail_args; - } - if (metric == COUNT_METRIC_HAMMING && indel_window != 0) { - fprintf(stderr, "--indel-window is only valid with --metric levenshtein\n"); - goto fail_args; - } - if (metric == COUNT_METRIC_LEVENSHTEIN && k > 2) { - fprintf(stderr, "--metric levenshtein supports --k up to 2\n"); - goto fail_args; - } - if (indel_window != 0 && k != 1) { - fprintf(stderr, "--indel-window requires --k 1\n"); - goto fail_args; - } - if (labels.count == 0) { - for (size_t i = 0; i < reads.count; ++i) { - if (push_string(&labels, path_basename(reads.items[i])) != 0) { - fprintf(stderr, "out of memory\n"); - goto fail_args; - } - } - } - if (labels.count != reads.count) { - fprintf(stderr, "--sample-label count must match --reads count\n"); - goto fail_args; - } - if (validate_unique_sample_labels(&labels, crispr_mode ? "--samples sample" : "--sample-label") != 0) { - goto fail_args; - } - if (threads > 1 && (assignments_path != NULL || ambiguous_path != NULL || unmatched_path != NULL)) { - fprintf(stderr, "--threads > 1 is not supported with row-level diagnostic outputs\n"); - goto fail_args; - } - if (threads == 0) { - size_t auto_t = get_cpu_count(); - if (auto_t > 1 && (assignments_path != NULL || ambiguous_path != NULL || unmatched_path != NULL)) { - threads = 1; /* fall back for safe ordered diagnostic outputs */ - } else { - threads = auto_t; - } - } - int count_only = assignments_path == NULL && ambiguous_path == NULL && unmatched_path == NULL; - - seq_table targets = {0}; - qdaln_index *index = NULL; - hamming_lookup hlookup = {0}; - hamming_lookup offset_lookup = {0}; - levenshtein1_lookup levlookup = {0}; - const char **target_ptrs = NULL; - size_t *target_lens = NULL; - unsigned char *ambiguous_nearby = NULL; - unsigned long long *counts = NULL; - count_stats *stats_by_sample = NULL; - offset_list *selected_offsets = NULL; - FILE *out = NULL; - FILE *assignments = NULL; - FILE *ambiguous_out = NULL; - FILE *unmatched_out = NULL; - int rc = 1; - double run_start_seconds = seconds_now(); - double target_index_seconds = 0.0; - double offset_detection_seconds = 0.0; - double hamming_precompute_seconds = 0.0; - double counting_seconds = 0.0; - const char *offset_detection_strategy = auto_offset == 0 ? "none" : "prepass"; - const char *count_engine = "generic_indexed"; - size_t effective_read_threads = threads; - const char *backend_effective = "cpu"; - int metal_hamming_counts = 0; - uint64_t *metal_target_codes = NULL; - - double phase_start_seconds = seconds_now(); - if (read_target_table(targets_path, &targets) != 0) { - fprintf(stderr, "failed to read targets\n"); - goto done; - } - int target_id_check = validate_unique_seq_ids(&targets, crispr_mode ? "guide" : "target"); - if (target_id_check != 0) { - if (target_id_check == -1) fprintf(stderr, "out of memory\n"); - goto done; - } - if (metric == COUNT_METRIC_HAMMING && !all_targets_have_length(&targets, target_len)) { - fprintf(stderr, "--metric hamming requires every target to have --target-length bases\n"); - goto done; - } - target_index_seconds = seconds_now() - phase_start_seconds; - - int hamming_lookup_eligible = hamming_lookup_counts_eligible( - count_only, max_correction_qual, metric, indel_window, k, target_len, hamming_strategy); - int direct_hamming_counts = hamming_direct_worker_eligible(hamming_lookup_eligible, assignment_policy, k); - int may_use_metal = backend_mode == COUNT_BACKEND_METAL && hamming_lookup_eligible && qdmetal_available() && - (k == 0 || assignment_policy == AMBIGUITY_POLICY_BEST); - if (direct_hamming_counts) { - phase_start_seconds = seconds_now(); - int use_mismatch_precompute_now = k == 1 && - (hamming_strategy == HAMMING_INDEX_PRECOMPUTE || - (hamming_strategy == HAMMING_INDEX_AUTO && auto_offset != 0 && - offsets_mode == OFFSET_MODE_MULTI)); - int lookup_rc = 0; - if (k == 0) { - lookup_rc = build_hamming_exact_lookup(&targets, target_len, &hlookup); - } else if (use_mismatch_precompute_now) { - lookup_rc = build_hamming_lookup(&targets, target_len, &hlookup); - } else { - lookup_rc = build_hamming_seed_lookup(&targets, target_len, &hlookup); - } - if (lookup_rc != 0) { - fprintf(stderr, "failed to build Hamming lookup\n"); - goto done; - } - hamming_precompute_seconds = seconds_now() - phase_start_seconds; - if (hlookup.ready) { - count_engine = "hamming_lookup_direct"; - } else { - direct_hamming_counts = 0; - } - } - - int need_general_index = !direct_hamming_counts || strcmp(format, "dotmatch") == 0; - if (need_general_index) { - phase_start_seconds = seconds_now(); - if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - index = qdaln_index_build(target_ptrs, target_lens, targets.count); - if (index == NULL) { - fprintf(stderr, "failed to build target index\n"); - goto done; - } - target_index_seconds += seconds_now() - phase_start_seconds; - } - - size_t sample_target_slots = 0; - size_t total_slots = 0; - if (checked_mul_size(reads.count, targets.count, &sample_target_slots) != 0 || - checked_mul_size(sample_target_slots, 5, &total_slots) != 0) { - fprintf(stderr, "count matrix is too large\n"); - goto done; - } - counts = (unsigned long long *)calloc(alloc_count_or_one(total_slots), sizeof(unsigned long long)); - stats_by_sample = (count_stats *)calloc(alloc_count_or_one(reads.count), sizeof(count_stats)); - ambiguous_nearby = (unsigned char *)calloc(alloc_count_or_one(targets.count), sizeof(unsigned char)); - selected_offsets = (offset_list *)calloc(alloc_count_or_one(reads.count), sizeof(offset_list)); - if (counts == NULL || stats_by_sample == NULL || ambiguous_nearby == NULL || selected_offsets == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t sample = 0; sample < reads.count; ++sample) { - if (push_offset_unique(&selected_offsets[sample], target_start) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - } - - if (strcmp(format, "dotmatch") == 0) { - for (size_t i = 0; i < targets.count; ++i) { - qdaln_match_result r; - qdaln_index_stats s; - const char *seq_ptr = targets.records[i].seq; - size_t seq_len = targets.records[i].len; - int assign_rc = metric == COUNT_METRIC_HAMMING - ? qdaln_index_assign_hamming_stats(index, &seq_ptr, &seq_len, 1, k, &r, &s) - : qdaln_index_assign_stats(index, &seq_ptr, &seq_len, 1, k, &r, &s); - if (assign_rc != 0) { - fprintf(stderr, "target ambiguity check failed\n"); - goto done; - } - ambiguous_nearby[i] = r.match_count > 1 ? 1 : 0; - } - } - - if (assignments_path != NULL) { - assignments = open_output_file(assignments_path); - if (assignments == NULL) { - fprintf(stderr, "failed to open assignments output\n"); - goto done; - } - fprintf(assignments, "sample\tread_id\tobserved_seq\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\tcorrection\n"); - } - if (ambiguous_path != NULL) { - ambiguous_out = open_output_file(ambiguous_path); - if (ambiguous_out == NULL) { - fprintf(stderr, "failed to open ambiguous output\n"); - goto done; - } - fprintf(ambiguous_out, "sample\tread_id\tobserved_seq\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\tcorrection\n"); - } - if (unmatched_path != NULL) { - unmatched_out = open_output_file(unmatched_path); - if (unmatched_out == NULL) { - fprintf(stderr, "failed to open unmatched output\n"); - goto done; - } - fprintf(unmatched_out, "sample\tread_id\tobserved_seq\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\tcorrection\n"); - } - - int metal_blocks_fused_offset = may_use_metal && offsets_mode != OFFSET_MODE_MULTI; - int fused_offset_detection = direct_hamming_counts && auto_offset != 0 && !metal_blocks_fused_offset; - if (fused_offset_detection) { - offset_detection_strategy = "fused"; - } - - if (auto_offset != 0 && !fused_offset_detection) { - phase_start_seconds = seconds_now(); - const hamming_lookup *offset_lookup_ptr = hlookup.ready ? &hlookup : NULL; - if (metric == COUNT_METRIC_HAMMING && offset_lookup_ptr == NULL) { - int lookup_rc = build_hamming_exact_lookup(&targets, target_len, &offset_lookup); - if (lookup_rc != 0) { - fprintf(stderr, "failed to build offset detection lookup\n"); - goto done; - } - if (offset_lookup.ready) offset_lookup_ptr = &offset_lookup; - } - if (detect_offsets_for_samples(index, offset_lookup_ptr, &reads, target_start, target_len, auto_offset, - auto_offset_sample, offsets_mode, offset_min_fraction, selected_offsets, - threads) != 0) { - fprintf(stderr, "automatic offset detection failed\n"); - goto done; - } - offset_detection_seconds = seconds_now() - phase_start_seconds; - free_hamming_lookup(&offset_lookup); - } - - size_t max_selected_offsets = 0; - for (size_t sample = 0; sample < reads.count; ++sample) { - if (selected_offsets[sample].count > max_selected_offsets) max_selected_offsets = selected_offsets[sample].count; - } - int direct_levenshtein_counts = levenshtein1_lookup_counts_eligible( - count_only, max_correction_qual, metric, indel_window, k, target_len, max_selected_offsets, - assignments, ambiguous_out, unmatched_out, assignment_policy); - if (direct_levenshtein_counts) { - phase_start_seconds = seconds_now(); - int lookup_rc = build_levenshtein1_lookup(&targets, target_len, &levlookup); - if (lookup_rc != 0) { - fprintf(stderr, "failed to build Levenshtein k=1 lookup\n"); - goto done; - } - if (levlookup.ready) { - count_engine = "levenshtein_k1_lookup_direct"; - target_index_seconds += seconds_now() - phase_start_seconds; - } else { - direct_levenshtein_counts = 0; - } - } - if (metal_hamming_count_eligible(backend_mode, hamming_lookup_eligible, assignment_policy, k, - max_selected_offsets, fused_offset_detection)) { - int pack_rc = build_packed_target_codes(&targets, target_len, &metal_target_codes); - if (pack_rc == 1) { - metal_hamming_counts = 1; - direct_hamming_counts = 0; - count_engine = metal_count_engine_name(targets.count); - backend_effective = "gpu-metal-experimental"; - } else if (pack_rc < 0) { - goto done; - } else if (backend_mode == COUNT_BACKEND_METAL) { - fprintf(stderr, - "Metal backend requires packable A/C/G/T targets, count-only output, single offset, and " - "best-distance policy for k=1\n"); - goto done; - } - } else if (backend_mode == COUNT_BACKEND_METAL) { - fprintf(stderr, - "Metal backend unavailable for this workload; requires Darwin Metal, --metric hamming, k 0|1, " - "count-only output, and single offset"); - if (k == 1) fprintf(stderr, " with --ambiguity-policy best"); - fprintf(stderr, "\n"); - goto done; - } else if (direct_hamming_counts && !fused_offset_detection && max_selected_offsets <= 1) { - count_engine = "hamming_lookup_direct_single_offset"; - } - int use_precomputed_hamming = metric == COUNT_METRIC_HAMMING && k == 1 && - (hamming_strategy == HAMMING_INDEX_PRECOMPUTE || - (hamming_strategy == HAMMING_INDEX_AUTO && max_selected_offsets > 1)); - if (use_precomputed_hamming && (!hlookup.ready || hlookup.mismatch == NULL)) { - phase_start_seconds = seconds_now(); - free_hamming_lookup(&hlookup); - int lookup_rc = build_hamming_lookup(&targets, target_len, &hlookup); - if (lookup_rc != 0) { - fprintf(stderr, "failed to build Hamming lookup\n"); - goto done; - } - hamming_precompute_seconds = seconds_now() - phase_start_seconds; - } - - if (show_progress) { - progress_by_sample = (count_progress *)calloc(reads.count, sizeof(count_progress)); - if (progress_by_sample == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t sample = 0; sample < reads.count; ++sample) { - count_progress_init(&progress_by_sample[sample], labels.items[sample], progress_interval_reads); - } - } - - phase_start_seconds = seconds_now(); - size_t sample_threads = threads; - if ((direct_hamming_counts || metal_hamming_counts) && reads.count == 1 && threads > 1) { - effective_read_threads = threads; - sample_threads = 1; - } else if (count_only && reads.count == 1 && threads > 1) { - effective_read_threads = threads; - sample_threads = 1; - } else if (sample_threads > reads.count) { - sample_threads = reads.count; - } - count_samples_args sample_args = { - index, &hlookup, &levlookup, &targets, target_ptrs, target_lens, &reads, &labels, selected_offsets, target_len, k, metric, - indel_window, counts, stats_by_sample, assignments, ambiguous_out, unmatched_out, ambiguous_policy, - assignment_policy, direct_hamming_counts, metal_hamming_counts, metal_target_codes, fused_offset_detection, - target_start, auto_offset, auto_offset_sample, offsets_mode, offset_min_fraction, effective_read_threads, - max_correction_qual, sample_threads, progress_by_sample}; - if (run_count_samples_phase(&sample_args) != 0) goto done; - - if (metal_validate && metal_hamming_counts) { - size_t count_slots = total_slots == 0 ? 1 : total_slots; - size_t sample_slots = reads.count == 0 ? 1 : reads.count; - unsigned long long *metal_counts_snapshot = - (unsigned long long *)calloc(count_slots, sizeof(unsigned long long)); - count_stats *metal_stats_snapshot = (count_stats *)calloc(sample_slots, sizeof(count_stats)); - if (metal_counts_snapshot == NULL || metal_stats_snapshot == NULL) { - free(metal_counts_snapshot); - free(metal_stats_snapshot); - fprintf(stderr, "out of memory\n"); - goto done; - } - memcpy(metal_counts_snapshot, counts, total_slots * sizeof(unsigned long long)); - memcpy(metal_stats_snapshot, stats_by_sample, reads.count * sizeof(count_stats)); - memset(counts, 0, count_slots * sizeof(unsigned long long)); - memset(stats_by_sample, 0, sample_slots * sizeof(count_stats)); - - int cpu_direct = hamming_direct_worker_eligible(hamming_lookup_eligible, assignment_policy, k) && hlookup.ready; - if (!cpu_direct) { - fprintf(stderr, "Metal validation requires a CPU Hamming direct lookup for this workload\n"); - free(metal_counts_snapshot); - free(metal_stats_snapshot); - goto done; - } - count_samples_args cpu_args = sample_args; - cpu_args.direct_hamming_counts = 1; - cpu_args.metal_hamming_counts = 0; - cpu_args.metal_target_codes = NULL; - cpu_args.assignments = NULL; - cpu_args.ambiguous_out = NULL; - cpu_args.unmatched_out = NULL; - cpu_args.progress_by_sample = NULL; - if (run_count_samples_phase(&cpu_args) != 0) { - free(metal_counts_snapshot); - free(metal_stats_snapshot); - goto done; - } - - size_t diff_guides = 0; - long long delta_reads = 0; - char example[256]; - example[0] = '\0'; - if (validate_metal_counts_against_cpu(metal_counts_snapshot, counts, reads.count, targets.count, &diff_guides, - &delta_reads, example, sizeof(example)) != 0) { - metal_validation_status = "failed"; - fprintf(stderr, - "dotmatch: Metal validation failed: %zu guides differ across samples (net read delta %lld); %s\n", - diff_guides, delta_reads, example[0] == '\0' ? "no example" : example); - fprintf(stderr, "dotmatch: rerun with --backend cpu for authoritative counts\n"); - free(metal_counts_snapshot); - free(metal_stats_snapshot); - goto done; - } - metal_validation_status = "passed"; - fprintf(stderr, "dotmatch: Metal validation passed (%zu guides, CPU authority check)\n", targets.count); - memcpy(counts, metal_counts_snapshot, total_slots * sizeof(unsigned long long)); - memcpy(stats_by_sample, metal_stats_snapshot, reads.count * sizeof(count_stats)); - free(metal_counts_snapshot); - free(metal_stats_snapshot); - } else if (metal_validate && backend_mode == COUNT_BACKEND_METAL) { - fprintf(stderr, "dotmatch: --metal-validate requires an active Metal counting backend for this workload\n"); - goto done; - } - - counting_seconds = seconds_now() - phase_start_seconds; - if (show_progress && progress_by_sample != NULL) { - for (size_t sample = 0; sample < reads.count; ++sample) { - count_progress_finish(&progress_by_sample[sample]); - } - } - - out = open_output_file(out_path); - if (out == NULL) { - fprintf(stderr, "failed to open count output\n"); - goto done; - } - if (strcmp(format, "mageck") == 0) { - fprintf(out, "sgRNA\tGene"); - for (size_t sample = 0; sample < reads.count; ++sample) fprintf(out, "\t%s", labels.items[sample]); - fprintf(out, "\n"); - for (size_t t = 0; t < targets.count; ++t) { - fprintf(out, "%s\t%s", targets.records[t].id, targets.records[t].gene); - for (size_t sample = 0; sample < reads.count; ++sample) { - unsigned long long total = 0; - for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * targets.count + t) * 5) + kind]; - fprintf(out, "\t%llu", total); - } - fprintf(out, "\n"); - } - } else { - fprintf(out, "target_id\ttarget_seq\tgene\tambiguous_nearby"); - for (size_t sample = 0; sample < reads.count; ++sample) { - fprintf(out, "\t%s_count_exact\t%s_count_corrected_substitution\t%s_count_corrected_insertion\t%s_count_corrected_deletion\t%s_count_corrected_other\t%s_count_total", - labels.items[sample], labels.items[sample], labels.items[sample], labels.items[sample], labels.items[sample], labels.items[sample]); - } - fprintf(out, "\n"); - for (size_t t = 0; t < targets.count; ++t) { - fprintf(out, "%s\t%s\t%s\t%d", targets.records[t].id, targets.records[t].seq, targets.records[t].gene, (int)ambiguous_nearby[t]); - for (size_t sample = 0; sample < reads.count; ++sample) { - unsigned long long exact = counts[((sample * targets.count + t) * 5) + 0]; - unsigned long long sub = counts[((sample * targets.count + t) * 5) + 1]; - unsigned long long ins = counts[((sample * targets.count + t) * 5) + 2]; - unsigned long long del = counts[((sample * targets.count + t) * 5) + 3]; - unsigned long long other = counts[((sample * targets.count + t) * 5) + 4]; - fprintf(out, "\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu", exact, sub, ins, del, other, exact + sub + ins + del + other); - } - fprintf(out, "\n"); - } - } - - if (target_counts_long_path != NULL) { - FILE *long_out = open_output_file(target_counts_long_path); - if (long_out == NULL) { - fprintf(stderr, "failed to open long target-count output\n"); - goto done; - } - fprintf(long_out, "sample_id\ttarget_id\tgroup\tsequence\texact_count\tk1_sub_count\tk1_ins_count\tk1_del_count\tother_count\ttotal_count\tambiguous_nearby\n"); - for (size_t sample = 0; sample < reads.count; ++sample) { - for (size_t t = 0; t < targets.count; ++t) { - unsigned long long exact = counts[((sample * targets.count + t) * 5) + 0]; - unsigned long long sub = counts[((sample * targets.count + t) * 5) + 1]; - unsigned long long ins = counts[((sample * targets.count + t) * 5) + 2]; - unsigned long long del = counts[((sample * targets.count + t) * 5) + 3]; - unsigned long long other = counts[((sample * targets.count + t) * 5) + 4]; - fprintf(long_out, "%s\t%s\t%s\t%s\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%d\n", - labels.items[sample], targets.records[t].id, targets.records[t].gene, targets.records[t].seq, - exact, sub, ins, del, other, exact + sub + ins + del + other, (int)ambiguous_nearby[t]); - } - } - fclose(long_out); - } - - if (sample_qc_path != NULL) { - FILE *qc = open_output_file(sample_qc_path); - if (qc == NULL) { - fprintf(stderr, "failed to open sample QC output\n"); - goto done; - } - fprintf(qc, "sample_id\tfastq\ttotal_reads\tvalid_extracted_reads\tassigned_reads\texact_reads\tk1_rescued_reads\tk1_sub_reads\tk1_ins_reads\tk1_del_reads\tambiguous_reads\tno_match_reads\tinvalid_reads\tassignment_rate\texact_rate\trescue_rate\tambiguous_rate\tno_match_rate\ttargets_observed\tzero_count_targets\tgini_index\ttop_1pct_read_fraction\tcandidates_verified\n"); - for (size_t sample = 0; sample < reads.count; ++sample) { - unsigned long long *target_totals = (unsigned long long *)calloc(targets.count == 0 ? 1 : targets.count, sizeof(unsigned long long)); - if (target_totals == NULL) { - fclose(qc); - fprintf(stderr, "out of memory\n"); - goto done; - } - unsigned long long sub = 0; - unsigned long long ins = 0; - unsigned long long del = 0; - unsigned long long observed_targets = 0; - for (size_t t = 0; t < targets.count; ++t) { - unsigned long long exact = counts[((sample * targets.count + t) * 5) + 0]; - sub += counts[((sample * targets.count + t) * 5) + 1]; - ins += counts[((sample * targets.count + t) * 5) + 2]; - del += counts[((sample * targets.count + t) * 5) + 3]; - target_totals[t] = exact + counts[((sample * targets.count + t) * 5) + 1] + - counts[((sample * targets.count + t) * 5) + 2] + - counts[((sample * targets.count + t) * 5) + 3] + - counts[((sample * targets.count + t) * 5) + 4]; - if (target_totals[t] != 0) ++observed_targets; - } - count_stats *s = &stats_by_sample[sample]; - unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; - double valid_denom = valid == 0 ? 1.0 : (double)valid; - fprintf(qc, "%s\t%s\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%.8f\t%.8f\t%.8f\t%.8f\t%.8f\t%llu\t%llu\t%.8f\t%.8f\t%llu\n", - labels.items[sample], reads.items[sample], s->total, valid, s->unique, s->exact, s->corrected, - sub, ins, del, s->ambiguous, s->unmatched, s->invalid, - (double)s->unique / valid_denom, (double)s->exact / valid_denom, (double)s->corrected / valid_denom, - (double)s->ambiguous / valid_denom, (double)s->unmatched / valid_denom, - observed_targets, (unsigned long long)(targets.count - observed_targets), - gini_from_counts(target_totals, targets.count), - top_fraction_from_counts(target_totals, targets.count, 0.01), - s->candidates_verified); - free(target_totals); - } - fclose(qc); - } - - if (summary_path != NULL) { - FILE *summary = open_output_file(summary_path); - if (summary == NULL) { - fprintf(stderr, "failed to open summary output\n"); - goto done; - } - double total_before_summary_seconds = seconds_now() - run_start_seconds; - fprintf(summary, - "{\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"max_correction_qual\": ", - k, metric_name(metric), ambiguity_policy_name(assignment_policy), qdaln_alphabet_policy()); - if (max_correction_qual >= 0) { - fprintf(summary, "%d", max_correction_qual); - } else { - fprintf(summary, "null"); - } - fprintf(summary, - ",\n \"indel_window\": %zu,\n \"target_start\": %zu,\n \"auto_offset\": %zu,\n \"offset_mode\": \"%s\",\n \"offset_min_fraction\": %.8f,\n \"offset_detection_strategy\": \"%s\",\n \"backend_requested\": \"%s\",\n \"backend_effective\": \"%s\",\n \"metal_device\": ", - indel_window, target_start, auto_offset, offset_mode_name(offsets_mode), offset_min_fraction, - offset_detection_strategy, count_backend_mode_name(backend_mode), backend_effective); - if (metal_hamming_counts && qdmetal_device_name() != NULL) { - fprintf(summary, "\"%s\"", qdmetal_device_name()); - } else { - fprintf(summary, "null"); - } - fprintf(summary, ",\n \"metal_validation\": "); - if (metal_validation_status != NULL) { - fprintf(summary, "\"%s\"", metal_validation_status); - } else { - fprintf(summary, "null"); - } - fprintf(summary, - ",\n \"count_engine\": \"%s\",\n \"hamming_index\": \"%s\",\n \"target_length\": %zu,\n \"n_targets\": %zu,\n \"read_threads\": %zu,\n \"phase_seconds\": {\"target_index\": %.6f, \"offset_detection\": %.6f, \"hamming_precompute\": %.6f, \"counting\": %.6f, \"total_before_summary\": %.6f},\n \"samples\": [\n", - count_engine, hamming_lookup_kind(&hlookup), target_len, targets.count, effective_read_threads, - target_index_seconds, offset_detection_seconds, hamming_precompute_seconds, counting_seconds, - total_before_summary_seconds); - for (size_t sample = 0; sample < reads.count; ++sample) { - count_stats *s = &stats_by_sample[sample]; - unsigned long long covered = 0; - unsigned long long top_count = 0; - size_t top_target = 0; - for (size_t t = 0; t < targets.count; ++t) { - unsigned long long total = 0; - for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * targets.count + t) * 5) + kind]; - if (total != 0) ++covered; - if (total > top_count) { - top_count = total; - top_target = t; - } - } - double rescued_percent = s->total == 0 ? 0.0 : 100.0 * (double)s->corrected / (double)s->total; - double ambiguous_percent = s->total == 0 ? 0.0 : 100.0 * (double)s->ambiguous / (double)s->total; - double unmatched_percent = s->total == 0 ? 0.0 : 100.0 * (double)s->unmatched / (double)s->total; - fprintf(summary, - " {\"sample\": \"%s\", \"selected_target_start\": %zu, \"selected_target_starts\": [", - labels.items[sample], first_selected_offset(&selected_offsets[sample], target_start)); - for (size_t oi = 0; oi < selected_offsets[sample].count; ++oi) { - if (oi != 0) fprintf(summary, ", "); - fprintf(summary, "%zu", selected_offsets[sample].items[oi]); - } - fprintf(summary, - "], \"total_reads\": %llu, \"assigned_unique\": %llu, \"assigned_exact\": %llu, \"assigned_corrected\": %llu, \"k1_rescued_reads\": %llu, \"percent_rescued_by_k1\": %.6f, \"ambiguous\": %llu, \"percent_ambiguous\": %.6f, \"unmatched\": %llu, \"percent_unmatched\": %.6f, \"invalid\": %llu, \"library_covered_targets\": %llu, \"library_coverage_fraction\": %.6f, \"top_target_id\": \"%s\", \"top_target_count\": %llu, \"candidates_considered\": %llu, \"candidates_verified\": %llu}%s\n", - s->total, s->unique, s->exact, s->corrected, - s->corrected, rescued_percent, s->ambiguous, ambiguous_percent, s->unmatched, unmatched_percent, - s->invalid, - covered, targets.count == 0 ? 0.0 : (double)covered / (double)targets.count, - targets.count == 0 ? "" : targets.records[top_target].id, top_count, s->candidates_considered, - s->candidates_verified, sample + 1 == reads.count ? "" : ","); - } - fprintf(summary, " ]\n}\n"); - fclose(summary); - } - if (report_path != NULL) { - if (write_count_html_report(report_path, &targets, &reads, &labels, counts, stats_by_sample, selected_offsets, - k, metric, assignment_policy, target_len, report_audit_dir, - report_unmatched_path) != 0) { - fprintf(stderr, "failed to write HTML report\n"); - goto done; - } - } - { - sample_qc_metrics *qc_metrics = (sample_qc_metrics *)calloc(reads.count, sizeof(sample_qc_metrics)); - if (qc_metrics != NULL) { - int metrics_ok = 1; - for (size_t sample = 0; sample < reads.count; ++sample) { - if (compute_sample_qc_metrics(&targets, counts, sample, &stats_by_sample[sample], - &qc_metrics[sample]) != 0) { - metrics_ok = 0; - break; - } - } - if (metrics_ok) { - int enough_reads = 0; - for (size_t sample = 0; sample < reads.count; ++sample) { - if (stats_by_sample[sample].total >= 1000) enough_reads = 1; - } - if (enough_reads || crispr_mode) emit_sample_qc_review_warnings(&labels, qc_metrics, reads.count); - } - free(qc_metrics); - } - } - rc = 0; - -done: - if (progress_by_sample != NULL) { - for (size_t sample = 0; sample < reads.count; ++sample) count_progress_fini(&progress_by_sample[sample]); - free(progress_by_sample); - } - if (out != NULL) fclose(out); - if (assignments != NULL) fclose(assignments); - if (ambiguous_out != NULL) fclose(ambiguous_out); - if (unmatched_out != NULL) fclose(unmatched_out); - qdaln_index_free(index); - free_hamming_lookup(&hlookup); - free_hamming_lookup(&offset_lookup); - free_levenshtein1_lookup(&levlookup); - free(metal_target_codes); - free(target_ptrs); - free(target_lens); - free(ambiguous_nearby); - free(counts); - free(stats_by_sample); - if (selected_offsets != NULL) { - for (size_t sample = 0; sample < reads.count; ++sample) free_offset_list(&selected_offsets[sample]); - } - free(selected_offsets); - free_table(&targets); - free_string_list(&reads); - free_string_list(&labels); - return rc; - -fail_args: - free_string_list(&reads); - free_string_list(&labels); - return 2; -} - -static int string_list_contains_exact(const string_list *list, const char *s) { - if (list == NULL || s == NULL) return 0; - for (size_t i = 0; i < list->count; ++i) { - if (strcmp(list->items[i], s) == 0) return 1; - } - return 0; -} - -static int read_first_column_values(const char *path, string_list *values) { - if (path == NULL) return 0; - fastq_reader reader = {0}; - if (fastq_reader_open(&reader, path) != 0) return -1; - char buf[8192]; - size_t len = 0; - int rc = 0; - while ((rc = fastq_getline_len(&reader, buf, sizeof(buf), &len)) > 0) { - (void)len; - trim_line(buf); - if (buf[0] == '\0' || buf[0] == '#') continue; - char *tab = strchr(buf, '\t'); - if (tab != NULL) *tab = '\0'; - if (push_string(values, buf) != 0) { - fastq_reader_close(&reader); - return -1; - } - } - fastq_reader_close(&reader); - return rc < 0 ? -1 : 0; -} - -static int guide_counter_push_sample_name(string_list *labels, const char *path, size_t idx) { - const char *base = path_basename(path); - char fallback[32]; - if (base == NULL || base[0] == '\0') { - int n = snprintf(fallback, sizeof(fallback), "s%zu", idx + 1); - if (n < 0 || (size_t)n >= sizeof(fallback)) return -1; - return push_string(labels, fallback); - } - char *name = xstrndup(base, strlen(base)); - if (name == NULL) return -1; - if (ends_with(name, ".gz")) name[strlen(name) - 3] = '\0'; - if (ends_with(name, ".fastq")) { - name[strlen(name) - 6] = '\0'; - } else if (ends_with(name, ".fq")) { - name[strlen(name) - 3] = '\0'; - } - int rc = push_string(labels, name); - free(name); - return rc; -} - -static const char *guide_counter_type_for_target(const seq_record *target, const string_list *essential_genes, - const string_list *nonessential_genes, - const string_list *control_guides, - regex_t *control_re) { - if (string_list_contains_exact(essential_genes, target->gene)) return "Essential"; - if (string_list_contains_exact(nonessential_genes, target->gene)) return "Nonessential"; - if (string_list_contains_exact(control_guides, target->id)) return "Control"; - if (control_re != NULL && - (regexec(control_re, target->id, 0, NULL, 0) == 0 || - regexec(control_re, target->gene, 0, NULL, 0) == 0)) { - return "Control"; - } - return "Other"; -} - -static int parse_ull_value(const char *s, unsigned long long *out) { - if (s == NULL || s[0] == '-' || s[0] == '\0') return -1; - char *end = NULL; - errno = 0; - unsigned long long v = strtoull(s, &end, 10); - if (errno == ERANGE || end == s || *end != '\0') return -1; - *out = v; - return 0; -} - -static double round_positive_dp(double value, int places) { - double factor = 1.0; - for (int i = 0; i < places; ++i) factor *= 10.0; - unsigned long long scaled = (unsigned long long)(value * factor + 0.5); - return (double)scaled / factor; -} - -static int guide_counter_write_outputs(const char *output_prefix, const char *tmp_counts_path, - const char *tmp_qc_path, const seq_table *targets, - const string_list *reads, const string_list *labels, - const string_list *essential_genes, - const string_list *nonessential_genes, - const string_list *control_guides, regex_t *control_re) { - char counts_path[4096]; - char extended_path[4096]; - char stats_path[4096]; - int n = snprintf(counts_path, sizeof(counts_path), "%s.counts.txt", output_prefix); - if (n < 0 || (size_t)n >= sizeof(counts_path)) return -1; - n = snprintf(extended_path, sizeof(extended_path), "%s.extended-counts.txt", output_prefix); - if (n < 0 || (size_t)n >= sizeof(extended_path)) return -1; - n = snprintf(stats_path, sizeof(stats_path), "%s.stats.txt", output_prefix); - if (n < 0 || (size_t)n >= sizeof(stats_path)) return -1; - - size_t matrix_slots = 0; - if (checked_mul_size(targets->count, labels->count, &matrix_slots) != 0) return -1; - unsigned long long *matrix = (unsigned long long *)calloc(alloc_count_or_one(matrix_slots), - sizeof(unsigned long long)); - const char **types = (const char **)calloc(alloc_count_or_one(targets->count), sizeof(const char *)); - if (matrix == NULL || types == NULL) { - free(matrix); - free(types); - return -1; - } - for (size_t t = 0; t < targets->count; ++t) { - types[t] = guide_counter_type_for_target(&targets->records[t], essential_genes, nonessential_genes, - control_guides, control_re); - } - - FILE *in = fopen(tmp_counts_path, "r"); - FILE *counts = open_output_file(counts_path); - FILE *extended = open_output_file(extended_path); - if (in == NULL || counts == NULL || extended == NULL) { - if (in != NULL) fclose(in); - if (counts != NULL) fclose(counts); - if (extended != NULL) fclose(extended); - free(matrix); - free(types); - return -1; - } - - fprintf(counts, "guide\tgene"); - fprintf(extended, "guide\tgene\tguide_type"); - for (size_t sample = 0; sample < labels->count; ++sample) { - fprintf(counts, "\t%s", labels->items[sample]); - fprintf(extended, "\t%s", labels->items[sample]); - } - fprintf(counts, "\n"); - fprintf(extended, "\n"); - - char buf[65536]; - size_t row = 0; - int first = 1; - while (fgets(buf, sizeof(buf), in) != NULL) { - trim_line(buf); - if (first) { - first = 0; - continue; - } - char *fields[1024]; - size_t nf = split_fields(buf, '\t', fields, sizeof(fields) / sizeof(fields[0])); - if (nf < 2 + labels->count || row >= targets->count) { - fclose(in); - fclose(counts); - fclose(extended); - free(matrix); - free(types); - return -1; - } - fprintf(counts, "%s\t%s", targets->records[row].id, targets->records[row].gene); - fprintf(extended, "%s\t%s\t%s", targets->records[row].id, targets->records[row].gene, types[row]); - for (size_t sample = 0; sample < labels->count; ++sample) { - unsigned long long value = 0; - if (parse_ull_value(fields[2 + sample], &value) != 0) { - fclose(in); - fclose(counts); - fclose(extended); - free(matrix); - free(types); - return -1; - } - matrix[row * labels->count + sample] = value; - fprintf(counts, "\t%llu", value); - fprintf(extended, "\t%llu", value); - } - fprintf(counts, "\n"); - fprintf(extended, "\n"); - ++row; - } - int matrix_ok = !ferror(in) && row == targets->count; - fclose(in); - fclose(counts); - fclose(extended); - if (!matrix_ok) { - free(matrix); - free(types); - return -1; - } - - unsigned long long *total_reads = (unsigned long long *)calloc(alloc_count_or_one(labels->count), - sizeof(unsigned long long)); - if (total_reads == NULL) { - free(matrix); - free(types); - return -1; - } - FILE *qc = fopen(tmp_qc_path, "r"); - if (qc == NULL) { - free(total_reads); - free(matrix); - free(types); - return -1; - } - int total_reads_col = -1; - size_t qc_row = 0; - first = 1; - while (fgets(buf, sizeof(buf), qc) != NULL) { - trim_line(buf); - char *fields[64]; - size_t nf = split_fields(buf, '\t', fields, sizeof(fields) / sizeof(fields[0])); - if (first) { - total_reads_col = find_column(fields, nf, "total_reads", NULL, NULL); - first = 0; - continue; - } - if (total_reads_col < 0 || (size_t)total_reads_col >= nf || qc_row >= labels->count || - parse_ull_value(fields[total_reads_col], &total_reads[qc_row]) != 0) { - fclose(qc); - free(total_reads); - free(matrix); - free(types); - return -1; - } - ++qc_row; - } - int qc_ok = !ferror(qc) && qc_row == labels->count; - fclose(qc); - if (!qc_ok) { - free(total_reads); - free(matrix); - free(types); - return -1; - } - - FILE *stats = open_output_file(stats_path); - if (stats == NULL) { - free(total_reads); - free(matrix); - free(types); - return -1; - } - fprintf(stats, "file\tlabel\ttotal_guides\ttotal_reads\tmapped_reads\tfrac_mapped\tmean_reads_per_guide\tmean_reads_essential\tmean_reads_nonessential\tmean_reads_control\tmean_reads_other\tzero_read_guides\n"); - for (size_t sample = 0; sample < labels->count; ++sample) { - unsigned long long mapped = 0; - unsigned long long zero = 0; - double essential_sum = 0.0; - double nonessential_sum = 0.0; - double control_sum = 0.0; - double other_sum = 0.0; - size_t essential_count = 0; - size_t nonessential_count = 0; - size_t control_count = 0; - size_t other_count = 0; - for (size_t t = 0; t < targets->count; ++t) { - unsigned long long value = matrix[t * labels->count + sample]; - mapped += value; - if (value == 0) ++zero; - if (strcmp(types[t], "Essential") == 0) { - essential_sum += (double)value; - ++essential_count; - } else if (strcmp(types[t], "Nonessential") == 0) { - nonessential_sum += (double)value; - ++nonessential_count; - } else if (strcmp(types[t], "Control") == 0) { - control_sum += (double)value; - ++control_count; - } else { - other_sum += (double)value; - ++other_count; - } - } - double total = (double)total_reads[sample]; - double frac = total == 0.0 ? 0.0 : round_positive_dp((double)mapped / total, 4); - double mean_all = targets->count == 0 ? 0.0 : round_positive_dp((double)mapped / (double)targets->count, 2); - double mean_essential = essential_count == 0 ? 0.0 : round_positive_dp(essential_sum / (double)essential_count, 2); - double mean_nonessential = nonessential_count == 0 ? 0.0 : round_positive_dp(nonessential_sum / (double)nonessential_count, 2); - double mean_control = control_count == 0 ? 0.0 : round_positive_dp(control_sum / (double)control_count, 2); - double mean_other = other_count == 0 ? 0.0 : round_positive_dp(other_sum / (double)other_count, 2); - fprintf(stats, "%s\t%s\t%zu\t%llu\t%llu\t%.4f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%llu\n", - reads->items[sample], labels->items[sample], targets->count, total_reads[sample], mapped, - frac, mean_all, mean_essential, mean_nonessential, mean_control, mean_other, zero); - } - fclose(stats); - free(total_reads); - free(matrix); - free(types); - return 0; -} - -static int push_count_arg(string_list *args, const char *s) { - return push_string(args, s); -} - -static int run_guide_counter_compatible(const char *argv0, int argc, char **argv) { - int start = 2; - if (strcmp(argv[1], "guide-counter") == 0) { - if (argc < 3 || strcmp(argv[2], "count") != 0) { - usage(argv0); - return 2; - } - start = 3; - } - - const char *library_path = NULL; - const char *output_prefix = NULL; - const char *essential_path = NULL; - const char *nonessential_path = NULL; - const char *control_guides_path = NULL; - const char *control_pattern = NULL; - size_t offset_sample_size = 100000; - double offset_min_fraction = 0.0025; - int exact_match = 0; - string_list reads = {0}; - string_list labels = {0}; - - int i = start; - while (i < argc) { - const char *arg = argv[i++]; - if ((strcmp(arg, "--input") == 0 || strcmp(arg, "-i") == 0) && i < argc) { - while (i < argc && argv[i][0] != '-') { - if (push_string(&reads, argv[i++]) != 0) goto oom; - } - } else if ((strcmp(arg, "--samples") == 0 || strcmp(arg, "-s") == 0) && i < argc) { - while (i < argc && argv[i][0] != '-') { - if (push_string(&labels, argv[i++]) != 0) goto oom; - } - } else if ((strcmp(arg, "--library") == 0 || strcmp(arg, "-l") == 0) && i < argc) { - library_path = argv[i++]; - } else if ((strcmp(arg, "--output") == 0 || strcmp(arg, "-o") == 0) && i < argc) { - output_prefix = argv[i++]; - } else if ((strcmp(arg, "--essential-genes") == 0 || strcmp(arg, "-e") == 0) && i < argc) { - essential_path = argv[i++]; - } else if ((strcmp(arg, "--nonessential-genes") == 0 || strcmp(arg, "-n") == 0) && i < argc) { - nonessential_path = argv[i++]; - } else if ((strcmp(arg, "--control-guides") == 0 || strcmp(arg, "-c") == 0) && i < argc) { - control_guides_path = argv[i++]; - } else if ((strcmp(arg, "--control-pattern") == 0 || strcmp(arg, "-C") == 0) && i < argc) { - control_pattern = argv[i++]; - } else if ((strcmp(arg, "--offset-sample-size") == 0 || strcmp(arg, "-N") == 0) && i < argc) { - if (parse_size_value(argv[i++], &offset_sample_size) != 0 || offset_sample_size == 0) goto bad_args; - } else if ((strcmp(arg, "--offset-min-fraction") == 0 || strcmp(arg, "-f") == 0) && i < argc) { - if (parse_double_value(argv[i++], &offset_min_fraction) != 0 || - offset_min_fraction < 0.0 || offset_min_fraction > 1.0) { - goto bad_args; - } - } else if (strcmp(arg, "--exact-match") == 0 || strcmp(arg, "-x") == 0) { - exact_match = 1; - } else if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) { - usage(argv0); - free_string_list(&reads); - free_string_list(&labels); - return 0; - } else { - goto bad_args; - } - } - - if (library_path == NULL || output_prefix == NULL || reads.count == 0) goto bad_args; - if (labels.count == 0) { - for (size_t sample = 0; sample < reads.count; ++sample) { - if (guide_counter_push_sample_name(&labels, reads.items[sample], sample) != 0) goto oom; - } - } - if (labels.count != reads.count) { - fprintf(stderr, "--samples count must match --input count\n"); - free_string_list(&reads); - free_string_list(&labels); - return 2; - } - if (validate_unique_sample_labels(&labels, "--samples") != 0) { - free_string_list(&reads); - free_string_list(&labels); - return 2; - } - - seq_table targets = {0}; - string_list essential_genes = {0}; - string_list nonessential_genes = {0}; - string_list control_guides = {0}; - regex_t control_re; - int have_control_re = 0; - string_list count_args = {0}; - int rc = 1; - char target_len_s[32]; - char k_s[8]; - char auto_offset_s[16]; - char offset_sample_s[32]; - char offset_min_s[64]; - char label_csv[8192]; - char tmp_counts_path[4096] = ""; - char tmp_qc_path[4096] = ""; - - if (read_target_table(library_path, &targets) != 0 || targets.count == 0) { - fprintf(stderr, "failed to read guide library\n"); - goto done; - } - int guide_id_check = validate_unique_seq_ids(&targets, "guide"); - if (guide_id_check != 0) { - if (guide_id_check == -1) fprintf(stderr, "out of memory\n"); - goto done; - } - size_t guide_len = targets.records[0].len; - for (size_t t = 1; t < targets.count; ++t) { - if (targets.records[t].len != guide_len) { - fprintf(stderr, "GuideCounter compatibility requires one guide length\n"); - goto done; - } - } - if (read_first_column_values(essential_path, &essential_genes) != 0 || - read_first_column_values(nonessential_path, &nonessential_genes) != 0 || - read_first_column_values(control_guides_path, &control_guides) != 0) { - fprintf(stderr, "failed to read guide annotation files\n"); - goto done; - } - if (control_pattern != NULL) { - if (regcomp(&control_re, control_pattern, REG_EXTENDED | REG_ICASE | REG_NOSUB) != 0) { - fprintf(stderr, "failed to compile --control-pattern\n"); - goto done; - } - have_control_re = 1; - } - - label_csv[0] = '\0'; - for (size_t sample = 0; sample < labels.count; ++sample) { - size_t used = strlen(label_csv); - int n = snprintf(label_csv + used, sizeof(label_csv) - used, "%s%s", - sample == 0 ? "" : ",", labels.items[sample]); - if (n < 0 || (size_t)n >= sizeof(label_csv) - used) { - fprintf(stderr, "too many sample labels for GuideCounter compatibility wrapper\n"); - goto done; - } - } - int n = snprintf(tmp_counts_path, sizeof(tmp_counts_path), "%s.dotmatch-counts.tmp", output_prefix); - if (n < 0 || (size_t)n >= sizeof(tmp_counts_path)) goto done; - n = snprintf(tmp_qc_path, sizeof(tmp_qc_path), "%s.dotmatch-qc.tmp", output_prefix); - if (n < 0 || (size_t)n >= sizeof(tmp_qc_path)) goto done; - snprintf(target_len_s, sizeof(target_len_s), "%zu", guide_len); - snprintf(k_s, sizeof(k_s), "%d", exact_match ? 0 : 1); - snprintf(auto_offset_s, sizeof(auto_offset_s), "%d", 499); - snprintf(offset_sample_s, sizeof(offset_sample_s), "%zu", offset_sample_size); - snprintf(offset_min_s, sizeof(offset_min_s), "%.8g", offset_min_fraction); - - if (push_count_arg(&count_args, argv0) != 0 || - push_count_arg(&count_args, "count") != 0 || - push_count_arg(&count_args, "--targets") != 0 || - push_count_arg(&count_args, library_path) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t sample = 0; sample < reads.count; ++sample) { - if (push_count_arg(&count_args, "--reads") != 0 || - push_count_arg(&count_args, reads.items[sample]) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - } - const char *fixed_args[] = { - "--sample-label", label_csv, - "--target-start", "0", - "--target-length", target_len_s, - "--k", k_s, - "--metric", "hamming", - "--ambiguity-policy", "best", - "--format", "mageck", - "--auto-offset", auto_offset_s, - "--auto-offset-sample", offset_sample_s, - "--offset-mode", "multi", - "--offset-min-fraction", offset_min_s, - "--out", tmp_counts_path, - "--sample-qc", tmp_qc_path - }; - for (size_t ai = 0; ai < sizeof(fixed_args) / sizeof(fixed_args[0]); ++ai) { - if (push_count_arg(&count_args, fixed_args[ai]) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - } - - rc = run_count(argv0, (int)count_args.count, count_args.items); - if (rc != 0) goto done; - if (guide_counter_write_outputs(output_prefix, tmp_counts_path, tmp_qc_path, &targets, &reads, &labels, - &essential_genes, &nonessential_genes, &control_guides, - have_control_re ? &control_re : NULL) != 0) { - fprintf(stderr, "failed to write GuideCounter-compatible outputs\n"); - rc = 1; - goto done; - } - rc = 0; - -done: - if (have_control_re) regfree(&control_re); - unlink(tmp_counts_path); - unlink(tmp_qc_path); - free_string_list(&count_args); - free_string_list(&essential_genes); - free_string_list(&nonessential_genes); - free_string_list(&control_guides); - free_table(&targets); - free_string_list(&reads); - free_string_list(&labels); - return rc; - -oom: - fprintf(stderr, "out of memory\n"); - free_string_list(&reads); - free_string_list(&labels); - return 1; - -bad_args: - usage(argv0); - free_string_list(&reads); - free_string_list(&labels); - return 2; -} - -static int run_fastq_assign(const char *argv0, int argc, char **argv) { - const char *barcodes_path = NULL; - const char *reads_path = NULL; - const char *out_path = NULL; - ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; - size_t barcode_start = 0; - size_t barcode_len = 0; - int k = -1; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if (strcmp(arg, "--barcodes") == 0 && i < argc) { - barcodes_path = argv[i++]; - } else if (strcmp(arg, "--reads") == 0 && i < argc) { - reads_path = argv[i++]; - } else if (strcmp(arg, "--barcode-start") == 0 && i < argc) { - if (parse_size_value(argv[i++], &barcode_start) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--barcode-length") == 0 && i < argc) { - if (parse_size_value(argv[i++], &barcode_len) != 0 || barcode_len == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || (k != 0 && k != 1)) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "best") == 0) { - assignment_policy = AMBIGUITY_POLICY_BEST; - } else if (strcmp(value, "radius") == 0) { - assignment_policy = AMBIGUITY_POLICY_RADIUS; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--out") == 0 && i < argc) { - out_path = argv[i++]; - } else { - usage(argv0); - return 2; - } - } - - if (barcodes_path == NULL || reads_path == NULL || out_path == NULL || barcode_len == 0 || k < 0) { - usage(argv0); - return 2; - } - - seq_table targets = {0}; - fastq_reader reader = {0}; - FILE *out = NULL; - qdaln_index *index = NULL; - int rc = 1; - - if (read_table(barcodes_path, &targets) != 0) { - fprintf(stderr, "failed to read barcode file\n"); - goto done; - } - int barcode_id_check = validate_unique_seq_ids(&targets, "barcode"); - if (barcode_id_check != 0) { - if (barcode_id_check == -1) fprintf(stderr, "out of memory\n"); - goto done; - } - - const char **target_ptrs = (const char **)malloc(targets.count * sizeof(char *)); - size_t *target_lens = (size_t *)malloc(targets.count * sizeof(size_t)); - if (targets.count != 0 && (target_ptrs == NULL || target_lens == NULL)) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t i = 0; i < targets.count; ++i) { - target_ptrs[i] = targets.records[i].seq; - target_lens[i] = targets.records[i].len; - } - index = qdaln_index_build(target_ptrs, target_lens, targets.count); - free(target_ptrs); - free(target_lens); - if (index == NULL) { - fprintf(stderr, "failed to build barcode index\n"); - goto done; - } - - if (fastq_reader_open(&reader, reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - goto done; - } - out = open_output_file(out_path); - if (out == NULL) { - fprintf(stderr, "failed to open output file\n"); - goto done; - } - - fprintf(out, "read_id\tobserved_barcode\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\n"); - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - char read_id[8192]; - char observed[8192]; - int got = 0; - size_t seq_len = 0; - while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - fastq_read_id(header, read_id, sizeof(read_id)); - qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - observed[0] = '\0'; - if (barcode_start <= seq_len && barcode_len <= seq_len - barcode_start && barcode_len < sizeof(observed)) { - memcpy(observed, seq + barcode_start, barcode_len); - observed[barcode_len] = '\0'; - const char *read_ptr = observed; - size_t read_len = barcode_len; - qdaln_index_stats stats; - if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &result, &stats) != 0) { - fprintf(stderr, "FASTQ assignment failed\n"); - goto done; - } - apply_ambiguity_policy(&result, assignment_policy); - } - print_fastq_row(out, &targets, read_id, observed, result); - } - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - goto done; - } - rc = 0; - -done: - if (out != NULL) fclose(out); - fastq_reader_close(&reader); - qdaln_index_free(index); - free_table(&targets); - return rc; -} - -static void sanitize_filename(const char *in, char *out, size_t out_cap) { - size_t j = 0; - if (out_cap == 0) return; - for (size_t i = 0; in[i] != '\0' && j + 1 < out_cap; ++i) { - char c = in[i]; - if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || - c == '_' || c == '-' || c == '.') { - out[j++] = c; - } else { - out[j++] = '_'; - } - } - if (j == 0 && out_cap > 1) out[j++] = '_'; - out[j] = '\0'; -} - -typedef struct sanitized_name_entry { - char *name; - const char *id; -} sanitized_name_entry; - -static int compare_sanitized_name_entry(const void *a, const void *b) { - const sanitized_name_entry *ea = (const sanitized_name_entry *)a; - const sanitized_name_entry *eb = (const sanitized_name_entry *)b; - return strcmp(ea->name, eb->name); -} - -static int validate_unique_sanitized_filenames(const seq_table *targets) { - sanitized_name_entry *entries = (sanitized_name_entry *)calloc(targets->count == 0 ? 1 : targets->count, - sizeof(sanitized_name_entry)); - if (entries == NULL) return -1; - - int rc = 0; - for (size_t i = 0; i < targets->count; ++i) { - char safe_id[512]; - sanitize_filename(targets->records[i].id, safe_id, sizeof(safe_id)); - entries[i].name = xstrndup(safe_id, strlen(safe_id)); - entries[i].id = targets->records[i].id; - if (entries[i].name == NULL) { - rc = -1; - goto done; - } - } - - qsort(entries, targets->count, sizeof(entries[0]), compare_sanitized_name_entry); - for (size_t i = 1; i < targets->count; ++i) { - if (strcmp(entries[i - 1].name, entries[i].name) == 0) { - fprintf(stderr, - "barcode IDs produce the same output filename after sanitization: \"%s\" and \"%s\" -> %s.fastq\n", - entries[i - 1].id, entries[i].id, entries[i].name); - rc = -2; - goto done; - } - } - -done: - for (size_t i = 0; i < targets->count; ++i) free(entries[i].name); - free(entries); - return rc; -} - -static int ensure_dir(const char *path) { - if (mkdir(path, 0777) == 0) return 0; - if (errno == EEXIST) { - struct stat st; - return stat(path, &st) == 0 && S_ISDIR(st.st_mode) ? 0 : -1; - } - return -1; -} - -static int path_join(char *out, size_t out_cap, const char *dir, const char *name) { - int n = snprintf(out, out_cap, "%s/%s", dir, name); - return n < 0 || (size_t)n >= out_cap ? -1 : 0; -} - -static size_t uf_find(size_t *parent, size_t x) { - while (parent[x] != x) { - parent[x] = parent[parent[x]]; - x = parent[x]; - } - return x; -} - -static void uf_union(size_t *parent, size_t a, size_t b) { - size_t ra = uf_find(parent, a); - size_t rb = uf_find(parent, b); - if (ra == rb) return; - if (ra < rb) parent[rb] = ra; - else parent[ra] = rb; -} - -static int string_list_contains(const string_list *list, const char *s) { - for (size_t i = 0; i < list->count; ++i) { - if (strcmp(list->items[i], s) == 0) return 1; - } - return 0; -} - -static int push_unique_string(string_list *list, const char *s) { - if (string_list_contains(list, s)) return 0; - return push_string(list, s); -} - -static int add_k1_variants_for_target(string_list *variants, const char *seq, size_t len) { - static const char dna[] = "ACGT"; - if (push_unique_string(variants, seq) != 0) return -1; - char buf[8192]; - if (len + 2 > sizeof(buf)) return -1; - - for (size_t pos = 0; pos < len; ++pos) { - for (size_t bi = 0; bi < 4; ++bi) { - if (seq[pos] == dna[bi]) continue; - memcpy(buf, seq, len); - buf[pos] = dna[bi]; - buf[len] = '\0'; - if (push_unique_string(variants, buf) != 0) return -1; - } - } - - if (len > 0) { - for (size_t pos = 0; pos < len; ++pos) { - memcpy(buf, seq, pos); - memcpy(buf + pos, seq + pos + 1, len - pos - 1); - buf[len - 1] = '\0'; - if (push_unique_string(variants, buf) != 0) return -1; - } - } - - for (size_t pos = 0; pos <= len; ++pos) { - for (size_t bi = 0; bi < 4; ++bi) { - memcpy(buf, seq, pos); - buf[pos] = dna[bi]; - memcpy(buf + pos + 1, seq + pos, len - pos); - buf[len + 1] = '\0'; - if (push_unique_string(variants, buf) != 0) return -1; - } - } - return 0; -} - -typedef struct variant_record { - char *key; - size_t target; -} variant_record; - -typedef struct variant_record_list { - variant_record *items; - size_t count; - size_t cap; -} variant_record_list; - -static void free_variant_record_list(variant_record_list *list) { - for (size_t i = 0; i < list->count; ++i) free(list->items[i].key); - free(list->items); - list->items = NULL; - list->count = 0; - list->cap = 0; -} - -static int push_variant_record(variant_record_list *list, const char *key, size_t target) { - if (list->count == list->cap) { - size_t next_cap = list->cap == 0 ? 1024 : list->cap * 2; - variant_record *next = (variant_record *)realloc(list->items, next_cap * sizeof(variant_record)); - if (next == NULL) return -1; - list->items = next; - list->cap = next_cap; - } - list->items[list->count].key = xstrndup(key, strlen(key)); - if (list->items[list->count].key == NULL) return -1; - list->items[list->count].target = target; - ++list->count; - return 0; -} - -static int cmp_variant_record(const void *a, const void *b) { - const variant_record *aa = (const variant_record *)a; - const variant_record *bb = (const variant_record *)b; - int c = strcmp(aa->key, bb->key); - if (c != 0) return c; - return aa->target > bb->target ? 1 : (aa->target < bb->target ? -1 : 0); -} - -typedef struct pair_record { - size_t a; - size_t b; -} pair_record; - -typedef struct pair_record_list { - pair_record *items; - size_t count; - size_t cap; -} pair_record_list; - -static void free_pair_record_list(pair_record_list *list) { - free(list->items); - list->items = NULL; - list->count = 0; - list->cap = 0; -} - -static int cmp_pair_record(const void *a, const void *b) { - const pair_record *aa = (const pair_record *)a; - const pair_record *bb = (const pair_record *)b; - if (aa->a != bb->a) return aa->a > bb->a ? 1 : -1; - return aa->b > bb->b ? 1 : (aa->b < bb->b ? -1 : 0); -} - -static int push_pair_record(pair_record_list *list, size_t a, size_t b) { - if (a > b) { - size_t tmp = a; - a = b; - b = tmp; - } - if (list->count == list->cap) { - size_t next_cap = list->cap == 0 ? 1024 : list->cap * 2; - pair_record *next = (pair_record *)realloc(list->items, next_cap * sizeof(pair_record)); - if (next == NULL) return -1; - list->items = next; - list->cap = next_cap; - } - list->items[list->count++] = (pair_record){a, b}; - return 0; -} - -typedef struct seq_ref { - const char *seq; - size_t len; -} seq_ref; - -static int cmp_seq_ref(const void *a, const void *b) { - const seq_ref *aa = (const seq_ref *)a; - const seq_ref *bb = (const seq_ref *)b; - size_t min_len = aa->len < bb->len ? aa->len : bb->len; - int c = memcmp(aa->seq, bb->seq, min_len); - if (c != 0) return c; - return aa->len > bb->len ? 1 : (aa->len < bb->len ? -1 : 0); -} - -static size_t count_unique_target_sequences(const seq_table *targets) { - if (targets->count == 0) return 0; - seq_ref *refs = (seq_ref *)malloc(targets->count * sizeof(seq_ref)); - if (refs == NULL) return 0; - for (size_t i = 0; i < targets->count; ++i) { - refs[i].seq = targets->records[i].seq; - refs[i].len = targets->records[i].len; - } - qsort(refs, targets->count, sizeof(seq_ref), cmp_seq_ref); - size_t unique = 1; - for (size_t i = 1; i < targets->count; ++i) { - if (refs[i].len != refs[i - 1].len || memcmp(refs[i].seq, refs[i - 1].seq, refs[i].len) != 0) { - ++unique; - } - } - free(refs); - return unique; -} - -static int write_audit_summary_json(const char *out_dir, const char *audit_mode, int k, - size_t n_targets, size_t unique_sequences, - const char *min_edit_distance_json, const char *min_hamming_distance_json, - int safe_at_k0, int safe_at_k1, const char *safe_at_k2_json, - const char *safe_at_hamming_k2_json, const char *safe_at_hamming_k3_json, - unsigned long long pairs_d0, unsigned long long pairs_d1, - unsigned long long pairs_d2, unsigned long long pairs_within_k, - unsigned long long risk_pairs_k1, const char *risk_pairs_k2_json, - const char *risk_pairs_hamming_k2_json, const char *risk_pairs_hamming_k3_json, - unsigned long long ambiguous_query_variants_k1, int recommended_k) { - char path[4096]; - if (path_join(path, sizeof(path), out_dir, "audit_summary.json") != 0) return -1; - FILE *out = open_output_file(path); - if (out == NULL) return -1; - fprintf(out, - "{\n" - " \"audit_mode\": \"%s\",\n" - " \"k\": %d,\n" - " \"targets\": %zu,\n" - " \"unique_sequences\": %zu,\n" - " \"duplicate_sequences\": %zu,\n" - " \"min_edit_distance\": %s,\n" - " \"min_hamming_distance\": %s,\n" - " \"safe_at_k0\": %s,\n" - " \"safe_at_k1\": %s,\n" - " \"safe_at_k2\": %s,\n" - " \"safe_at_hamming_k2\": %s,\n" - " \"safe_at_hamming_k3\": %s,\n" - " \"pairs_distance_0\": %llu,\n" - " \"pairs_distance_1\": %llu,\n" - " \"pairs_distance_2\": %llu,\n" - " \"pairs_within_requested_k\": %llu,\n" - " \"risk_pairs_for_k1\": %llu,\n" - " \"risk_pairs_for_k2\": %s,\n" - " \"risk_pairs_for_hamming_k2\": %s,\n" - " \"risk_pairs_for_hamming_k3\": %s,\n" - " \"ambiguous_query_variants_k1\": %llu,\n" - " \"recommended_k\": %d\n" - "}\n", - audit_mode, k, n_targets, unique_sequences, n_targets - unique_sequences, - min_edit_distance_json, min_hamming_distance_json, - safe_at_k0 ? "true" : "false", safe_at_k1 ? "true" : "false", safe_at_k2_json, - safe_at_hamming_k2_json, safe_at_hamming_k3_json, - pairs_d0, pairs_d1, pairs_d2, pairs_within_k, risk_pairs_k1, risk_pairs_k2_json, - risk_pairs_hamming_k2_json, risk_pairs_hamming_k3_json, - ambiguous_query_variants_k1, recommended_k); - if (fclose(out) != 0) return -1; - return 0; -} - -static int audit_fast_outputs(const seq_table *targets, const char *out_dir, int k) { - int rc = -1; - int min_dist = -1; - unsigned long long pairs_d0 = 0; - unsigned long long pairs_d1 = 0; - unsigned long long pairs_d2 = 0; - unsigned long long pairs_within_k = 0; - unsigned long long risk_pairs_k1 = 0; - unsigned long long ambiguous_query_variants_k1 = 0; - int *nearest_dist = NULL; - size_t *nearest_idx = NULL; - unsigned long long *near_k1 = NULL; - size_t *parent = NULL; - variant_record_list variants = {0}; - pair_record_list candidate_pairs = {0}; - pair_record_list unique_pairs = {0}; - FILE *pairs = NULL; - FILE *clusters = NULL; - FILE *safety = NULL; - FILE *summary = NULL; - FILE *variants_out = NULL; - char path[4096]; - - nearest_dist = (int *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(int)); - nearest_idx = (size_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(size_t)); - near_k1 = (unsigned long long *)calloc(targets->count == 0 ? 1 : targets->count, sizeof(unsigned long long)); - parent = (size_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(size_t)); - if (nearest_dist == NULL || nearest_idx == NULL || near_k1 == NULL || parent == NULL) goto done; - for (size_t i = 0; i < targets->count; ++i) { - nearest_dist[i] = -1; - nearest_idx[i] = (size_t)-1; - parent[i] = i; - } - - for (size_t i = 0; i < targets->count; ++i) { - string_list local = {0}; - if (add_k1_variants_for_target(&local, targets->records[i].seq, targets->records[i].len) != 0) { - free_string_list(&local); - goto done; - } - for (size_t v = 0; v < local.count; ++v) { - if (push_variant_record(&variants, local.items[v], i) != 0) { - free_string_list(&local); - goto done; - } - } - free_string_list(&local); - } - qsort(variants.items, variants.count, sizeof(variant_record), cmp_variant_record); - - if (path_join(path, sizeof(path), out_dir, "ambiguous_variants.tsv") != 0) goto done; - variants_out = open_output_file(path); - if (variants_out == NULL) goto done; - fprintf(variants_out, "query_variant\ttargets_within_k1\n"); - - for (size_t start = 0; start < variants.count;) { - size_t end = start + 1; - while (end < variants.count && strcmp(variants.items[start].key, variants.items[end].key) == 0) ++end; - size_t unique_targets = 0; - size_t last_target = (size_t)-1; - for (size_t i = start; i < end; ++i) { - if (variants.items[i].target != last_target) { - variants.items[start + unique_targets].target = variants.items[i].target; - last_target = variants.items[i].target; - ++unique_targets; - } - } - if (unique_targets >= 2) { - ++ambiguous_query_variants_k1; - fprintf(variants_out, "%s\t%zu\n", variants.items[start].key, unique_targets); - for (size_t i = 0; i < unique_targets; ++i) { - for (size_t j = i + 1; j < unique_targets; ++j) { - if (push_pair_record(&candidate_pairs, variants.items[start + i].target, - variants.items[start + j].target) != 0) { - goto done; - } - } - } - } - start = end; - } - fclose(variants_out); - variants_out = NULL; - - qsort(candidate_pairs.items, candidate_pairs.count, sizeof(pair_record), cmp_pair_record); - for (size_t i = 0; i < candidate_pairs.count; ++i) { - if (i > 0 && candidate_pairs.items[i].a == candidate_pairs.items[i - 1].a && - candidate_pairs.items[i].b == candidate_pairs.items[i - 1].b) { - continue; - } - if (push_pair_record(&unique_pairs, candidate_pairs.items[i].a, candidate_pairs.items[i].b) != 0) goto done; - } - - if (path_join(path, sizeof(path), out_dir, "collision_pairs.tsv") != 0) goto done; - pairs = open_output_file(path); - if (pairs == NULL) goto done; - fprintf(pairs, "target_a\ttarget_b\tsequence_a\tsequence_b\tdistance\trisk_at_k1\trisk_at_k2\texample_ambiguous_query\n"); - - for (size_t p = 0; p < unique_pairs.count; ++p) { - size_t i = unique_pairs.items[p].a; - size_t j = unique_pairs.items[p].b; - int d = qdaln_edit_distance(targets->records[i].seq, targets->records[i].len, - targets->records[j].seq, targets->records[j].len); - if (d < 0) goto done; - if (min_dist < 0 || d < min_dist) min_dist = d; - if (nearest_dist[i] < 0 || d < nearest_dist[i]) { - nearest_dist[i] = d; - nearest_idx[i] = j; - } - if (nearest_dist[j] < 0 || d < nearest_dist[j]) { - nearest_dist[j] = d; - nearest_idx[j] = i; - } - if (d == 0) ++pairs_d0; - if (d == 1) ++pairs_d1; - if (d == 2) ++pairs_d2; - if (d <= k) ++pairs_within_k; - if (d <= 2) { - ++risk_pairs_k1; - ++near_k1[i]; - ++near_k1[j]; - uf_union(parent, i, j); - } - fprintf(pairs, "%s\t%s\t%s\t%s\t%d\t%s\tnot_computed\t\n", - targets->records[i].id, targets->records[j].id, targets->records[i].seq, targets->records[j].seq, - d, d <= 2 ? "yes" : "no"); - } - fclose(pairs); - pairs = NULL; - - if (path_join(path, sizeof(path), out_dir, "target_safety.tsv") != 0) goto done; - safety = open_output_file(path); - if (safety == NULL) goto done; - fprintf(safety, "target_id\tsequence\tnearest_target\tnearest_distance\tsafe_at_k1\tsafe_at_k2\tnum_nearby_k1_risk_targets\n"); - for (size_t i = 0; i < targets->count; ++i) { - const char *near_id = nearest_idx[i] == (size_t)-1 ? "" : targets->records[nearest_idx[i]].id; - int nd = nearest_dist[i]; - fprintf(safety, "%s\t%s\t%s\t%d\t%s\tnot_computed\t%llu\n", - targets->records[i].id, targets->records[i].seq, near_id, nd, - (nd < 0 || nd >= 3) ? "yes" : "no", near_k1[i]); - } - fclose(safety); - safety = NULL; - - if (path_join(path, sizeof(path), out_dir, "collision_clusters.tsv") != 0) goto done; - clusters = open_output_file(path); - if (clusters == NULL) goto done; - fprintf(clusters, "cluster_id\ttarget_id\tsequence\n"); - for (size_t i = 0; i < targets->count; ++i) { - if (near_k1[i] == 0) continue; - fprintf(clusters, "%zu\t%s\t%s\n", uf_find(parent, i), targets->records[i].id, targets->records[i].seq); - } - fclose(clusters); - clusters = NULL; - - size_t unique_sequences = count_unique_target_sequences(targets); - if (path_join(path, sizeof(path), out_dir, "audit_summary.tsv") != 0) goto done; - summary = open_output_file(path); - if (summary == NULL) goto done; - fprintf(summary, "metric\tvalue\n"); - fprintf(summary, "audit_mode\tfast\n"); - fprintf(summary, "targets\t%zu\n", targets->count); - fprintf(summary, "unique_sequences\t%zu\n", unique_sequences); - fprintf(summary, "duplicate_sequences\t%zu\n", targets->count - unique_sequences); - fprintf(summary, "min_edit_distance\t%s\n", min_dist < 0 ? ">=3" : (min_dist == 0 ? "0" : (min_dist == 1 ? "1" : "2"))); - fprintf(summary, "safe_at_k0\t%s\n", pairs_d0 == 0 ? "yes" : "no"); - fprintf(summary, "safe_at_k1\t%s\n", risk_pairs_k1 == 0 ? "yes" : "no"); - fprintf(summary, "safe_at_k2\tnot_computed\n"); - fprintf(summary, "safe_at_hamming_k2\tnot_computed\n"); - fprintf(summary, "safe_at_hamming_k3\tnot_computed\n"); - fprintf(summary, "pairs_distance_0\t%llu\n", pairs_d0); - fprintf(summary, "pairs_distance_1\t%llu\n", pairs_d1); - fprintf(summary, "pairs_distance_2\t%llu\n", pairs_d2); - fprintf(summary, "pairs_within_requested_k\t%llu\n", pairs_within_k); - fprintf(summary, "risk_pairs_for_k1\t%llu\n", risk_pairs_k1); - fprintf(summary, "risk_pairs_for_k2\tnot_computed\n"); - fprintf(summary, "risk_pairs_for_hamming_k2\tnot_computed\n"); - fprintf(summary, "risk_pairs_for_hamming_k3\tnot_computed\n"); - fprintf(summary, "ambiguous_query_variants_k1\t%llu\n", ambiguous_query_variants_k1); - fprintf(summary, "recommended_k\t%d\n", pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0); - fclose(summary); - summary = NULL; - if (write_audit_summary_json(out_dir, "fast", k, targets->count, unique_sequences, - min_dist < 0 ? "\">=3\"" : (min_dist == 0 ? "0" : (min_dist == 1 ? "1" : "2")), - "null", - pairs_d0 == 0, risk_pairs_k1 == 0, "null", - "null", "null", - pairs_d0, pairs_d1, pairs_d2, pairs_within_k, risk_pairs_k1, "null", - "null", "null", - ambiguous_query_variants_k1, - pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0) != 0) { - goto done; - } - rc = 0; - -done: - if (pairs != NULL) fclose(pairs); - if (clusters != NULL) fclose(clusters); - if (safety != NULL) fclose(safety); - if (summary != NULL) fclose(summary); - if (variants_out != NULL) fclose(variants_out); - free(nearest_dist); - free(nearest_idx); - free(near_k1); - free(parent); - free_variant_record_list(&variants); - free_pair_record_list(&candidate_pairs); - free_pair_record_list(&unique_pairs); - return rc; -} - -static int run_audit(const char *argv0, int argc, char **argv) { - const char *targets_path = NULL; - const char *out_dir = NULL; - const char *audit_mode = "auto"; - int k = 1; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if ((strcmp(arg, "--targets") == 0 || strcmp(arg, "--library") == 0) && i < argc) { - targets_path = argv[i++]; - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 3) { - usage(argv0); - return 2; - } - } else if ((strcmp(arg, "--out-dir") == 0 || strcmp(arg, "--out") == 0) && i < argc) { - out_dir = argv[i++]; - } else if (strcmp(arg, "--audit-mode") == 0 && i < argc) { - audit_mode = argv[i++]; - if (strcmp(audit_mode, "auto") != 0 && strcmp(audit_mode, "exact") != 0 && strcmp(audit_mode, "fast") != 0) { - usage(argv0); - return 2; - } - } else { - usage(argv0); - return 2; - } - } - if (targets_path == NULL || out_dir == NULL) { - usage(argv0); - return 2; - } - - seq_table targets = {0}; - int rc = 1; - int min_dist = -1; - unsigned long long pairs_d0 = 0; - unsigned long long pairs_d1 = 0; - unsigned long long pairs_d2 = 0; - unsigned long long pairs_within_k = 0; - unsigned long long risk_pairs_k1 = 0; - unsigned long long risk_pairs_k2 = 0; - unsigned long long risk_pairs_hamming_k2 = 0; - unsigned long long risk_pairs_hamming_k3 = 0; - int min_hamming_dist = -1; - int *nearest_dist = NULL; - size_t *nearest_idx = NULL; - unsigned long long *near_k1 = NULL; - size_t *parent = NULL; - FILE *pairs = NULL; - FILE *clusters = NULL; - FILE *safety = NULL; - FILE *summary = NULL; - FILE *variants_out = NULL; - string_list k1_variants = {0}; - unsigned long long ambiguous_query_variants_k1 = 0; - char path[4096]; - - if (read_target_table(targets_path, &targets) != 0) { - fprintf(stderr, "failed to read targets\n"); - goto done; - } - if (ensure_dir(out_dir) != 0) { - fprintf(stderr, "failed to create audit output directory\n"); - goto done; - } - int use_fast = strcmp(audit_mode, "fast") == 0 || (strcmp(audit_mode, "auto") == 0 && targets.count > 2000); - if (use_fast) { - rc = audit_fast_outputs(&targets, out_dir, k) == 0 ? 0 : 1; - if (rc == 0) printf("%s\n", out_dir); - goto done; - } - nearest_dist = (int *)malloc((targets.count == 0 ? 1 : targets.count) * sizeof(int)); - nearest_idx = (size_t *)malloc((targets.count == 0 ? 1 : targets.count) * sizeof(size_t)); - near_k1 = (unsigned long long *)calloc(targets.count == 0 ? 1 : targets.count, sizeof(unsigned long long)); - parent = (size_t *)malloc((targets.count == 0 ? 1 : targets.count) * sizeof(size_t)); - if (nearest_dist == NULL || nearest_idx == NULL || near_k1 == NULL || parent == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t i = 0; i < targets.count; ++i) { - nearest_dist[i] = -1; - nearest_idx[i] = (size_t)-1; - parent[i] = i; - } - - if (path_join(path, sizeof(path), out_dir, "collision_pairs.tsv") != 0) goto done; - pairs = open_output_file(path); - if (pairs == NULL) goto done; - fprintf(pairs, "target_a\ttarget_b\tsequence_a\tsequence_b\tdistance\trisk_at_k1\trisk_at_k2\texample_ambiguous_query\n"); - - for (size_t i = 0; i < targets.count; ++i) { - for (size_t j = i + 1; j < targets.count; ++j) { - int d = qdaln_edit_distance(targets.records[i].seq, targets.records[i].len, - targets.records[j].seq, targets.records[j].len); - if (d < 0) goto done; - if (min_dist < 0 || d < min_dist) min_dist = d; - if (nearest_dist[i] < 0 || d < nearest_dist[i]) { - nearest_dist[i] = d; - nearest_idx[i] = j; - } - if (nearest_dist[j] < 0 || d < nearest_dist[j]) { - nearest_dist[j] = d; - nearest_idx[j] = i; - } - if (d == 0) ++pairs_d0; - if (d == 1) ++pairs_d1; - if (d == 2) ++pairs_d2; - if (d <= k) ++pairs_within_k; - if (d <= 2) { - ++risk_pairs_k1; - ++near_k1[i]; - ++near_k1[j]; - uf_union(parent, i, j); - } - if (d <= 4) ++risk_pairs_k2; - int hd = hamming_distance_cli(targets.records[i].seq, targets.records[i].len, - targets.records[j].seq, targets.records[j].len); - if (hd >= 0) { - if (min_hamming_dist < 0 || hd < min_hamming_dist) min_hamming_dist = hd; - if (hd <= 4) ++risk_pairs_hamming_k2; - if (hd <= 6) ++risk_pairs_hamming_k3; - } - if (d <= 2 || d <= 2 * k) { - const char *example = d == 0 ? targets.records[i].seq : ""; - fprintf(pairs, "%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\n", - targets.records[i].id, targets.records[j].id, targets.records[i].seq, targets.records[j].seq, - d, d <= 2 ? "yes" : "no", d <= 4 ? "yes" : "no", example); - } - } - } - fclose(pairs); - pairs = NULL; - - if (path_join(path, sizeof(path), out_dir, "target_safety.tsv") != 0) goto done; - safety = open_output_file(path); - if (safety == NULL) goto done; - fprintf(safety, "target_id\tsequence\tnearest_target\tnearest_distance\tsafe_at_k1\tsafe_at_k2\tnum_nearby_k1_risk_targets\n"); - for (size_t i = 0; i < targets.count; ++i) { - const char *near_id = nearest_idx[i] == (size_t)-1 ? "" : targets.records[nearest_idx[i]].id; - int nd = nearest_dist[i]; - fprintf(safety, "%s\t%s\t%s\t%d\t%s\t%s\t%llu\n", - targets.records[i].id, targets.records[i].seq, near_id, nd, - (nd < 0 || nd >= 3) ? "yes" : "no", - (nd < 0 || nd >= 5) ? "yes" : "no", - near_k1[i]); - } - fclose(safety); - safety = NULL; - - if (path_join(path, sizeof(path), out_dir, "collision_clusters.tsv") != 0) goto done; - clusters = open_output_file(path); - if (clusters == NULL) goto done; - fprintf(clusters, "cluster_id\ttarget_id\tsequence\n"); - for (size_t i = 0; i < targets.count; ++i) { - if (near_k1[i] == 0) continue; - fprintf(clusters, "%zu\t%s\t%s\n", uf_find(parent, i), targets.records[i].id, targets.records[i].seq); - } - fclose(clusters); - clusters = NULL; - - size_t unique_sequences = 0; - for (size_t i = 0; i < targets.count; ++i) { - int seen = 0; - for (size_t j = 0; j < i; ++j) { - if (targets.records[i].len == targets.records[j].len && - memcmp(targets.records[i].seq, targets.records[j].seq, targets.records[i].len) == 0) { - seen = 1; - break; - } - } - if (!seen) ++unique_sequences; - } - - if (path_join(path, sizeof(path), out_dir, "ambiguous_variants.tsv") != 0) goto done; - variants_out = open_output_file(path); - if (variants_out == NULL) goto done; - fprintf(variants_out, "query_variant\ttargets_within_k1\n"); - for (size_t i = 0; i < targets.count; ++i) { - if (add_k1_variants_for_target(&k1_variants, targets.records[i].seq, targets.records[i].len) != 0) { - fprintf(stderr, "failed to enumerate k=1 variants\n"); - goto done; - } - } - for (size_t vi = 0; vi < k1_variants.count; ++vi) { - unsigned long long within = 0; - size_t q_len = strlen(k1_variants.items[vi]); - for (size_t ti = 0; ti < targets.count; ++ti) { - int ok = qdaln_edit_distance_leq(k1_variants.items[vi], q_len, targets.records[ti].seq, - targets.records[ti].len, 1); - if (ok < 0) goto done; - if (ok) ++within; - } - if (within >= 2) { - ++ambiguous_query_variants_k1; - fprintf(variants_out, "%s\t%llu\n", k1_variants.items[vi], within); - } - } - fclose(variants_out); - variants_out = NULL; - - if (path_join(path, sizeof(path), out_dir, "audit_summary.tsv") != 0) goto done; - summary = open_output_file(path); - if (summary == NULL) goto done; - fprintf(summary, "metric\tvalue\n"); - fprintf(summary, "audit_mode\texact\n"); - fprintf(summary, "targets\t%zu\n", targets.count); - fprintf(summary, "unique_sequences\t%zu\n", unique_sequences); - fprintf(summary, "duplicate_sequences\t%zu\n", targets.count - unique_sequences); - fprintf(summary, "min_edit_distance\t%d\n", min_dist); - if (min_hamming_dist >= 0) { - fprintf(summary, "min_hamming_distance\t%d\n", min_hamming_dist); - } else { - fprintf(summary, "min_hamming_distance\tnot_computed\n"); - } - fprintf(summary, "safe_at_k0\t%s\n", pairs_d0 == 0 ? "yes" : "no"); - fprintf(summary, "safe_at_k1\t%s\n", risk_pairs_k1 == 0 ? "yes" : "no"); - fprintf(summary, "safe_at_k2\t%s\n", risk_pairs_k2 == 0 ? "yes" : "no"); - fprintf(summary, "safe_at_hamming_k2\t%s\n", risk_pairs_hamming_k2 == 0 ? "yes" : "no"); - fprintf(summary, "safe_at_hamming_k3\t%s\n", risk_pairs_hamming_k3 == 0 ? "yes" : "no"); - fprintf(summary, "pairs_distance_0\t%llu\n", pairs_d0); - fprintf(summary, "pairs_distance_1\t%llu\n", pairs_d1); - fprintf(summary, "pairs_distance_2\t%llu\n", pairs_d2); - fprintf(summary, "pairs_within_requested_k\t%llu\n", pairs_within_k); - fprintf(summary, "risk_pairs_for_k1\t%llu\n", risk_pairs_k1); - fprintf(summary, "risk_pairs_for_k2\t%llu\n", risk_pairs_k2); - fprintf(summary, "risk_pairs_for_hamming_k2\t%llu\n", risk_pairs_hamming_k2); - fprintf(summary, "risk_pairs_for_hamming_k3\t%llu\n", risk_pairs_hamming_k3); - fprintf(summary, "ambiguous_query_variants_k1\t%llu\n", ambiguous_query_variants_k1); - fprintf(summary, "recommended_k\t%d\n", pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0); - fclose(summary); - summary = NULL; - char min_dist_json[32]; - char min_hamming_dist_json[32]; - char risk_pairs_k2_json[32]; - char risk_pairs_hamming_k2_json[32]; - char risk_pairs_hamming_k3_json[32]; - snprintf(min_dist_json, sizeof(min_dist_json), "%d", min_dist); - snprintf(min_hamming_dist_json, sizeof(min_hamming_dist_json), "%d", min_hamming_dist); - snprintf(risk_pairs_k2_json, sizeof(risk_pairs_k2_json), "%llu", risk_pairs_k2); - snprintf(risk_pairs_hamming_k2_json, sizeof(risk_pairs_hamming_k2_json), "%llu", risk_pairs_hamming_k2); - snprintf(risk_pairs_hamming_k3_json, sizeof(risk_pairs_hamming_k3_json), "%llu", risk_pairs_hamming_k3); - if (write_audit_summary_json(out_dir, "exact", k, targets.count, unique_sequences, - min_dist_json, min_hamming_dist >= 0 ? min_hamming_dist_json : "null", - pairs_d0 == 0, risk_pairs_k1 == 0, - risk_pairs_k2 == 0 ? "true" : "false", - risk_pairs_hamming_k2 == 0 ? "true" : "false", - risk_pairs_hamming_k3 == 0 ? "true" : "false", - pairs_d0, pairs_d1, pairs_d2, pairs_within_k, risk_pairs_k1, - risk_pairs_k2_json, risk_pairs_hamming_k2_json, risk_pairs_hamming_k3_json, - ambiguous_query_variants_k1, - pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0) != 0) { - goto done; - } - - printf("%s\n", out_dir); - rc = 0; - -done: - if (pairs != NULL) fclose(pairs); - if (clusters != NULL) fclose(clusters); - if (safety != NULL) fclose(safety); - if (summary != NULL) fclose(summary); - if (variants_out != NULL) fclose(variants_out); - free_string_list(&k1_variants); - free(nearest_dist); - free(nearest_idx); - free(near_k1); - free(parent); - free_table(&targets); - return rc; -} - -static void write_fastq_record(FILE *out, const char *header, const char *seq, const char *plus, const char *qual) { - fprintf(out, "%s\n%s\n%s\n%s\n", header, seq, plus, qual); -} - -typedef struct unmatched_entry { - char *seq; - unsigned long long count; - int offset_hint; - unsigned long long low_quality_count; - char *adapter_hint; -} unmatched_entry; - -typedef struct unmatched_table { - unmatched_entry *entries; - size_t count; - size_t cap; -} unmatched_table; - -static void free_unmatched_table(unmatched_table *table) { - for (size_t i = 0; i < table->count; ++i) { - free(table->entries[i].seq); - free(table->entries[i].adapter_hint); - } - free(table->entries); - table->entries = NULL; - table->count = 0; - table->cap = 0; -} - -static int add_unmatched_observation(unmatched_table *table, const char *seq, int offset_hint, - int low_quality, const char *adapter_hint) { - for (size_t i = 0; i < table->count; ++i) { - if (strcmp(table->entries[i].seq, seq) == 0) { - ++table->entries[i].count; - if (table->entries[i].offset_hint == 0 && offset_hint != 0) table->entries[i].offset_hint = offset_hint; - if (low_quality) ++table->entries[i].low_quality_count; - if ((table->entries[i].adapter_hint == NULL || table->entries[i].adapter_hint[0] == '\0') && - adapter_hint != NULL && adapter_hint[0] != '\0') { - free(table->entries[i].adapter_hint); - table->entries[i].adapter_hint = xstrndup(adapter_hint, strlen(adapter_hint)); - if (table->entries[i].adapter_hint == NULL) return -1; - } - return 0; - } - } - if (table->count == table->cap) { - size_t next_cap = table->cap == 0 ? 16 : table->cap * 2; - unmatched_entry *next = (unmatched_entry *)realloc(table->entries, next_cap * sizeof(unmatched_entry)); - if (next == NULL) return -1; - table->entries = next; - table->cap = next_cap; - } - table->entries[table->count].seq = xstrndup(seq, strlen(seq)); - if (table->entries[table->count].seq == NULL) return -1; - table->entries[table->count].count = 1; - table->entries[table->count].offset_hint = offset_hint; - table->entries[table->count].low_quality_count = low_quality ? 1 : 0; - table->entries[table->count].adapter_hint = adapter_hint == NULL ? xstrndup("", 0) : xstrndup(adapter_hint, strlen(adapter_hint)); - if (table->entries[table->count].adapter_hint == NULL) { - free(table->entries[table->count].seq); - return -1; - } - ++table->count; - return 0; -} - -static int cmp_unmatched_entry_desc(const void *a, const void *b) { - const unmatched_entry *aa = (const unmatched_entry *)a; - const unmatched_entry *bb = (const unmatched_entry *)b; - if (aa->count != bb->count) return aa->count < bb->count ? 1 : -1; - return strcmp(aa->seq, bb->seq); -} - -static int contains_base_n(const char *seq) { - for (; *seq != '\0'; ++seq) { - if (*seq == 'N' || *seq == 'n') return 1; - } - return 0; -} - -static char complement_base(char c) { - switch (c) { - case 'A': - case 'a': - return 'T'; - case 'C': - case 'c': - return 'G'; - case 'G': - case 'g': - return 'C'; - case 'T': - case 't': - return 'A'; - default: - return 'N'; - } -} - -static int reverse_complement_seq(const char *seq, char *out, size_t out_cap) { - size_t len = strlen(seq); - if (len + 1 > out_cap) return -1; - for (size_t i = 0; i < len; ++i) out[i] = complement_base(seq[len - 1 - i]); - out[len] = '\0'; - return 0; -} - -static int nearest_target_for_query(const seq_table *targets, const char *query, int *nearest_index, int *nearest_dist) { - *nearest_index = -1; - *nearest_dist = -1; - size_t q_len = strlen(query); - for (size_t i = 0; i < targets->count; ++i) { - int d = qdaln_edit_distance(query, q_len, targets->records[i].seq, targets->records[i].len); - if (d < 0) return -1; - if (*nearest_dist < 0 || d < *nearest_dist) { - *nearest_dist = d; - *nearest_index = (int)i; - } - } - return 0; -} - -static int find_offset_hint(const qdaln_index *index, const char *seq, size_t seq_len, size_t target_start, - size_t target_len, int k, size_t offset_window) { - if (offset_window == 0) return 0; - char observed[8192]; - if (target_len >= sizeof(observed)) return 0; - for (size_t step = 1; step <= offset_window; ++step) { - for (int sign = 1; sign >= -1; sign -= 2) { - if (sign < 0 && target_start < step) continue; - size_t offset = sign > 0 ? target_start + step : target_start - step; - if (offset > seq_len || target_len > seq_len - offset) continue; - memcpy(observed, seq + offset, target_len); - observed[target_len] = '\0'; - uppercase_ascii(observed); - const char *read_ptr = observed; - size_t read_len = target_len; - qdaln_match_result r; - qdaln_index_stats stats; - if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &r, &stats) != 0) return 0; - if (r.status == QDALN_MATCH_UNIQUE) return sign > 0 ? (int)step : -(int)step; - } - } - return 0; -} - -static int window_has_low_quality(const char *qual, size_t target_start, size_t target_len, int threshold) { - if (threshold < 0) return 0; - size_t qual_len = strlen(qual); - if (target_start > qual_len || target_len > qual_len - target_start) return 0; - for (size_t i = 0; i < target_len; ++i) { - int phred = (int)((unsigned char)qual[target_start + i]) - 33; - if (phred < threshold) return 1; - } - return 0; -} - -static int run_inspect_unmatched(const char *argv0, int argc, char **argv) { - const char *targets_path = NULL; - const char *reads_path = NULL; - const char *out_path = NULL; - size_t target_start = 0; - size_t target_len = 0; - size_t top_n = 100; - size_t offset_window = 0; - char adapter[1024] = ""; - int low_quality_threshold = -1; - int k = -1; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if ((strcmp(arg, "--targets") == 0 || strcmp(arg, "--library") == 0) && i < argc) { - targets_path = argv[i++]; - } else if (strcmp(arg, "--reads") == 0 && i < argc) { - reads_path = argv[i++]; - } else if ((strcmp(arg, "--target-start") == 0 || strcmp(arg, "--guide-start") == 0) && i < argc) { - if (parse_size_value(argv[i++], &target_start) != 0) { - usage(argv0); - return 2; - } - } else if ((strcmp(arg, "--target-length") == 0 || strcmp(arg, "--guide-length") == 0) && i < argc) { - if (parse_size_value(argv[i++], &target_len) != 0 || target_len == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || (k != 0 && k != 1)) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--top") == 0 && i < argc) { - if (parse_size_value(argv[i++], &top_n) != 0 || top_n == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--offset-window") == 0 && i < argc) { - if (parse_size_value(argv[i++], &offset_window) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--adapter") == 0 && i < argc) { - const char *value = argv[i++]; - size_t n = strlen(value); - if (n == 0 || n >= sizeof(adapter)) { - usage(argv0); - return 2; - } - memcpy(adapter, value, n + 1); - uppercase_ascii(adapter); - } else if (strcmp(arg, "--low-quality-threshold") == 0 && i < argc) { - if (parse_int_value(argv[i++], &low_quality_threshold) != 0 || - low_quality_threshold < 0 || low_quality_threshold > 93) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--out") == 0 && i < argc) { - out_path = argv[i++]; - } else { - usage(argv0); - return 2; - } - } - if (targets_path == NULL || reads_path == NULL || out_path == NULL || target_len == 0 || k < 0) { - usage(argv0); - return 2; - } - - seq_table targets = {0}; - const char **target_ptrs = NULL; - size_t *target_lens = NULL; - qdaln_index *index = NULL; - fastq_reader reader = {0}; - unmatched_table unmatched = {0}; - FILE *out = NULL; - int rc = 1; - - if (read_target_table(targets_path, &targets) != 0) { - fprintf(stderr, "failed to read targets\n"); - goto done; - } - if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - index = qdaln_index_build(target_ptrs, target_lens, targets.count); - if (index == NULL) { - fprintf(stderr, "failed to build target index\n"); - goto done; - } - if (fastq_reader_open(&reader, reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - goto done; - } - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - char observed[8192]; - int got = 0; - size_t seq_len = 0; - while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - qdaln_match_result r = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - observed[0] = '\0'; - if (target_start <= seq_len && target_len <= seq_len - target_start && target_len < sizeof(observed)) { - memcpy(observed, seq + target_start, target_len); - observed[target_len] = '\0'; - uppercase_ascii(observed); - const char *read_ptr = observed; - size_t read_len = target_len; - qdaln_index_stats stats; - if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &r, &stats) != 0) { - fprintf(stderr, "assignment failed\n"); - goto done; - } - } else { - strncpy(observed, "", sizeof(observed) - 1); - observed[sizeof(observed) - 1] = '\0'; - } - if (r.status == QDALN_MATCH_NONE || r.status == QDALN_MATCH_INVALID) { - int offset_hint = strcmp(observed, "") == 0 ? 0 : - find_offset_hint(index, seq, seq_len, target_start, target_len, k, offset_window); - char seq_upper[8192]; - seq_upper[0] = '\0'; - const char *adapter_hint = ""; - int low_quality = strcmp(observed, "") == 0 ? 0 : - window_has_low_quality(qual, target_start, target_len, low_quality_threshold); - if (adapter[0] != '\0') { - strncpy(seq_upper, seq, sizeof(seq_upper) - 1); - seq_upper[sizeof(seq_upper) - 1] = '\0'; - uppercase_ascii(seq_upper); - if (strstr(seq_upper, adapter) != NULL || strstr(observed, adapter) != NULL) adapter_hint = adapter; - } - if (add_unmatched_observation(&unmatched, observed, offset_hint, low_quality, adapter_hint) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - } - } - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - goto done; - } - - qsort(unmatched.entries, unmatched.count, sizeof(unmatched_entry), cmp_unmatched_entry_desc); - out = open_output_file(out_path); - if (out == NULL) { - fprintf(stderr, "failed to open unmatched inspection output\n"); - goto done; - } - fprintf(out, "sequence\tcount\tlength\tnearest_target\tnearest_distance\tnearest_edit_class\tpossible_reason\treverse_complement\trevcomp_nearest_target\trevcomp_nearest_distance\toffset_hint\tadapter_hint\n"); - size_t limit = unmatched.count < top_n ? unmatched.count : top_n; - for (size_t i = 0; i < limit; ++i) { - int nearest_index = -1; - int nearest_dist = -1; - int rc_nearest_index = -1; - int rc_nearest_dist = -1; - const char *nearest_id = ""; - const char *rc_nearest_id = ""; - const char *edit_class = "invalid"; - const char *reason = "wrong_length"; - char rc_seq[8192] = ""; - if (strcmp(unmatched.entries[i].seq, "") != 0 && - nearest_target_for_query(&targets, unmatched.entries[i].seq, &nearest_index, &nearest_dist) == 0) { - if (nearest_index >= 0) { - nearest_id = targets.records[nearest_index].id; - int kind = correction_kind(unmatched.entries[i].seq, strlen(unmatched.entries[i].seq), - targets.records[nearest_index].seq, targets.records[nearest_index].len, - nearest_dist); - edit_class = correction_name(kind); - } - if (contains_base_n(unmatched.entries[i].seq)) reason = "contains_N"; - else if (nearest_dist > k) reason = "near_known_target_above_k"; - else reason = "unknown"; - if (unmatched.entries[i].low_quality_count != 0) reason = "low_quality_candidate"; - if (unmatched.entries[i].adapter_hint != NULL && unmatched.entries[i].adapter_hint[0] != '\0') { - reason = "adapter_or_primer_candidate"; - } - if (unmatched.entries[i].offset_hint != 0) reason = "offset_shift_candidate"; - if (reverse_complement_seq(unmatched.entries[i].seq, rc_seq, sizeof(rc_seq)) == 0 && - nearest_target_for_query(&targets, rc_seq, &rc_nearest_index, &rc_nearest_dist) == 0 && - rc_nearest_index >= 0) { - rc_nearest_id = targets.records[rc_nearest_index].id; - if (unmatched.entries[i].offset_hint == 0 && - rc_nearest_dist <= k && (nearest_dist < 0 || rc_nearest_dist < nearest_dist)) { - reason = "reverse_complement_candidate"; - } - } - } - fprintf(out, "%s\t%llu\t%zu\t%s\t%d\t%s\t%s\t%s\t%s\t%d\t", - unmatched.entries[i].seq, unmatched.entries[i].count, strlen(unmatched.entries[i].seq), - nearest_id, nearest_dist, edit_class, reason, rc_seq, rc_nearest_id, rc_nearest_dist); - if (unmatched.entries[i].offset_hint != 0) fprintf(out, "%d", unmatched.entries[i].offset_hint); - fprintf(out, "\t%s\n", unmatched.entries[i].adapter_hint == NULL ? "" : unmatched.entries[i].adapter_hint); - } - rc = 0; - -done: - if (out != NULL) fclose(out); - fastq_reader_close(&reader); - qdaln_index_free(index); - free(target_ptrs); - free(target_lens); - free_unmatched_table(&unmatched); - free_table(&targets); - return rc; -} - -typedef struct pair_count_stats { - unsigned long long total_reads; - unsigned long long assigned_pairs; - unsigned long long pair_ambiguous; - unsigned long long left_unmatched; - unsigned long long right_unmatched; - unsigned long long invalid; - unsigned long long candidates_considered; - unsigned long long candidates_verified; -} pair_count_stats; - -static const char *pair_status_name(qdaln_match_result left, qdaln_match_result right) { - if (left.status == QDALN_MATCH_INVALID || right.status == QDALN_MATCH_INVALID) return "invalid"; - if (left.status == QDALN_MATCH_AMBIGUOUS || right.status == QDALN_MATCH_AMBIGUOUS) return "ambiguous"; - if (left.status == QDALN_MATCH_NONE || right.status == QDALN_MATCH_NONE) return "none"; - if (left.status == QDALN_MATCH_UNIQUE && right.status == QDALN_MATCH_UNIQUE) return "unique"; - return "invalid"; -} - -static void print_pair_assignment_row(FILE *out, const char *read_id, const seq_table *left_targets, - const seq_table *right_targets, const char *left_observed, - qdaln_match_result left, const char *right_observed, - qdaln_match_result right) { - const char *left_id = left.target_index >= 0 ? left_targets->records[left.target_index].id : ""; - const char *right_id = right.target_index >= 0 ? right_targets->records[right.target_index].id : ""; - fprintf(out, "%s\t%s\t%d\t%s\t%s\t%d\t%s\t%d\t%s\t%s\t%d\t%s\n", - read_id, left_observed, left.target_index, left_id, status_name(left.status), left.best_distance, - right_observed, right.target_index, right_id, status_name(right.status), right.best_distance, - pair_status_name(left, right)); -} - -static int run_pair_count(const char *argv0, int argc, char **argv) { - const char *left_path = NULL; - const char *right_path = NULL; - const char *reads_path = NULL; - const char *out_path = NULL; - const char *summary_path = NULL; - const char *assignments_path = NULL; - size_t left_start = 0; - size_t right_start = 0; - size_t left_len = 0; - size_t right_len = 0; - int k = -1; - count_metric metric = COUNT_METRIC_LEVENSHTEIN; - ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if (strcmp(arg, "--left-targets") == 0 && i < argc) { - left_path = argv[i++]; - } else if (strcmp(arg, "--right-targets") == 0 && i < argc) { - right_path = argv[i++]; - } else if (strcmp(arg, "--reads") == 0 && i < argc) { - reads_path = argv[i++]; - } else if (strcmp(arg, "--left-start") == 0 && i < argc) { - if (parse_size_value(argv[i++], &left_start) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--left-length") == 0 && i < argc) { - if (parse_size_value(argv[i++], &left_len) != 0 || left_len == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--right-start") == 0 && i < argc) { - if (parse_size_value(argv[i++], &right_start) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--right-length") == 0 && i < argc) { - if (parse_size_value(argv[i++], &right_len) != 0 || right_len == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 2) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--metric") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "hamming") == 0) { - metric = COUNT_METRIC_HAMMING; - } else if (strcmp(value, "levenshtein") == 0) { - metric = COUNT_METRIC_LEVENSHTEIN; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "radius") == 0) { - assignment_policy = AMBIGUITY_POLICY_RADIUS; - } else if (strcmp(value, "best") == 0) { - assignment_policy = AMBIGUITY_POLICY_BEST; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--out") == 0 && i < argc) { - out_path = argv[i++]; - } else if (strcmp(arg, "--summary") == 0 && i < argc) { - summary_path = argv[i++]; - } else if (strcmp(arg, "--assignments") == 0 && i < argc) { - assignments_path = argv[i++]; - } else { - usage(argv0); - return 2; - } - } - - if (left_path == NULL || right_path == NULL || reads_path == NULL || out_path == NULL || - left_len == 0 || right_len == 0 || k < 0) { - usage(argv0); - return 2; - } - if (metric == COUNT_METRIC_HAMMING && k > 1) { - fprintf(stderr, "--k 2 is only valid with --metric levenshtein\n"); - return 2; - } - - seq_table left_targets = {0}; - seq_table right_targets = {0}; - const char **left_ptrs = NULL; - const char **right_ptrs = NULL; - size_t *left_lens = NULL; - size_t *right_lens = NULL; - qdaln_index *left_index = NULL; - qdaln_index *right_index = NULL; - fastq_reader reader = {0}; - FILE *out = NULL; - FILE *summary = NULL; - FILE *assignments = NULL; - unsigned long long *pair_counts = NULL; - pair_count_stats stats = {0}; - int rc = 1; - - if (read_target_table(left_path, &left_targets) != 0 || read_target_table(right_path, &right_targets) != 0) { - fprintf(stderr, "failed to read pair target tables\n"); - goto done; - } - int left_id_check = validate_unique_seq_ids(&left_targets, "left target"); - int right_id_check = validate_unique_seq_ids(&right_targets, "right target"); - if (left_id_check != 0 || right_id_check != 0) { - if (left_id_check == -1 || right_id_check == -1) fprintf(stderr, "out of memory\n"); - goto done; - } - if (metric == COUNT_METRIC_HAMMING && - (!all_targets_have_length(&left_targets, left_len) || !all_targets_have_length(&right_targets, right_len))) { - fprintf(stderr, "--metric hamming requires targets to match their configured window lengths\n"); - goto done; - } - if (build_target_arrays(&left_targets, &left_ptrs, &left_lens) != 0 || - build_target_arrays(&right_targets, &right_ptrs, &right_lens) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - left_index = qdaln_index_build(left_ptrs, left_lens, left_targets.count); - right_index = qdaln_index_build(right_ptrs, right_lens, right_targets.count); - if (left_index == NULL || right_index == NULL) { - fprintf(stderr, "failed to build pair target indexes\n"); - goto done; - } - size_t pair_count_slots = 0; - if (checked_mul_size(left_targets.count, right_targets.count, &pair_count_slots) != 0) { - fprintf(stderr, "pair count matrix is too large\n"); - goto done; - } - pair_counts = (unsigned long long *)calloc(alloc_count_or_one(pair_count_slots), sizeof(unsigned long long)); - if (pair_counts == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - if (fastq_reader_open(&reader, reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - goto done; - } - if (assignments_path != NULL) { - assignments = open_output_file(assignments_path); - if (assignments == NULL) { - fprintf(stderr, "failed to open assignments output\n"); - goto done; - } - fprintf(assignments, "read_id\tleft_observed\tleft_index\tleft_id\tleft_status\tleft_distance\tright_observed\tright_index\tright_id\tright_status\tright_distance\tpair_status\n"); - } - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - char read_id[8192]; - char left_observed[8192]; - char right_observed[8192]; - size_t seq_len = 0; - int got = 0; - while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - (void)plus; - (void)qual; - qdaln_match_result left = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_match_result right = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_index_stats left_stats = {0, 0}; - qdaln_index_stats right_stats = {0, 0}; - fastq_read_id(header, read_id, sizeof(read_id)); - left_observed[0] = '\0'; - right_observed[0] = '\0'; - ++stats.total_reads; - - if (assign_count_window(left_index, seq, seq_len, left_start, left_len, k, metric, 0, - &left, &left_stats, left_observed, sizeof(left_observed), 0) != 0 || - assign_count_window(right_index, seq, seq_len, right_start, right_len, k, metric, 0, - &right, &right_stats, right_observed, sizeof(right_observed), 0) != 0) { - fprintf(stderr, "FASTQ pair assignment failed\n"); - goto done; - } - apply_ambiguity_policy(&left, assignment_policy); - apply_ambiguity_policy(&right, assignment_policy); - stats.candidates_considered += left_stats.candidates_considered + right_stats.candidates_considered; - stats.candidates_verified += left_stats.candidates_verified + right_stats.candidates_verified; - - const char *pair_status = pair_status_name(left, right); - if (strcmp(pair_status, "unique") == 0) { - size_t slot = (size_t)left.target_index * right_targets.count + (size_t)right.target_index; - ++pair_counts[slot]; - ++stats.assigned_pairs; - } else if (strcmp(pair_status, "invalid") == 0) { - ++stats.invalid; - } else { - if (left.status == QDALN_MATCH_AMBIGUOUS || right.status == QDALN_MATCH_AMBIGUOUS) ++stats.pair_ambiguous; - if (left.status == QDALN_MATCH_NONE) ++stats.left_unmatched; - if (right.status == QDALN_MATCH_NONE) ++stats.right_unmatched; - } - if (assignments != NULL) { - print_pair_assignment_row(assignments, read_id, &left_targets, &right_targets, - left_observed, left, right_observed, right); - } - } - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - goto done; - } - - out = open_output_file(out_path); - if (out == NULL) { - fprintf(stderr, "failed to open pair-count output\n"); - goto done; - } - fprintf(out, "left_id\tright_id\tcount\n"); - for (size_t li = 0; li < left_targets.count; ++li) { - for (size_t ri = 0; ri < right_targets.count; ++ri) { - unsigned long long count = pair_counts[li * right_targets.count + ri]; - if (count == 0) continue; - fprintf(out, "%s\t%s\t%llu\n", left_targets.records[li].id, right_targets.records[ri].id, count); - } - } - - if (summary_path != NULL) { - summary = open_output_file(summary_path); - if (summary == NULL) { - fprintf(stderr, "failed to open pair-count summary\n"); - goto done; - } - fprintf(summary, - "{\n \"workflow\": \"pair-count\",\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"left_start\": %zu,\n \"left_length\": %zu,\n \"right_start\": %zu,\n \"right_length\": %zu,\n \"n_left_targets\": %zu,\n \"n_right_targets\": %zu,\n \"total_reads\": %llu,\n \"assigned_pairs\": %llu,\n \"pair_ambiguous\": %llu,\n \"left_unmatched\": %llu,\n \"right_unmatched\": %llu,\n \"invalid\": %llu,\n \"candidates_considered\": %llu,\n \"candidates_verified\": %llu\n}\n", - k, metric_name(metric), ambiguity_policy_name(assignment_policy), qdaln_alphabet_policy(), left_start, left_len, right_start, right_len, - left_targets.count, right_targets.count, stats.total_reads, stats.assigned_pairs, - stats.pair_ambiguous, stats.left_unmatched, stats.right_unmatched, stats.invalid, - stats.candidates_considered, stats.candidates_verified); - } - - rc = 0; - -done: - if (out != NULL) fclose(out); - if (summary != NULL) fclose(summary); - if (assignments != NULL) fclose(assignments); - fastq_reader_close(&reader); - qdaln_index_free(left_index); - qdaln_index_free(right_index); - free(left_ptrs); - free(right_ptrs); - free(left_lens); - free(right_lens); - free(pair_counts); - free_table(&left_targets); - free_table(&right_targets); - return rc; -} - -static FILE *open_demux_target_file(FILE **files, const seq_table *targets, size_t target_index, const char *out_dir) { - if (files[target_index] != NULL) return files[target_index]; - char safe_id[512]; - sanitize_filename(targets->records[target_index].id, safe_id, sizeof(safe_id)); - char name[600]; - int n = snprintf(name, sizeof(name), "%s.fastq", safe_id); - if (n < 0 || (size_t)n >= sizeof(name)) return NULL; - char path[4096]; - if (path_join(path, sizeof(path), out_dir, name) != 0) return NULL; - files[target_index] = open_output_file(path); - return files[target_index]; -} - -static int run_demux(const char *argv0, int argc, char **argv) { - const char *barcodes_path = NULL; - const char *reads_path = NULL; - const char *out_dir = NULL; - const char *summary_path = NULL; - const char *assignments_path = NULL; - const char *ambiguous_path = NULL; - const char *unmatched_path = NULL; - count_metric metric = COUNT_METRIC_LEVENSHTEIN; - size_t barcode_start = 0; - size_t barcode_len = 0; - int auto_barcode_len = 0; - size_t indel_window = 0; - int max_correction_qual = -1; - int k = -1; - ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if ((strcmp(arg, "--barcodes") == 0 || strcmp(arg, "--targets") == 0) && i < argc) { - barcodes_path = argv[i++]; - } else if (strcmp(arg, "--reads") == 0 && i < argc) { - reads_path = argv[i++]; - } else if ((strcmp(arg, "--barcode-start") == 0 || strcmp(arg, "--target-start") == 0) && i < argc) { - if (parse_size_value(argv[i++], &barcode_start) != 0) { - usage(argv0); - return 2; - } - } else if ((strcmp(arg, "--barcode-length") == 0 || strcmp(arg, "--target-length") == 0) && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "auto") == 0) { - auto_barcode_len = 1; - barcode_len = 0; - } else if (parse_size_value(value, &barcode_len) != 0 || barcode_len == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 2) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--metric") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "hamming") == 0) { - metric = COUNT_METRIC_HAMMING; - } else if (strcmp(value, "levenshtein") == 0) { - metric = COUNT_METRIC_LEVENSHTEIN; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "radius") == 0) { - assignment_policy = AMBIGUITY_POLICY_RADIUS; - } else if (strcmp(value, "best") == 0) { - assignment_policy = AMBIGUITY_POLICY_BEST; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--indel-window") == 0 && i < argc) { - if (parse_size_value(argv[i++], &indel_window) != 0 || indel_window > 1) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--max-correction-qual") == 0 && i < argc) { - if (parse_int_value(argv[i++], &max_correction_qual) != 0 || - max_correction_qual < 0 || max_correction_qual > 93) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--out-dir") == 0 && i < argc) { - out_dir = argv[i++]; - } else if (strcmp(arg, "--summary") == 0 && i < argc) { - summary_path = argv[i++]; - } else if (strcmp(arg, "--qc") == 0 && i < argc) { - summary_path = argv[i++]; - } else if (strcmp(arg, "--assignments") == 0 && i < argc) { - assignments_path = argv[i++]; - } else if (strcmp(arg, "--ambiguous-out") == 0 && i < argc) { - ambiguous_path = argv[i++]; - } else if (strcmp(arg, "--unmatched-out") == 0 && i < argc) { - unmatched_path = argv[i++]; - } else { - usage(argv0); - return 2; - } - } - - if (barcodes_path == NULL || reads_path == NULL || out_dir == NULL || (barcode_len == 0 && !auto_barcode_len) || k < 0) { - usage(argv0); - return 2; - } - if (metric == COUNT_METRIC_HAMMING && indel_window != 0) { - fprintf(stderr, "--indel-window is only valid with --metric levenshtein\n"); - return 2; - } - if (metric == COUNT_METRIC_HAMMING && k > 1) { - fprintf(stderr, "--k 2 is only valid with --metric levenshtein\n"); - return 2; - } - if (indel_window != 0 && k != 1) { - fprintf(stderr, "--indel-window requires --k 1\n"); - return 2; - } - - seq_table targets = {0}; - fastq_reader reader = {0}; - qdaln_index *index = NULL; - hamming_lookup hlookup = {0}; - levenshtein1_lookup levlookup = {0}; - const char **target_ptrs = NULL; - size_t *target_lens = NULL; - size_t *auto_barcode_lens = NULL; - size_t auto_barcode_lens_count = 0; - size_t fixed_barcode_lens[1] = {0}; - const size_t *barcode_lens = NULL; - size_t barcode_lens_count = 0; - FILE **target_files = NULL; - FILE *assignments = NULL; - FILE *ambiguous_out = NULL; - FILE *unmatched_out = NULL; - count_stats stats = {0}; - unsigned long long *target_counts = NULL; - const char *assignment_engine = "generic_indexed"; - int rc = 1; - - if (read_target_table(barcodes_path, &targets) != 0) { - fprintf(stderr, "failed to read barcodes\n"); - goto done; - } - int barcode_id_check = validate_unique_seq_ids(&targets, "barcode"); - if (barcode_id_check != 0) { - if (barcode_id_check == -1) fprintf(stderr, "out of memory\n"); - goto done; - } - if (!auto_barcode_len && metric == COUNT_METRIC_HAMMING && !all_targets_have_length(&targets, barcode_len)) { - fprintf(stderr, "--metric hamming requires every barcode to have --barcode-length bases\n"); - goto done; - } - if (auto_barcode_len) { - if (collect_target_lengths(&targets, &auto_barcode_lens, &auto_barcode_lens_count) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - barcode_lens = auto_barcode_lens; - barcode_lens_count = auto_barcode_lens_count; - } else { - fixed_barcode_lens[0] = barcode_len; - barcode_lens = fixed_barcode_lens; - barcode_lens_count = 1; - } - if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - index = qdaln_index_build(target_ptrs, target_lens, targets.count); - if (index == NULL) { - fprintf(stderr, "failed to build barcode index\n"); - goto done; - } - if (!auto_barcode_len && metric == COUNT_METRIC_HAMMING && indel_window == 0 && (k == 0 || k == 1) && - barcode_len <= 32) { - int lookup_rc = k == 0 ? build_hamming_exact_lookup(&targets, barcode_len, &hlookup) - : build_hamming_lookup(&targets, barcode_len, &hlookup); - if (lookup_rc < 0) { - fprintf(stderr, "failed to build barcode Hamming lookup\n"); - goto done; - } - if (hlookup.ready) assignment_engine = k == 0 ? "hamming_exact_lookup_direct" : "hamming_k1_lookup_direct"; - } - if (!auto_barcode_len && metric == COUNT_METRIC_LEVENSHTEIN && indel_window == 1 && k == 1 && - assignment_policy == AMBIGUITY_POLICY_BEST && barcode_len <= 31) { - int lookup_rc = build_levenshtein1_lookup(&targets, barcode_len, &levlookup); - if (lookup_rc < 0) { - fprintf(stderr, "failed to build barcode Levenshtein lookup\n"); - goto done; - } - if (levlookup.ready) assignment_engine = "levenshtein_k1_lookup_direct"; - } - int filename_check = validate_unique_sanitized_filenames(&targets); - if (filename_check != 0) { - if (filename_check == -1) fprintf(stderr, "out of memory\n"); - goto done; - } - if (ensure_dir(out_dir) != 0) { - fprintf(stderr, "failed to create output directory\n"); - goto done; - } - target_files = (FILE **)calloc(targets.count == 0 ? 1 : targets.count, sizeof(FILE *)); - target_counts = (unsigned long long *)calloc(targets.count == 0 ? 1 : targets.count, sizeof(unsigned long long)); - if (target_files == NULL || target_counts == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - if (fastq_reader_open(&reader, reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - goto done; - } - if (assignments_path != NULL) { - assignments = open_output_file(assignments_path); - if (assignments == NULL) { - fprintf(stderr, "failed to open assignments output\n"); - goto done; - } - fprintf(assignments, "read_id\tobserved_barcode\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\n"); - } - if (ambiguous_path != NULL) { - ambiguous_out = open_output_file(ambiguous_path); - if (ambiguous_out == NULL) { - fprintf(stderr, "failed to open ambiguous FASTQ output\n"); - goto done; - } - } - if (unmatched_path != NULL) { - unmatched_out = open_output_file(unmatched_path); - if (unmatched_out == NULL) { - fprintf(stderr, "failed to open unmatched FASTQ output\n"); - goto done; - } - } - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - char read_id[8192]; - char observed[8192]; - int got = 0; - size_t seq_len = 0; - while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - fastq_read_id(header, read_id, sizeof(read_id)); - qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; - qdaln_index_stats istats = {0, 0}; - observed[0] = '\0'; - ++stats.total; - - int handled = 0; - if (hlookup.ready) { - int exact_merge = assignment_policy == AMBIGUITY_POLICY_RADIUS || - assignments != NULL || ambiguous_out != NULL || unmatched_out != NULL; - int lookup_rc = assign_hamming_lookup_offsets(&hlookup, seq, seq_len, NULL, barcode_start, k, - &result, &istats, observed, sizeof(observed), - exact_merge); - if (lookup_rc < 0) { - fprintf(stderr, "FASTQ assignment failed\n"); - goto done; - } - handled = lookup_rc; - } - if (!handled && levlookup.ready) { - int lookup_rc = assign_levenshtein1_lookup_offset(&levlookup, seq, seq_len, barcode_start, &result, - &istats, observed, sizeof(observed)); - if (lookup_rc < 0) { - fprintf(stderr, "FASTQ assignment failed\n"); - goto done; - } - handled = lookup_rc; - } - if (!handled && - assign_count_length_set(index, seq, seq_len, barcode_start, barcode_lens, barcode_lens_count, k, metric, - indel_window, &result, &istats, observed, sizeof(observed), 0) != 0) { - fprintf(stderr, "FASTQ assignment failed\n"); - goto done; - } - apply_ambiguity_policy(&result, assignment_policy); - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0 && result.best_distance > 0) { - seq_record *target = &targets.records[result.target_index]; - offset_list barcode_offset = {0}; - if (push_offset_unique(&barcode_offset, barcode_start) != 0) { - free_offset_list(&barcode_offset); - fprintf(stderr, "out of memory\n"); - goto done; - } - if (!quality_allows_unique_correction(seq, seq_len, qual, &barcode_offset, barcode_start, - target->len, metric, indel_window, k, observed, target, result, - max_correction_qual)) { - result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; - } - free_offset_list(&barcode_offset); - } - if (result.status != QDALN_MATCH_INVALID) { - stats.candidates_considered += (unsigned long long)istats.candidates_considered; - stats.candidates_verified += (unsigned long long)istats.candidates_verified; - } - - if (assignments != NULL) print_fastq_row(assignments, &targets, read_id, observed, result); - - if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { - FILE *target_out = open_demux_target_file(target_files, &targets, (size_t)result.target_index, out_dir); - if (target_out == NULL) { - fprintf(stderr, "failed to open per-barcode FASTQ output\n"); - goto done; - } - write_fastq_record(target_out, header, seq, plus, qual); - ++target_counts[result.target_index]; - ++stats.unique; - if (result.best_distance == 0) ++stats.exact; - else ++stats.corrected; - } else if (result.status == QDALN_MATCH_AMBIGUOUS) { - ++stats.ambiguous; - if (ambiguous_out != NULL) write_fastq_record(ambiguous_out, header, seq, plus, qual); - } else if (result.status == QDALN_MATCH_NONE) { - ++stats.unmatched; - if (unmatched_out != NULL) write_fastq_record(unmatched_out, header, seq, plus, qual); - } else { - ++stats.invalid; - if (unmatched_out != NULL) write_fastq_record(unmatched_out, header, seq, plus, qual); - } - } - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - goto done; - } - - if (summary_path != NULL) { - FILE *summary = open_output_file(summary_path); - if (summary == NULL) { - fprintf(stderr, "failed to open summary output\n"); - goto done; - } - unsigned long long top_count = 0; - size_t top_target = 0; - size_t nonempty = 0; - for (size_t i = 0; i < targets.count; ++i) { - if (target_counts[i] != 0) ++nonempty; - if (target_counts[i] > top_count) { - top_count = target_counts[i]; - top_target = i; - } - } - fprintf(summary, - "{\n \"workflow\": \"demux\",\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"assignment_engine\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"max_correction_qual\": ", - k, metric_name(metric), ambiguity_policy_name(assignment_policy), assignment_engine, - qdaln_alphabet_policy()); - if (max_correction_qual >= 0) { - fprintf(summary, "%d", max_correction_qual); - } else { - fprintf(summary, "null"); - } - fprintf(summary, - ",\n \"indel_window\": %zu,\n \"barcode_start\": %zu,\n \"barcode_length\": %zu,\n \"barcode_length_mode\": \"%s\",\n \"barcode_lengths\": [", - indel_window, barcode_start, barcode_len, auto_barcode_len ? "auto" : "fixed"); - for (size_t i = 0; i < barcode_lens_count; ++i) { - fprintf(summary, "%s%zu", i == 0 ? "" : ", ", barcode_lens[i]); - } - fprintf(summary, - "],\n \"n_barcodes\": %zu,\n \"total_reads\": %llu,\n \"assigned_unique\": %llu,\n \"assigned_exact\": %llu,\n \"assigned_corrected\": %llu,\n \"ambiguous\": %llu,\n \"unmatched\": %llu,\n \"invalid\": %llu,\n \"nonempty_outputs\": %zu,\n \"top_barcode_id\": \"%s\",\n \"top_barcode_count\": %llu,\n \"candidates_considered\": %llu,\n \"candidates_verified\": %llu\n}\n", - targets.count, stats.total, - stats.unique, stats.exact, stats.corrected, stats.ambiguous, stats.unmatched, stats.invalid, - nonempty, targets.count == 0 ? "" : targets.records[top_target].id, top_count, - stats.candidates_considered, stats.candidates_verified); - fclose(summary); - } - - rc = 0; - -done: - if (target_files != NULL) { - for (size_t i = 0; i < targets.count; ++i) { - if (target_files[i] != NULL) fclose(target_files[i]); - } - } - if (assignments != NULL) fclose(assignments); - if (ambiguous_out != NULL) fclose(ambiguous_out); - if (unmatched_out != NULL) fclose(unmatched_out); - fastq_reader_close(&reader); - qdaln_index_free(index); - free_hamming_lookup(&hlookup); - free_levenshtein1_lookup(&levlookup); - free(target_ptrs); - free(target_lens); - free(auto_barcode_lens); - free(target_files); - free(target_counts); - free_table(&targets); - return rc; -} - -typedef struct bcl_sample { - char *id; - char *name; - char *index1; - char *index2; - int lane; - size_t output_index; - int is_alias; - unsigned long long assigned; -} bcl_sample; - -typedef struct bcl_sample_table { - bcl_sample *items; - size_t count; - size_t cap; -} bcl_sample_table; - -typedef struct bcl_read_info { - int number; - size_t cycles; - int indexed; - size_t start_cycle; -} bcl_read_info; - -typedef struct bcl_run_info { - bcl_read_info reads[16]; - size_t read_count; - size_t total_cycles; -} bcl_run_info; - -typedef struct bcl_unknown_barcode { - char *index; - unsigned long long count; -} bcl_unknown_barcode; - -typedef struct bcl_unknown_table { - bcl_unknown_barcode *items; - size_t count; - size_t cap; -} bcl_unknown_table; - -typedef struct text_buffer { - char *data; - size_t len; - size_t cap; -} text_buffer; - -static void free_bcl_unknowns(bcl_unknown_table *table) { - for (size_t i = 0; i < table->count; ++i) free(table->items[i].index); - free(table->items); - table->items = NULL; - table->count = 0; - table->cap = 0; -} - -static int add_bcl_unknown_count(bcl_unknown_table *table, const char *index, unsigned long long count) { - if (count == 0) return 0; - for (size_t i = 0; i < table->count; ++i) { - if (strcmp(table->items[i].index, index) == 0) { - table->items[i].count += count; - return 0; - } - } - if (table->count == table->cap) { - size_t next_cap = table->cap == 0 ? 64 : table->cap * 2; - bcl_unknown_barcode *next = (bcl_unknown_barcode *)realloc(table->items, next_cap * sizeof(bcl_unknown_barcode)); - if (next == NULL) return -1; - table->items = next; - table->cap = next_cap; - } - table->items[table->count].index = xstrndup(index, strlen(index)); - if (table->items[table->count].index == NULL) return -1; - table->items[table->count].count = count; - ++table->count; - return 0; -} - -static int add_bcl_unknown(bcl_unknown_table *table, const char *index) { - return add_bcl_unknown_count(table, index, 1); -} - -static int cmp_bcl_unknown_desc(const void *a, const void *b) { - const bcl_unknown_barcode *aa = (const bcl_unknown_barcode *)a; - const bcl_unknown_barcode *bb = (const bcl_unknown_barcode *)b; - if (aa->count < bb->count) return 1; - if (aa->count > bb->count) return -1; - return strcmp(aa->index, bb->index); -} - -static int merge_bcl_unknowns(bcl_unknown_table *dst, const bcl_unknown_table *src) { - for (size_t i = 0; i < src->count; ++i) { - if (add_bcl_unknown_count(dst, src->items[i].index, src->items[i].count) != 0) return -1; - } - return 0; -} - -static void free_text_buffer(text_buffer *buf) { - free(buf->data); - buf->data = NULL; - buf->len = 0; - buf->cap = 0; -} - -static int text_buffer_reserve(text_buffer *buf, size_t extra) { - if (extra > SIZE_MAX - buf->len) return -1; - size_t need = buf->len + extra; - if (need <= buf->cap) return 0; - size_t next = buf->cap == 0 ? 65536 : buf->cap; - while (next < need) { - if (next > SIZE_MAX / 2) { - next = need; - break; - } - next *= 2; - } - char *p = (char *)realloc(buf->data, next); - if (p == NULL) return -1; - buf->data = p; - buf->cap = next; - return 0; -} - -static int text_buffer_append(text_buffer *buf, const char *s, size_t n) { - if (text_buffer_reserve(buf, n) != 0) return -1; - memcpy(buf->data + buf->len, s, n); - buf->len += n; - return 0; -} - -static int gzwrite_all(gzFile gz, const char *data, size_t len) { - size_t written = 0; - while (written < len) { - size_t remaining = len - written; - unsigned int chunk = remaining > (size_t)UINT_MAX ? UINT_MAX : (unsigned int)remaining; - int rc = gzwrite(gz, data + written, chunk); - if (rc <= 0 || (unsigned int)rc != chunk) return -1; - written += (size_t)rc; - } - return 0; -} - -static void free_bcl_samples(bcl_sample_table *table) { - for (size_t i = 0; i < table->count; ++i) { - free(table->items[i].id); - free(table->items[i].name); - free(table->items[i].index1); - free(table->items[i].index2); - } - free(table->items); - table->items = NULL; - table->count = 0; - table->cap = 0; -} - -static int push_bcl_sample(bcl_sample_table *table, const char *id, const char *name, - const char *index1, const char *index2, int lane) { - size_t output_index = table->count; - int is_alias = 0; - for (size_t i = 0; i < table->count; ++i) { - bcl_sample *existing = &table->items[i]; - if (strcmp(existing->id, id) == 0 && existing->lane == lane) { - output_index = existing->output_index; - is_alias = 1; - break; - } - } - if (table->count == table->cap) { - size_t next_cap = table->cap == 0 ? 16 : table->cap * 2; - bcl_sample *next = (bcl_sample *)realloc(table->items, next_cap * sizeof(bcl_sample)); - if (next == NULL) return -1; - table->items = next; - table->cap = next_cap; - } - bcl_sample *s = &table->items[table->count]; - memset(s, 0, sizeof(*s)); - s->id = xstrndup(id, strlen(id)); - s->name = xstrndup(name != NULL && name[0] != '\0' ? name : id, strlen(name != NULL && name[0] != '\0' ? name : id)); - s->index1 = xstrndup(index1, strlen(index1)); - s->index2 = xstrndup(index2 != NULL ? index2 : "", strlen(index2 != NULL ? index2 : "")); - s->lane = lane; - s->output_index = output_index; - s->is_alias = is_alias; - if (s->id == NULL || s->name == NULL || s->index1 == NULL || s->index2 == NULL) return -1; - uppercase_ascii(s->index1); - uppercase_ascii(s->index2); - ++table->count; - return 0; -} - -static char *read_text_file(const char *path) { - FILE *fp = fopen(path, "rb"); - if (fp == NULL) return NULL; - if (fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - return NULL; - } - long n = ftell(fp); - if (n < 0) { - fclose(fp); - return NULL; - } - rewind(fp); - char *text = (char *)malloc((size_t)n + 1); - if (text == NULL) { - fclose(fp); - return NULL; - } - if (fread(text, 1, (size_t)n, fp) != (size_t)n) { - free(text); - fclose(fp); - return NULL; - } - text[n] = '\0'; - fclose(fp); - return text; -} - -static int xml_attr_value(const char *tag, const char *name, char *out, size_t out_cap) { - char pattern[64]; - snprintf(pattern, sizeof(pattern), "%s=\"", name); - const char *p = strstr(tag, pattern); - if (p == NULL) return -1; - p += strlen(pattern); - const char *end = strchr(p, '"'); - if (end == NULL) return -1; - size_t n = (size_t)(end - p); - if (n >= out_cap) n = out_cap - 1; - memcpy(out, p, n); - out[n] = '\0'; - return 0; -} - -static int parse_run_info(const char *run_folder, bcl_run_info *info) { - char path[4096]; - if (path_join(path, sizeof(path), run_folder, "RunInfo.xml") != 0) return -1; - char *xml = read_text_file(path); - if (xml == NULL) return -1; - memset(info, 0, sizeof(*info)); - const char *p = xml; - while ((p = strstr(p, "'); - if (end == NULL) { - free(xml); - return -1; - } - char tag[1024]; - size_t tag_len = (size_t)(end - p + 1); - if (tag_len >= sizeof(tag)) tag_len = sizeof(tag) - 1; - memcpy(tag, p, tag_len); - tag[tag_len] = '\0'; - char number[32] = ""; - char cycles[32] = ""; - char indexed[32] = ""; - if (xml_attr_value(tag, "Number", number, sizeof(number)) == 0 && - xml_attr_value(tag, "NumCycles", cycles, sizeof(cycles)) == 0 && - xml_attr_value(tag, "IsIndexedRead", indexed, sizeof(indexed)) == 0) { - if (info->read_count >= 16) { - free(xml); - return -1; - } - int parsed_number = 0; - size_t parsed_cycles = 0; - if (parse_int_value(number, &parsed_number) != 0 || parsed_number < 1 || - parse_size_value(cycles, &parsed_cycles) != 0 || parsed_cycles == 0 || - parsed_cycles > MAX_BCL_READ_CYCLES || - info->total_cycles > MAX_BCL_TOTAL_CYCLES - parsed_cycles) { - free(xml); - return -1; - } - if (!(indexed[0] == 'Y' || indexed[0] == 'y' || indexed[0] == 'N' || indexed[0] == 'n') || - indexed[1] != '\0') { - free(xml); - return -1; - } - bcl_read_info *r = &info->reads[info->read_count++]; - r->number = parsed_number; - r->cycles = parsed_cycles; - r->indexed = indexed[0] == 'Y' || indexed[0] == 'y'; - r->start_cycle = info->total_cycles + 1; - info->total_cycles += r->cycles; - } - p = end + 1; - } - free(xml); - return info->read_count == 0 ? -1 : 0; -} - -static int read_bcl_sample_sheet(const char *path, bcl_sample_table *samples) { - FILE *fp = fopen(path, "r"); - if (fp == NULL) return -1; - char buf[16384]; - int in_data = 0; - int have_header = 0; - int id_col = -1; - int name_col = -1; - int index_col = -1; - int index2_col = -1; - int lane_col = -1; - size_t row = 0; - while (fgets(buf, sizeof(buf), fp) != NULL) { - ++row; - trim_line(buf); - if (buf[0] == '\0') continue; - if (buf[0] == '[') { - in_data = field_eq(buf, "[Data]") || field_eq(buf, "[BCLConvert_Data]"); - have_header = 0; - continue; - } - if (!in_data) continue; - char *fields[64]; - size_t nf = split_fields(buf, ',', fields, 64); - if (!have_header) { - id_col = find_column(fields, nf, "Sample_ID", "SampleID", "Sample_ID"); - if (id_col < 0) id_col = find_column(fields, nf, "Sample_Project", "SampleName", "Sample_Name"); - name_col = find_column(fields, nf, "Sample_Name", "SampleName", "sample_name"); - index_col = find_column(fields, nf, "index", "Index", "Index1"); - index2_col = find_column(fields, nf, "index2", "Index2", "index_2"); - lane_col = find_column(fields, nf, "Lane", "lane", NULL); - have_header = 1; - if (id_col < 0 || index_col < 0) { - fclose(fp); - return -1; - } - continue; - } - if ((size_t)id_col >= nf || (size_t)index_col >= nf || fields[id_col][0] == '\0' || fields[index_col][0] == '\0') { - fprintf(stderr, "%s:%zu: BCL sample sheet Sample_ID and index must be non-empty\n", path, row); - fclose(fp); - return -1; - } - const char *name = (name_col >= 0 && (size_t)name_col < nf) ? fields[name_col] : fields[id_col]; - const char *index2 = (index2_col >= 0 && (size_t)index2_col < nf) ? fields[index2_col] : ""; - int lane = 0; - if (lane_col >= 0 && (size_t)lane_col < nf && fields[lane_col][0] != '\0') { - if (parse_int_value(fields[lane_col], &lane) != 0 || lane < 1) { - fprintf(stderr, "%s:%zu: BCL sample sheet Lane must be a positive integer\n", path, row); - fclose(fp); - return -1; - } - if (lane != 1) { - fprintf(stderr, "%s:%zu: classic BCL demux currently supports sample-sheet Lane 1 only\n", path, row); - fclose(fp); - return -1; - } - } - if (push_bcl_sample(samples, fields[id_col], name, fields[index_col], index2, lane) != 0) { - fclose(fp); - return -1; - } - } - fclose(fp); - return samples->count == 0 ? -1 : 0; -} - -static int hamming_distance_limit(const char *a, const char *b, int limit) { - size_t na = strlen(a); - size_t nb = strlen(b); - if (na != nb) return limit + 1; - int d = 0; - for (size_t i = 0; i < na; ++i) { - if (a[i] != b[i] && ++d > limit) return d; - } - return d; -} - -static int assign_bcl_sample(const bcl_sample_table *samples, int lane, const char *index1, const char *index2, - int k1, int k2, int *match_count_out) { - int best = -1; - int best_d = 1000000; - size_t best_output = (size_t)-1; - int matches = 0; - for (size_t i = 0; i < samples->count; ++i) { - const bcl_sample *s = &samples->items[i]; - if (s->lane != 0 && s->lane != lane) continue; - int d1 = hamming_distance_limit(index1, s->index1, k1); - if (d1 > k1) continue; - int d2 = 0; - if (s->index2[0] != '\0' || index2[0] != '\0') { - d2 = hamming_distance_limit(index2, s->index2, k2); - if (d2 > k2) continue; - } - int d = d1 + d2; - if (d < best_d) { - best_d = d; - best = (int)i; - best_output = s->output_index; - matches = 1; - } else if (d == best_d && s->output_index != best_output) { - ++matches; - } - } - *match_count_out = matches; - return matches == 1 ? best : -1; -} - -static int read_u32_le(const unsigned char *p) { - return (int)((unsigned int)p[0] | ((unsigned int)p[1] << 8) | ((unsigned int)p[2] << 16) | ((unsigned int)p[3] << 24)); -} - -static int path_exists(const char *path) { - struct stat st; - return stat(path, &st) == 0; -} - -static int build_bcl_path(char *out, size_t out_cap, const char *basecalls, int lane, size_t cycle, const char *tile) { - int n = snprintf(out, out_cap, "%s/L%03d/C%zu.1/s_%d_%s.bcl.gz", basecalls, lane, cycle, lane, tile); - if (n < 0 || (size_t)n >= out_cap) return -1; - if (path_exists(out)) return 0; - n = snprintf(out, out_cap, "%s/L%03d/C%zu.1/s_%d_%s.bcl", basecalls, lane, cycle, lane, tile); - if (n < 0 || (size_t)n >= out_cap) return -1; - return path_exists(out) ? 0 : -1; -} - -static int read_bcl_cycle(const char *path, unsigned char **bytes_out, size_t *count_out) { - gzFile gz = gzopen(path, "rb"); - if (gz == NULL) return -1; - unsigned char header[4]; - if (gzread(gz, header, 4) != 4) { - gzclose(gz); - return -1; - } - int n = read_u32_le(header); - if (n < 0 || n > MAX_BCL_CYCLE_CLUSTERS) { - gzclose(gz); - return -1; - } - unsigned char *bytes = (unsigned char *)malloc((size_t)n == 0 ? 1 : (size_t)n); - if (bytes == NULL) { - gzclose(gz); - return -1; - } - if (n != 0 && gzread(gz, bytes, (unsigned int)n) != n) { - free(bytes); - gzclose(gz); - return -1; - } - gzclose(gz); - *bytes_out = bytes; - *count_out = (size_t)n; - return 0; -} - -static int read_filter_file(const char *basecalls, int lane, const char *tile, unsigned char **pf_out, size_t *count_out) { - char path[4096]; - int n = snprintf(path, sizeof(path), "%s/L%03d/s_%d_%s.filter", basecalls, lane, lane, tile); - if (n < 0 || (size_t)n >= sizeof(path)) return -1; - if (!path_exists(path)) { - *pf_out = NULL; - *count_out = 0; - return 0; - } - FILE *fp = fopen(path, "rb"); - if (fp == NULL) return -1; - unsigned char header[8]; - if (fread(header, 1, 8, fp) != 8) { - fclose(fp); - return -1; - } - int count = read_u32_le(header + 4); - if (count < 0 || count > MAX_BCL_CYCLE_CLUSTERS) { - fclose(fp); - return -1; - } - unsigned char *pf = (unsigned char *)malloc((size_t)count == 0 ? 1 : (size_t)count); - if (pf == NULL) { - fclose(fp); - return -1; - } - if (count != 0 && fread(pf, 1, (size_t)count, fp) != (size_t)count) { - free(pf); - fclose(fp); - return -1; - } - fclose(fp); - *pf_out = pf; - *count_out = (size_t)count; - return 0; -} - -static char bcl_base(unsigned char b) { - if (b == 0) return 'N'; - switch (b & 3u) { - case 0: return 'A'; - case 1: return 'C'; - case 2: return 'G'; - default: return 'T'; - } -} - -static char bcl_qual(unsigned char b) { - if (b == 0) return '#'; - unsigned int q = b >> 2; - if (q > 93) q = 93; - return (char)(q + 33); -} - -static int collect_tiles(const char *basecalls, int lane, char ***tiles_out, size_t *tile_count_out) { - char path[4096]; - int n = snprintf(path, sizeof(path), "%s/L%03d/C1.1", basecalls, lane); - if (n < 0 || (size_t)n >= sizeof(path)) return -1; - DIR *dir = opendir(path); - if (dir == NULL) return -1; - string_list tiles = {0}; - struct dirent *ent; - char prefix[32]; - n = snprintf(prefix, sizeof(prefix), "s_%d_", lane); - if (n < 0 || (size_t)n >= sizeof(prefix)) { - closedir(dir); - return -1; - } - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, prefix, strlen(prefix)) != 0) continue; - char *start = ent->d_name + strlen(prefix); - char *bcl = strstr(start, ".bcl"); - if (bcl == NULL) continue; - size_t len = (size_t)(bcl - start); - char tile[128]; - if (len == 0 || len >= sizeof(tile)) continue; - memcpy(tile, start, len); - tile[len] = '\0'; - if (push_string(&tiles, tile) != 0) { - closedir(dir); - free_string_list(&tiles); - return -1; - } - } - closedir(dir); - *tiles_out = tiles.items; - *tile_count_out = tiles.count; - return tiles.count == 0 ? -1 : 0; -} - -static gzFile open_bcl_fastq(const char *out_dir, const char *sample_id, size_t sample_number, int lane, - char read_kind, int read_number, int gzip_level) { - char safe_id[512]; - sanitize_filename(sample_id, safe_id, sizeof(safe_id)); - char name[700]; - int n = snprintf(name, sizeof(name), "%s_S%zu_L%03d_%c%d_001.fastq.gz", safe_id, sample_number, - lane, read_kind, read_number); - if (n < 0 || (size_t)n >= sizeof(name)) return NULL; - char path[4096]; - if (path_join(path, sizeof(path), out_dir, name) != 0) return NULL; - char mode[8]; - snprintf(mode, sizeof(mode), "wb%d", gzip_level); - gzFile gz = gzopen(path, mode); - if (gz != NULL) gzbuffer(gz, 1024 * 1024); - return gz; -} - -static int bcl_output_enabled(const bcl_read_info *read, int emit_index_fastqs) { - return !read->indexed || emit_index_fastqs; -} - -static char bcl_output_kind(const bcl_read_info *read) { - return read->indexed ? 'I' : 'R'; -} - -static int bcl_output_number(const bcl_run_info *run, size_t read_i) { - int n = 0; - for (size_t i = 0; i <= read_i && i < run->read_count; ++i) { - if (run->reads[i].indexed == run->reads[read_i].indexed) ++n; - } - return n == 0 ? 1 : n; -} - -static gzFile *bcl_output_slot(gzFile *files, size_t sample_i, size_t read_i, size_t read_count) { - return &files[sample_i * read_count + read_i]; -} - -static int append_fastq_record(text_buffer *out, const char *header, const char *seq, const char *qual) { - char buf[20000]; - int n = snprintf(buf, sizeof(buf), "%s\n%s\n+\n%s\n", header, seq, qual); - if (n < 0 || (size_t)n >= sizeof(buf)) return -1; - return text_buffer_append(out, buf, (size_t)n); -} - -typedef struct bcl_block_result { - text_buffer *sample_buffers; - text_buffer *undetermined_buffers; - unsigned long long *sample_assigned; - unsigned long long passed_clusters; - unsigned long long filtered_clusters; - unsigned long long undetermined_reads; - bcl_unknown_table unknowns; - int error; -} bcl_block_result; - -typedef struct bcl_block_job { - const bcl_run_info *run; - const bcl_sample_table *samples; - unsigned char **cycles; - const unsigned char *pf; - const char *tile; - size_t start; - size_t end; - int k1; - int k2; - int emit_index_fastqs; - bcl_block_result *result; -} bcl_block_job; - -static void free_bcl_block_result(bcl_block_result *result, size_t sample_count, size_t read_count) { - if (result->sample_buffers != NULL) { - size_t n = 0; - if (checked_mul_size(sample_count, read_count, &n) != 0) n = 0; - for (size_t i = 0; i < n; ++i) free_text_buffer(&result->sample_buffers[i]); - } - if (result->undetermined_buffers != NULL) { - for (size_t i = 0; i < read_count; ++i) free_text_buffer(&result->undetermined_buffers[i]); - } - free(result->sample_buffers); - free(result->undetermined_buffers); - free(result->sample_assigned); - free_bcl_unknowns(&result->unknowns); - memset(result, 0, sizeof(*result)); -} - -static int init_bcl_block_result(bcl_block_result *result, size_t sample_count, size_t read_count) { - memset(result, 0, sizeof(*result)); - size_t sample_read_slots = 0; - if (checked_mul_size(sample_count, read_count, &sample_read_slots) != 0) return -1; - result->sample_buffers = (text_buffer *)calloc(alloc_count_or_one(sample_read_slots), sizeof(text_buffer)); - result->undetermined_buffers = (text_buffer *)calloc(alloc_count_or_one(read_count), sizeof(text_buffer)); - result->sample_assigned = (unsigned long long *)calloc(alloc_count_or_one(sample_count), sizeof(unsigned long long)); - if (result->sample_buffers == NULL || result->undetermined_buffers == NULL || result->sample_assigned == NULL) { - free_bcl_block_result(result, sample_count, read_count); - return -1; - } - return 0; -} - -static int process_bcl_block(const bcl_block_job *job) { - const bcl_run_info *run = job->run; - const bcl_sample_table *samples = job->samples; - bcl_block_result *result = job->result; - for (size_t cluster = job->start; cluster < job->end; ++cluster) { - if (job->pf != NULL && job->pf[cluster] == 0) { - ++result->filtered_clusters; - continue; - } - ++result->passed_clusters; - char index1[MAX_BCL_READ_CYCLES + 1] = ""; - char index2[MAX_BCL_READ_CYCLES + 1] = ""; - char seqs[16][MAX_BCL_READ_CYCLES + 1]; - char quals[16][MAX_BCL_READ_CYCLES + 1]; - memset(seqs, 0, sizeof(seqs)); - memset(quals, 0, sizeof(quals)); - int indexed_seen = 0; - for (size_t r = 0; r < run->read_count; ++r) { - const bcl_read_info *ri = &run->reads[r]; - char *seq_out = seqs[r]; - char *qual_out = quals[r]; - size_t cap = sizeof(seqs[r]); - for (size_t j = 0; j < ri->cycles && j + 1 < cap; ++j) { - unsigned char b = job->cycles[ri->start_cycle - 1 + j][cluster]; - seq_out[j] = bcl_base(b); - qual_out[j] = bcl_qual(b); - } - seq_out[ri->cycles] = '\0'; - qual_out[ri->cycles] = '\0'; - if (ri->indexed) { - if (indexed_seen == 0) { - int n = snprintf(index1, sizeof(index1), "%s", seq_out); - if (n < 0 || (size_t)n >= sizeof(index1)) return -1; - } else if (indexed_seen == 1) { - int n = snprintf(index2, sizeof(index2), "%s", seq_out); - if (n < 0 || (size_t)n >= sizeof(index2)) return -1; - } - ++indexed_seen; - } - } - int match_count = 0; - int sample_index = assign_bcl_sample(samples, 1, index1, index2, job->k1, job->k2, &match_count); - if (sample_index >= 0) { - const bcl_sample *s = &samples->items[sample_index]; - size_t out_i = s->output_index; - for (size_t r = 0; r < run->read_count; ++r) { - const bcl_read_info *ri = &run->reads[r]; - if (!bcl_output_enabled(ri, job->emit_index_fastqs)) continue; - char header[4096]; - int n = snprintf(header, sizeof(header), "@DOTMATCH:1:%s:%zu %d:N:0:%s%s%s", - job->tile, cluster + 1, bcl_output_number(run, r), - index1, index2[0] ? "+" : "", index2); - if (n < 0 || (size_t)n >= sizeof(header)) return -1; - text_buffer *buf = &result->sample_buffers[out_i * run->read_count + r]; - if (append_fastq_record(buf, header, seqs[r], quals[r]) != 0) return -1; - } - ++result->sample_assigned[out_i]; - } else { - char unknown_index[(MAX_BCL_READ_CYCLES * 2) + 2]; - int n = snprintf(unknown_index, sizeof(unknown_index), "%s%s%s", index1, index2[0] ? "+" : "", index2); - if (n < 0 || (size_t)n >= sizeof(unknown_index)) return -1; - if (add_bcl_unknown(&result->unknowns, unknown_index) != 0) return -1; - for (size_t r = 0; r < run->read_count; ++r) { - const bcl_read_info *ri = &run->reads[r]; - if (!bcl_output_enabled(ri, job->emit_index_fastqs)) continue; - char header[4096]; - n = snprintf(header, sizeof(header), "@DOTMATCH:1:%s:%zu %d:N:0:%s%s%s", - job->tile, cluster + 1, bcl_output_number(run, r), - index1, index2[0] ? "+" : "", index2); - if (n < 0 || (size_t)n >= sizeof(header)) return -1; - if (append_fastq_record(&result->undetermined_buffers[r], header, seqs[r], quals[r]) != 0) return -1; - } - ++result->undetermined_reads; - } - } - return 0; -} - -static void *bcl_block_worker(void *arg) { - bcl_block_job *job = (bcl_block_job *)arg; - job->result->error = process_bcl_block(job); - return NULL; -} - -static int write_bcl_block_result(const bcl_block_result *result, gzFile *sample_fastqs, gzFile *undetermined_fastqs, - size_t sample_count, size_t read_count) { - for (size_t i = 0; i < sample_count; ++i) { - for (size_t r = 0; r < read_count; ++r) { - const text_buffer *buf = &result->sample_buffers[i * read_count + r]; - if (buf->len == 0) continue; - gzFile gz = *bcl_output_slot(sample_fastqs, i, r, read_count); - if (gz != NULL && gzwrite_all(gz, buf->data, buf->len) != 0) return -1; - } - } - for (size_t r = 0; r < read_count; ++r) { - const text_buffer *buf = &result->undetermined_buffers[r]; - if (buf->len == 0) continue; - if (undetermined_fastqs[r] != NULL && gzwrite_all(undetermined_fastqs[r], buf->data, buf->len) != 0) return -1; - } - return 0; -} - -static int parse_mismatches(const char *s, int *k1, int *k2) { - char *comma = strchr(s, ','); - if (comma == NULL) { - int k = 0; - if (parse_int_value(s, &k) != 0) return -1; - if (k < 0 || k > 1) return -1; - *k1 = k; - *k2 = k; - return 0; - } - char left[16]; - size_t n = (size_t)(comma - s); - if (n >= sizeof(left)) return -1; - memcpy(left, s, n); - left[n] = '\0'; - int a = 0; - int b = 0; - if (parse_int_value(left, &a) != 0 || parse_int_value(comma + 1, &b) != 0) return -1; - if (a < 0 || a > 1 || b < 0 || b > 1) return -1; - *k1 = a; - *k2 = b; - return 0; -} - -static int run_bcl_demux(const char *argv0, int argc, char **argv) { - const char *run_folder = NULL; - const char *sample_sheet = NULL; - const char *out_dir = NULL; - const char *summary_path = NULL; - const char *mismatches = "1"; - const char *lanes = "1"; - int k1 = 1; - int k2 = 1; - int emit_index_fastqs = 0; - size_t requested_threads = 0; - int gzip_level = 1; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if (strcmp(arg, "--run-folder") == 0 && i < argc) { - run_folder = argv[i++]; - } else if (strcmp(arg, "--sample-sheet") == 0 && i < argc) { - sample_sheet = argv[i++]; - } else if (strcmp(arg, "--out-dir") == 0 && i < argc) { - out_dir = argv[i++]; - } else if (strcmp(arg, "--summary") == 0 && i < argc) { - summary_path = argv[i++]; - } else if (strcmp(arg, "--barcode-mismatches") == 0 && i < argc) { - mismatches = argv[i++]; - } else if (strcmp(arg, "--emit-index-fastqs") == 0) { - emit_index_fastqs = 1; - } else if (strcmp(arg, "--threads") == 0 && i < argc) { - if (parse_size_value(argv[i++], &requested_threads) != 0) { - fprintf(stderr, "invalid --threads value\n"); - return 2; - } - } else if (strcmp(arg, "--gzip-level") == 0 && i < argc) { - if (parse_int_value(argv[i++], &gzip_level) != 0 || gzip_level < 0 || gzip_level > 9) { - fprintf(stderr, "invalid --gzip-level value\n"); - return 2; - } - } else if (strcmp(arg, "--lanes") == 0 && i < argc) { - lanes = argv[i++]; - } else if (strcmp(arg, "--interop-dir") == 0 && i < argc) { - i++; - } else { - usage(argv0); - return 2; - } - } - if (run_folder == NULL || sample_sheet == NULL || out_dir == NULL || parse_mismatches(mismatches, &k1, &k2) != 0) { - usage(argv0); - return 2; - } - if (strcmp(lanes, "1") != 0 && strcmp(lanes, "001") != 0) { - fprintf(stderr, "classic BCL demux currently supports lane 1 only; rerun with --lanes 1 or split the run externally\n"); - return 2; - } - - char basecalls[4096]; - int basecalls_n = snprintf(basecalls, sizeof(basecalls), "%s/Data/Intensities/BaseCalls", run_folder); - if (basecalls_n < 0 || (size_t)basecalls_n >= sizeof(basecalls)) { - fprintf(stderr, "run folder path is too long\n"); - return 2; - } - bcl_run_info run = {0}; - bcl_sample_table samples = {0}; - gzFile *sample_fastqs = NULL; - gzFile *undetermined_fastqs = NULL; - char **tiles = NULL; - size_t tile_count = 0; - unsigned long long total_clusters = 0; - unsigned long long passed_clusters = 0; - unsigned long long filtered_clusters = 0; - unsigned long long undetermined_reads = 0; - size_t effective_threads = 1; - bcl_unknown_table unknowns = {0}; - int rc = 1; - - if (parse_run_info(run_folder, &run) != 0) { - fprintf(stderr, "failed to parse RunInfo.xml\n"); - goto done; - } - if (read_bcl_sample_sheet(sample_sheet, &samples) != 0) { - fprintf(stderr, "failed to parse sample sheet\n"); - goto done; - } - if (samples.count > MAX_BCL_SAMPLE_ROWS) { - fprintf(stderr, "sample sheet exceeds supported BCL sample count\n"); - goto done; - } - if (requested_threads == 0) { - requested_threads = get_cpu_count(); - } - if (ensure_dir(out_dir) != 0) { - fprintf(stderr, "failed to create BCL output directory\n"); - goto done; - } - if (collect_tiles(basecalls, 1, &tiles, &tile_count) != 0) { - fprintf(stderr, "failed to find classic BCL tiles; CBCL is not supported in this milestone\n"); - goto done; - } - - size_t output_file_count = 0; - if (checked_mul_size(samples.count == 0 ? 1 : samples.count, - run.read_count == 0 ? 1 : run.read_count, - &output_file_count) != 0) { - fprintf(stderr, "BCL output file count overflow\n"); - goto done; - } - sample_fastqs = (gzFile *)calloc(output_file_count, sizeof(gzFile)); - undetermined_fastqs = (gzFile *)calloc(run.read_count == 0 ? 1 : run.read_count, sizeof(gzFile)); - if (sample_fastqs == NULL || undetermined_fastqs == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t i = 0; i < samples.count; ++i) { - if (samples.items[i].is_alias) continue; - for (size_t r = 0; r < run.read_count; ++r) { - bcl_read_info *ri = &run.reads[r]; - if (!bcl_output_enabled(ri, emit_index_fastqs)) continue; - gzFile *slot = bcl_output_slot(sample_fastqs, i, r, run.read_count); - *slot = open_bcl_fastq(out_dir, samples.items[i].id, i + 1, 1, bcl_output_kind(ri), - bcl_output_number(&run, r), gzip_level); - if (*slot == NULL) { - fprintf(stderr, "failed to open sample FASTQ\n"); - goto done; - } - } - } - for (size_t r = 0; r < run.read_count; ++r) { - bcl_read_info *ri = &run.reads[r]; - if (!bcl_output_enabled(ri, emit_index_fastqs)) continue; - undetermined_fastqs[r] = open_bcl_fastq(out_dir, "Undetermined", 0, 1, bcl_output_kind(ri), - bcl_output_number(&run, r), gzip_level); - if (undetermined_fastqs[r] == NULL) { - fprintf(stderr, "failed to open undetermined FASTQ\n"); - goto done; - } - } - - for (size_t tile_i = 0; tile_i < tile_count; ++tile_i) { - unsigned char **cycles = (unsigned char **)calloc(run.total_cycles == 0 ? 1 : run.total_cycles, sizeof(unsigned char *)); - size_t cluster_count = 0; - if (cycles == NULL) { - fprintf(stderr, "out of memory\n"); - goto done; - } - for (size_t c = 1; c <= run.total_cycles; ++c) { - char bcl_path[4096]; - size_t n = 0; - if (build_bcl_path(bcl_path, sizeof(bcl_path), basecalls, 1, c, tiles[tile_i]) != 0 || - read_bcl_cycle(bcl_path, &cycles[c - 1], &n) != 0) { - fprintf(stderr, "failed to read BCL cycle\n"); - for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); - free(cycles); - goto done; - } - if (c == 1) cluster_count = n; - else if (n != cluster_count) { - fprintf(stderr, "BCL cycle cluster counts do not match\n"); - for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); - free(cycles); - goto done; - } - } - unsigned char *pf = NULL; - size_t pf_count = 0; - if (read_filter_file(basecalls, 1, tiles[tile_i], &pf, &pf_count) != 0) { - fprintf(stderr, "failed to read filter file\n"); - for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); - free(cycles); - goto done; - } - if (pf != NULL && pf_count != cluster_count) { - fprintf(stderr, "filter cluster count does not match BCL\n"); - free(pf); - for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); - free(cycles); - goto done; - } - - total_clusters += cluster_count; - size_t threads = requested_threads; - if (threads > cluster_count) threads = cluster_count == 0 ? 1 : cluster_count; - if (threads > effective_threads) effective_threads = threads; - const size_t block_size = 8192; - for (size_t block_start = 0; block_start < cluster_count;) { - size_t batch = 0; - pthread_t *thread_ids = NULL; - bcl_block_job *jobs = (bcl_block_job *)calloc(threads, sizeof(bcl_block_job)); - bcl_block_result *results = (bcl_block_result *)calloc(threads, sizeof(bcl_block_result)); - if (jobs == NULL || results == NULL) { - free(jobs); - free(results); - fprintf(stderr, "out of memory\n"); - goto done; - } - if (threads > 1) { - thread_ids = (pthread_t *)calloc(threads, sizeof(pthread_t)); - if (thread_ids == NULL) { - free(jobs); - free(results); - fprintf(stderr, "out of memory\n"); - goto done; - } - } - while (batch < threads && block_start < cluster_count) { - size_t block_end = block_start + block_size; - if (block_end > cluster_count) block_end = cluster_count; - if (init_bcl_block_result(&results[batch], samples.count, run.read_count) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - jobs[batch].run = &run; - jobs[batch].samples = &samples; - jobs[batch].cycles = cycles; - jobs[batch].pf = pf; - jobs[batch].tile = tiles[tile_i]; - jobs[batch].start = block_start; - jobs[batch].end = block_end; - jobs[batch].k1 = k1; - jobs[batch].k2 = k2; - jobs[batch].emit_index_fastqs = emit_index_fastqs; - jobs[batch].result = &results[batch]; - if (threads > 1) { - if (pthread_create(&thread_ids[batch], NULL, bcl_block_worker, &jobs[batch]) != 0) { - fprintf(stderr, "failed to create BCL worker\n"); - goto done; - } - } else { - results[batch].error = process_bcl_block(&jobs[batch]); - } - ++batch; - block_start = block_end; - } - if (threads > 1) { - for (size_t i = 0; i < batch; ++i) pthread_join(thread_ids[i], NULL); - } - for (size_t i = 0; i < batch; ++i) { - if (results[i].error != 0) { - fprintf(stderr, "failed to format BCL block\n"); - goto done; - } - passed_clusters += results[i].passed_clusters; - filtered_clusters += results[i].filtered_clusters; - undetermined_reads += results[i].undetermined_reads; - for (size_t s = 0; s < samples.count; ++s) samples.items[s].assigned += results[i].sample_assigned[s]; - if (merge_bcl_unknowns(&unknowns, &results[i].unknowns) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - if (write_bcl_block_result(&results[i], sample_fastqs, undetermined_fastqs, samples.count, run.read_count) != 0) { - fprintf(stderr, "failed to write BCL block\n"); - goto done; - } - free_bcl_block_result(&results[i], samples.count, run.read_count); - } - free(thread_ids); - free(jobs); - free(results); - } - free(pf); - for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); - free(cycles); - } - - char stats_path[4096]; - if (path_join(stats_path, sizeof(stats_path), out_dir, "Demultiplex_Stats.csv") != 0) goto done; - FILE *stats = open_output_file(stats_path); - if (stats == NULL) goto done; - fprintf(stats, "sample_id,assigned_reads"); - int non_index_read_count = 0; - for (size_t r = 0; r < run.read_count; ++r) { - if (!run.reads[r].indexed) fprintf(stats, ",read%d_records", ++non_index_read_count); - } - fprintf(stats, "\n"); - unsigned long long assigned_reads = 0; - for (size_t i = 0; i < samples.count; ++i) { - if (samples.items[i].is_alias) continue; - fprintf(stats, "%s,%llu", samples.items[i].id, samples.items[i].assigned); - for (int r = 0; r < non_index_read_count; ++r) fprintf(stats, ",%llu", samples.items[i].assigned); - fprintf(stats, "\n"); - assigned_reads += samples.items[i].assigned; - } - fprintf(stats, "Undetermined,%llu", undetermined_reads); - for (int r = 0; r < non_index_read_count; ++r) fprintf(stats, ",%llu", undetermined_reads); - fprintf(stats, "\n"); - fclose(stats); - - if (unknowns.count > 0) { - qsort(unknowns.items, unknowns.count, sizeof(unknowns.items[0]), cmp_bcl_unknown_desc); - char unknown_path[4096]; - if (path_join(unknown_path, sizeof(unknown_path), out_dir, "Top_Unknown_Barcodes.csv") != 0) goto done; - FILE *unknown = open_output_file(unknown_path); - if (unknown == NULL) goto done; - fprintf(unknown, "index,count\n"); - size_t n = unknowns.count < 100 ? unknowns.count : 100; - for (size_t i = 0; i < n; ++i) fprintf(unknown, "%s,%llu\n", unknowns.items[i].index, unknowns.items[i].count); - fclose(unknown); - } - - char normalized_path[4096]; - if (path_join(normalized_path, sizeof(normalized_path), out_dir, "SampleSheet.normalized.csv") != 0) goto done; - FILE *normalized = open_output_file(normalized_path); - if (normalized != NULL) { - fprintf(normalized, "sample_id,sample_name,lane,index,index2\n"); - for (size_t i = 0; i < samples.count; ++i) { - fprintf(normalized, "%s,%s,%d,%s,%s\n", samples.items[i].id, samples.items[i].name, - samples.items[i].lane, samples.items[i].index1, samples.items[i].index2); - } - fclose(normalized); - } - - if (summary_path != NULL) { - FILE *summary = open_output_file(summary_path); - if (summary == NULL) goto done; - fprintf(summary, - "{\n \"workflow\": \"bcl-demux\",\n \"format\": \"classic_bcl\",\n \"lanes\": 1,\n \"tiles\": %zu,\n \"total_clusters\": %llu,\n \"passed_filter_clusters\": %llu,\n \"filtered_clusters\": %llu,\n \"assigned_reads\": %llu,\n \"undetermined_reads\": %llu,\n \"barcode_mismatches_index1\": %d,\n \"barcode_mismatches_index2\": %d,\n \"requested_threads\": %zu,\n \"effective_threads\": %zu,\n \"gzip_level\": %d,\n \"emit_index_fastqs\": %s\n}\n", - tile_count, total_clusters, passed_clusters, filtered_clusters, assigned_reads, undetermined_reads, - k1, k2, requested_threads, effective_threads, gzip_level, emit_index_fastqs ? "true" : "false"); - fclose(summary); - } - - rc = 0; - -done: - if (sample_fastqs != NULL) { - for (size_t i = 0; i < samples.count; ++i) { - if (samples.items[i].is_alias) continue; - for (size_t r = 0; r < run.read_count; ++r) { - gzFile *slot = bcl_output_slot(sample_fastqs, i, r, run.read_count); - if (*slot != NULL) gzclose(*slot); - } - } - } - if (undetermined_fastqs != NULL) { - for (size_t r = 0; r < run.read_count; ++r) { - if (undetermined_fastqs[r] != NULL) gzclose(undetermined_fastqs[r]); - } - } - if (tiles != NULL) { - for (size_t i = 0; i < tile_count; ++i) free(tiles[i]); - free(tiles); - } - free(sample_fastqs); - free(undetermined_fastqs); - free_bcl_unknowns(&unknowns); - free_bcl_samples(&samples); - return rc; -} - -static int compare_gzip_fastq_files(const char *a_path, const char *b_path, unsigned long long *records_out) { - gzFile a = gzopen(a_path, "rb"); - gzFile b = gzopen(b_path, "rb"); - if (a == NULL || b == NULL) { - if (a != NULL) gzclose(a); - if (b != NULL) gzclose(b); - return -1; - } - char abuf[8192]; - char bbuf[8192]; - unsigned long long lines = 0; - int mismatch = 0; - for (;;) { - char *ag = gzgets(a, abuf, sizeof(abuf)); - char *bg = gzgets(b, bbuf, sizeof(bbuf)); - if (ag == NULL || bg == NULL) { - if (ag != bg) mismatch = 1; - break; - } - if (strcmp(abuf, bbuf) != 0) mismatch = 1; - ++lines; - } - gzclose(a); - gzclose(b); - *records_out = lines / 4; - return mismatch ? 1 : 0; -} - -static int run_bcl_validate(const char *argv0, int argc, char **argv) { - const char *dotmatch_out = NULL; - const char *truth_out = NULL; - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if (strcmp(arg, "--dotmatch-out") == 0 && i < argc) { - dotmatch_out = argv[i++]; - } else if (strcmp(arg, "--truth-out") == 0 && i < argc) { - truth_out = argv[i++]; - } else { - usage(argv0); - return 2; - } - } - if (dotmatch_out == NULL || truth_out == NULL) { - usage(argv0); - return 2; - } - DIR *dir = opendir(truth_out); - if (dir == NULL) { - fprintf(stderr, "failed to open truth output directory\n"); - return 1; - } - unsigned long long compared_files = 0; - unsigned long long compared_records = 0; - unsigned long long missing_files = 0; - unsigned long long mismatched_files = 0; - struct dirent *ent; - while ((ent = readdir(dir)) != NULL) { - if (!ends_with(ent->d_name, ".fastq.gz")) continue; - char truth_path[4096]; - char dotmatch_path[4096]; - if (path_join(truth_path, sizeof(truth_path), truth_out, ent->d_name) != 0 || - path_join(dotmatch_path, sizeof(dotmatch_path), dotmatch_out, ent->d_name) != 0) { - closedir(dir); - fprintf(stderr, "BCL validation path is too long\n"); - return 1; - } - if (!path_exists(dotmatch_path)) { - ++missing_files; - continue; - } - unsigned long long records = 0; - int cmp = compare_gzip_fastq_files(dotmatch_path, truth_path, &records); - if (cmp != 0) ++mismatched_files; - compared_records += records; - ++compared_files; - } - closedir(dir); - printf("{\n \"compared_fastq_files\": %llu,\n \"compared_records\": %llu,\n \"missing_fastq_files\": %llu,\n \"mismatched_fastq_files\": %llu\n}\n", - compared_files, compared_records, missing_files, mismatched_files); - return missing_files == 0 && mismatched_files == 0 ? 0 : 1; -} - -static int run_edlib_validate_helper(const char *targets_path, const char *reads_path, - size_t target_start, size_t target_len, int k, size_t indel_window, size_t sample_limit, - size_t auto_offset, size_t auto_offset_sample, offset_mode offsets_mode, double offset_min_fraction, - size_t threads) { - const char *helper_path = "./build/dotmatch_edlib_validate"; - if (access(helper_path, X_OK) != 0) { - fprintf(stderr, "edlib oracle validation requires build/dotmatch_edlib_validate; run `make edlib-tools`\n"); - return 2; - } - - char target_start_buf[32]; - char target_len_buf[32]; - char k_buf[32]; - char indel_window_buf[32]; - char sample_buf[32]; - char auto_offset_buf[32]; - char auto_offset_sample_buf[32]; - char threads_buf[32]; - char offset_min_fraction_buf[64]; - snprintf(target_start_buf, sizeof(target_start_buf), "%zu", target_start); - snprintf(target_len_buf, sizeof(target_len_buf), "%zu", target_len); - snprintf(k_buf, sizeof(k_buf), "%d", k); - snprintf(indel_window_buf, sizeof(indel_window_buf), "%zu", indel_window); - snprintf(sample_buf, sizeof(sample_buf), "%zu", sample_limit); - snprintf(auto_offset_buf, sizeof(auto_offset_buf), "%zu", auto_offset); - snprintf(auto_offset_sample_buf, sizeof(auto_offset_sample_buf), "%zu", auto_offset_sample); - snprintf(threads_buf, sizeof(threads_buf), "%zu", threads); - snprintf(offset_min_fraction_buf, sizeof(offset_min_fraction_buf), "%.8f", offset_min_fraction); - - pid_t pid = fork(); - if (pid < 0) { - perror("fork"); - return 1; - } - if (pid == 0) { - execl(helper_path, helper_path, - "--targets", targets_path, - "--reads", reads_path, - "--target-start", target_start_buf, - "--target-length", target_len_buf, - "--k", k_buf, - "--indel-window", indel_window_buf, - "--auto-offset", auto_offset_buf, - "--auto-offset-sample", auto_offset_sample_buf, - "--offset-mode", offset_mode_name(offsets_mode), - "--offset-min-fraction", offset_min_fraction_buf, - "--sample", sample_buf, - "--threads", threads_buf, - (char *)NULL); - perror("execl"); - _exit(127); - } - - int status = 0; - if (waitpid(pid, &status, 0) < 0) { - perror("waitpid"); - return 1; - } - if (WIFEXITED(status)) { - return WEXITSTATUS(status); - } - return 1; -} - -static int run_validate(const char *argv0, int argc, char **argv) { - const char *targets_path = NULL; - const char *reads_path = NULL; - const char *oracle = "scan"; - size_t target_start = 0; - size_t target_len = 0; - size_t indel_window = 0; - size_t sample_limit = 100000; - size_t auto_offset = 0; - size_t auto_offset_sample = 1000; - size_t threads = 0; - offset_mode offsets_mode = OFFSET_MODE_BEST; - double offset_min_fraction = 0.005; - count_metric metric = COUNT_METRIC_LEVENSHTEIN; - int k = -1; - - int i = 2; - while (i < argc) { - const char *arg = argv[i++]; - if (strcmp(arg, "--targets") == 0 && i < argc) { - targets_path = argv[i++]; - } else if (strcmp(arg, "--reads") == 0 && i < argc) { - reads_path = argv[i++]; - } else if (strcmp(arg, "--target-start") == 0 && i < argc) { - if (parse_size_value(argv[i++], &target_start) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--target-length") == 0 && i < argc) { - if (parse_size_value(argv[i++], &target_len) != 0 || target_len == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--k") == 0 && i < argc) { - if (parse_int_value(argv[i++], &k) != 0 || (k != 0 && k != 1)) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--indel-window") == 0 && i < argc) { - if (parse_size_value(argv[i++], &indel_window) != 0 || indel_window > 1) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--metric") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "hamming") == 0) { - metric = COUNT_METRIC_HAMMING; - } else if (strcmp(value, "levenshtein") == 0) { - metric = COUNT_METRIC_LEVENSHTEIN; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--auto-offset") == 0 && i < argc) { - if (parse_size_value(argv[i++], &auto_offset) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--auto-offset-sample") == 0 && i < argc) { - if (parse_size_value(argv[i++], &auto_offset_sample) != 0 || auto_offset_sample == 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--offset-mode") == 0 && i < argc) { - const char *value = argv[i++]; - if (strcmp(value, "best") == 0) { - offsets_mode = OFFSET_MODE_BEST; - } else if (strcmp(value, "multi") == 0) { - offsets_mode = OFFSET_MODE_MULTI; - } else { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--offset-min-fraction") == 0 && i < argc) { - if (parse_double_value(argv[i++], &offset_min_fraction) != 0 || - offset_min_fraction < 0.0 || offset_min_fraction > 1.0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--oracle") == 0 && i < argc) { - oracle = argv[i++]; - } else if (strcmp(arg, "--sample") == 0 && i < argc) { - if (parse_size_value(argv[i++], &sample_limit) != 0) { - usage(argv0); - return 2; - } - } else if (strcmp(arg, "--threads") == 0 && i < argc) { - if (parse_size_value(argv[i++], &threads) != 0) { - usage(argv0); - return 2; - } - } else { - usage(argv0); - return 2; - } - } - - if (targets_path == NULL || reads_path == NULL || target_len == 0 || k < 0) { - usage(argv0); - return 2; - } - if (auto_offset > MAX_AUTO_OFFSET) { - fprintf(stderr, "--auto-offset must be <= %d\n", MAX_AUTO_OFFSET); - return 2; - } - if (metric == COUNT_METRIC_HAMMING && indel_window != 0) { - fprintf(stderr, "--indel-window is only valid with --metric levenshtein\n"); - return 2; - } - if (threads == 0) { - threads = get_cpu_count(); - } - if (strcmp(oracle, "edlib") == 0) { - if (metric != COUNT_METRIC_LEVENSHTEIN) { - fprintf(stderr, "--oracle edlib is only valid with --metric levenshtein\n"); - return 2; - } - int status = run_edlib_validate_helper(targets_path, reads_path, target_start, target_len, k, indel_window, - sample_limit, auto_offset, auto_offset_sample, offsets_mode, - offset_min_fraction, threads); - return status == 0 ? 0 : status; - } - if (strcmp(oracle, "scan") != 0) { - usage(argv0); - return 2; - } - - seq_table targets = {0}; - fastq_reader reader = {0}; - qdaln_index *index = NULL; - const char **target_ptrs = NULL; - size_t *target_lens = NULL; - int rc = 1; - size_t checked = 0; - size_t mismatches = 0; - offset_list offsets = {0}; - - if (read_target_table(targets_path, &targets) != 0) { - fprintf(stderr, "failed to read targets\n"); - goto done; - } - if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { - fprintf(stderr, "out of memory\n"); - goto done; - } - index = qdaln_index_build(target_ptrs, target_lens, targets.count); - if (index == NULL) { - fprintf(stderr, "failed to build target index\n"); - goto done; - } - if (fastq_reader_open(&reader, reads_path) != 0) { - fprintf(stderr, "failed to open FASTQ input\n"); - goto done; - } - if (detect_offsets(index, NULL, reads_path, target_start, target_len, auto_offset, auto_offset_sample, - offsets_mode, offset_min_fraction, &offsets) != 0) { - fprintf(stderr, "automatic offset detection failed\n"); - goto done; - } - - char header[8192]; - char seq[8192]; - char plus[8192]; - char qual[8192]; - char observed[8192]; - char scan_observed[8192]; - int got = 0; - size_t seq_len = 0; - while ((sample_limit == 0 || checked < sample_limit) && - (got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { - qdaln_match_result indexed; - qdaln_match_result scan; - qdaln_index_stats stats; - if (assign_count_offsets(index, seq, seq_len, &offsets, target_start, target_len, k, metric, - indel_window, &indexed, &stats, observed, sizeof(observed), 0) != 0 || - scan_count_offsets(target_ptrs, target_lens, targets.count, seq, seq_len, &offsets, target_start, - target_len, k, metric, indel_window, &scan, scan_observed, - sizeof(scan_observed)) != 0) { - fprintf(stderr, "validation assignment failed\n"); - goto done; - } - if (indexed.target_index != scan.target_index || - indexed.best_distance != scan.best_distance || - indexed.second_best_distance != scan.second_best_distance || - indexed.match_count != scan.match_count || - indexed.status != scan.status) { - ++mismatches; - } - ++checked; - } - if (got < 0) { - fprintf(stderr, "malformed FASTQ input\n"); - goto done; - } - printf("{\n \"oracle\": \"native_scan\",\n \"checked_reads\": %zu,\n \"mismatches\": %zu,\n \"k\": %d,\n \"metric\": \"%s\",\n \"target_start\": %zu,\n \"target_length\": %zu,\n \"offset_mode\": \"%s\",\n \"selected_target_starts\": [", - checked, mismatches, k, metric_name(metric), target_start, target_len, offset_mode_name(offsets_mode)); - for (size_t i = 0; i < offsets.count; ++i) { - if (i != 0) printf(", "); - printf("%zu", offsets.items[i]); - } - printf("]\n}\n"); - rc = mismatches == 0 ? 0 : 1; - -done: - fastq_reader_close(&reader); - qdaln_index_free(index); - free_offset_list(&offsets); - free(target_ptrs); - free(target_lens); - free_table(&targets); - return rc; -} - -int main(int argc, char **argv) { - if (argc < 2) { - usage(argv[0]); - return 2; - } - - if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "help") == 0) { - if (argc != 2) { - usage(argv[0]); - return 2; - } - help_manual(stdout, argv[0]); - return 0; - } - - if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "version") == 0) { - if (argc != 2) { - usage(argv[0]); - return 2; - } - printf("dotmatch %s\n", DOTMATCH_VERSION); - return 0; - } - - if (strcmp(argv[1], "citation") == 0 || strcmp(argv[1], "cite") == 0) { - if (argc != 2) { - usage(argv[0]); - return 2; - } - print_citation(stdout); - return 0; - } - - if (strcmp(argv[1], "dist") == 0) { - if (argc != 4) { - usage(argv[0]); - return 2; - } - int d = qdaln_edit_distance(argv[2], strlen(argv[2]), argv[3], strlen(argv[3])); - if (d < 0) return 1; - printf("%d\n", d); - return 0; - } - - if (strcmp(argv[1], "leq") == 0) { - if (argc != 5) { - usage(argv[0]); - return 2; - } - int k = 0; - if (parse_int_value(argv[2], &k) != 0 || k < 0) { - usage(argv[0]); - return 2; - } - int ok = qdaln_edit_distance_leq(argv[3], strlen(argv[3]), argv[4], strlen(argv[4]), k); - if (ok < 0) return 1; - printf("%s\n", ok ? "true" : "false"); - return 0; - } - - if (strcmp(argv[1], "assign") == 0 || strcmp(argv[1], "match") == 0) { - return run_batch(argv[0], argc, argv, argv[1]); - } - - if (strcmp(argv[1], "fastq-assign") == 0) { - return run_fastq_assign(argv[0], argc, argv); - } - - if (strcmp(argv[1], "pair-count") == 0) { - return run_pair_count(argv[0], argc, argv); - } - - if (strcmp(argv[1], "demux") == 0) { - return run_demux(argv[0], argc, argv); - } - - if (strcmp(argv[1], "bcl-demux") == 0) { - return run_bcl_demux(argv[0], argc, argv); - } - - if (strcmp(argv[1], "bcl-validate") == 0) { - return run_bcl_validate(argv[0], argc, argv); - } - - if (strcmp(argv[1], "count") == 0 || strcmp(argv[1], "crispr-count") == 0) { - if (help_requested(argc, argv)) { - count_help_manual(stdout, argv[0], strcmp(argv[1], "crispr-count") == 0); - return 0; - } - return run_count(argv[0], argc, argv); - } - - if (strcmp(argv[1], "guide-counter") == 0 || strcmp(argv[1], "guide-counter-count") == 0 || - strcmp(argv[1], "guide-count") == 0) { - return run_guide_counter_compatible(argv[0], argc, argv); - } - - if (strcmp(argv[1], "inspect-unmatched") == 0) { - return run_inspect_unmatched(argv[0], argc, argv); - } - - if (strcmp(argv[1], "audit") == 0 || strcmp(argv[1], "audit-targets") == 0) { - if (help_requested(argc, argv)) { - audit_help_manual(stdout, argv[0]); - return 0; - } - return run_audit(argv[0], argc, argv); - } - - if (strcmp(argv[1], "validate") == 0) { - return run_validate(argv[0], argc, argv); - } - - usage(argv[0]); - return 2; -} + if (list->items[i] == offset) r \ No newline at end of file From 164a56de2bc359b6f68e314f9f0b3769cb9db11e Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Mon, 13 Jul 2026 15:06:00 +0100 Subject: [PATCH 03/93] Restore complete source blob for native CLI --- src/qda.c | 8494 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 8492 insertions(+), 2 deletions(-) diff --git a/src/qda.c b/src/qda.c index 802d5640..5bce7552 100644 --- a/src/qda.c +++ b/src/qda.c @@ -29,7 +29,7 @@ #include #ifndef DOTMATCH_VERSION -#define DOTMATCH_VERSION "0.1.9" +#define DOTMATCH_VERSION "0.1.8" #endif #define MAX_AUTO_OFFSET 1024 @@ -1237,4 +1237,8494 @@ static void free_offset_list(offset_list *list) { static int offset_list_contains(const offset_list *list, size_t offset) { for (size_t i = 0; i < list->count; ++i) { - if (list->items[i] == offset) r \ No newline at end of file + if (list->items[i] == offset) return 1; + } + return 0; +} + +static int push_offset_unique(offset_list *list, size_t offset) { + if (offset_list_contains(list, offset)) return 0; + if (list->count == list->cap) { + size_t next_cap = list->cap == 0 ? 8 : list->cap * 2; + size_t *next = (size_t *)realloc(list->items, next_cap * sizeof(size_t)); + if (next == NULL) return -1; + list->items = next; + list->cap = next_cap; + } + list->items[list->count++] = offset; + return 0; +} + +static size_t first_selected_offset(const offset_list *list, size_t fallback) { + return list != NULL && list->count != 0 ? list->items[0] : fallback; +} + +typedef struct hamming_lookup_entry { + uint64_t code; + int target_index; + int match_count; +} hamming_lookup_entry; + +typedef struct hamming_seed_entry { + uint64_t code; + int target_index; + int next; + unsigned char seed_id; +} hamming_seed_entry; + +typedef struct hamming_lookup { + hamming_lookup_entry *exact; + hamming_lookup_entry *mismatch; + hamming_seed_entry *seeds; + int *seed_heads; + uint64_t *target_codes; + size_t exact_cap; + size_t mismatch_cap; + size_t seed_hash_cap; + size_t n_seeds; + size_t target_len; + size_t seed0_len; + int ready; + int seed_ready; +} hamming_lookup; + +typedef struct levenshtein1_lookup { + hamming_lookup_entry *exact; + hamming_lookup_entry *substitution; + hamming_lookup_entry *target_deletion; + hamming_lookup_entry *target_insertion; + size_t exact_cap; + size_t substitution_cap; + size_t target_deletion_cap; + size_t target_insertion_cap; + size_t target_len; + int ready; +} levenshtein1_lookup; + +static const char *hamming_lookup_kind(const hamming_lookup *lookup) { + if (lookup == NULL || !lookup->ready) return "query"; + if (lookup->mismatch != NULL && lookup->mismatch_cap != 0) return "precompute"; + if (lookup->seed_ready) return "seed"; + return "exact"; +} + +static size_t next_pow2_local(size_t n) { + size_t p = 1; + while (p < n && p <= (SIZE_MAX >> 1)) p <<= 1; + return p < n ? n : p; +} + +static inline size_t code_hash_local(uint64_t code, size_t len, size_t cap) { + uint64_t x = code ^ ((uint64_t)len * 0x9e3779b97f4a7c15ULL); + x *= 0x9e3779b97f4a7c15ULL; + x ^= x >> 32; + return (size_t)x & (cap - 1); +} + +static inline size_t seed_hash_local(uint64_t code, size_t len, unsigned char seed_id, size_t cap) { + return code_hash_local(code ^ ((uint64_t)seed_id * 0x517cc1b727220a95ULL), len + seed_id * 37U, cap); +} + +static uint64_t code_low_mask_local(size_t len) { + if (len == 0) return 0; + if (len >= 32) return UINT64_MAX; + return (1ULL << (2 * len)) - 1ULL; +} + +static uint64_t code_segment_local(uint64_t code, size_t start, size_t len) { + return (code >> (2 * start)) & code_low_mask_local(len); +} + +static int hamming_code_distance_local(uint64_t a, uint64_t b, size_t len) { + uint64_t diff = a ^ b; + diff |= diff >> 1; + diff &= code_low_mask_local(len); + diff &= 0x5555555555555555ULL; +#if defined(__GNUC__) || defined(__clang__) + return __builtin_popcountll(diff); +#else + int d = 0; + while (diff != 0) { + d += (int)(diff & 1ULL); + diff >>= 2; + } + return d; +#endif +} + +static int dna2_code_local(const char *s, size_t len, uint64_t *code_out) { + if (s == NULL && len != 0) return 0; + if (len > 32) return 0; + uint64_t code = 0; + for (size_t i = 0; i < len; ++i) { + uint64_t v; + switch (s[i]) { + case 'A': + v = 0; + break; + case 'C': + v = 1; + break; + case 'G': + v = 2; + break; + case 'T': + v = 3; + break; + default: + return 0; + } + code |= v << (2 * i); + } + *code_out = code; + return 1; +} + +static int dna2_code_local_fold(const char *s, size_t len, uint64_t *code_out) { + if (s == NULL && len != 0) return 0; + if (len > 32) return 0; + uint64_t code = 0; + for (size_t i = 0; i < len; ++i) { + uint64_t v; + switch (s[i]) { + case 'A': + case 'a': + v = 0; + break; + case 'C': + case 'c': + v = 1; + break; + case 'G': + case 'g': + v = 2; + break; + case 'T': + case 't': + v = 3; + break; + default: + return 0; + } + code |= v << (2 * i); + } + *code_out = code; + return 1; +} + +static int dna2_base_fold_value(char c, uint64_t *value_out) { + switch (c) { + case 'A': + case 'a': + *value_out = 0; + return 1; + case 'C': + case 'c': + *value_out = 1; + return 1; + case 'G': + case 'g': + *value_out = 2; + return 1; + case 'T': + case 't': + *value_out = 3; + return 1; + default: + *value_out = 0; + return 0; + } +} + +static void copy_upper_ascii_window(char *dst, size_t dst_cap, const char *src, size_t len) { + if (dst_cap == 0) return; + if (len >= dst_cap) len = dst_cap - 1; + for (size_t i = 0; i < len; ++i) { + unsigned char c = (unsigned char)src[i]; + dst[i] = (char)(c >= 'a' && c <= 'z' ? c - 32 : c); + } + dst[len] = '\0'; +} + +static char uppercase_ascii_char(char c) { + unsigned char uc = (unsigned char)c; + return (char)(uc >= 'a' && uc <= 'z' ? uc - 32 : uc); +} + +static int phred33_quality(char c) { + unsigned char uc = (unsigned char)c; + return uc < 33 ? 0 : (int)uc - 33; +} + +static int window_matches_observed_folded(const char *seq, size_t seq_len, size_t start, + const char *observed, size_t observed_len) { + if (start > seq_len || observed_len > seq_len - start) return 0; + for (size_t i = 0; i < observed_len; ++i) { + if (uppercase_ascii_char(seq[start + i]) != observed[i]) return 0; + } + return 1; +} + +static int observed_base_qualities_within_edit(const char *observed, size_t observed_len, + const char *target, size_t target_len, + const char *qual, size_t qual_start, + int max_correction_qual) { + if (observed_len == target_len) { + int saw_mismatch = 0; + for (size_t i = 0; i < observed_len; ++i) { + if (observed[i] == target[i]) continue; + saw_mismatch = 1; + if (phred33_quality(qual[qual_start + i]) > max_correction_qual) return 0; + } + return saw_mismatch; + } + + if (observed_len == target_len + 1) { + for (size_t drop = 0; drop < observed_len; ++drop) { + size_t ti = 0; + int matches = 1; + for (size_t oi = 0; oi < observed_len; ++oi) { + if (oi == drop) continue; + if (observed[oi] != target[ti++]) { + matches = 0; + break; + } + } + if (matches && phred33_quality(qual[qual_start + drop]) <= max_correction_qual) return 1; + } + return 0; + } + + if (observed_len + 1 == target_len) { + return 1; + } + + return 0; +} + +static int hamming_lookup_insert(hamming_lookup_entry *table, size_t cap, uint64_t code, int target_index) { + size_t slot = code_hash_local(code, 0, cap); + for (;;) { + hamming_lookup_entry *entry = &table[slot]; + if (entry->target_index < 0) { + entry->code = code; + entry->target_index = target_index; + entry->match_count = 1; + return 0; + } + if (entry->code == code) { + if (entry->target_index != target_index) { + if (target_index < entry->target_index) entry->target_index = target_index; + ++entry->match_count; + } + return 0; + } + slot = (slot + 1) & (cap - 1); + } +} + +static const hamming_lookup_entry *hamming_lookup_find(const hamming_lookup_entry *table, size_t cap, uint64_t code) { + if (table == NULL || cap == 0) return NULL; + size_t slot = code_hash_local(code, 0, cap); + for (;;) { + const hamming_lookup_entry *entry = &table[slot]; + if (entry->target_index < 0) return NULL; + if (entry->code == code) return entry; + slot = (slot + 1) & (cap - 1); + } +} + +static int levenshtein1_lookup_insert(hamming_lookup_entry *table, size_t cap, uint64_t code, size_t len, + int target_index) { + size_t slot = code_hash_local(code, len, cap); + for (;;) { + hamming_lookup_entry *entry = &table[slot]; + if (entry->target_index < 0) { + entry->code = code; + entry->target_index = target_index; + entry->match_count = 1; + return 0; + } + if (entry->code == code) { + if (entry->target_index != target_index) { + if (target_index < entry->target_index) entry->target_index = target_index; + ++entry->match_count; + } + return 0; + } + slot = (slot + 1) & (cap - 1); + } +} + +static const hamming_lookup_entry *levenshtein1_lookup_find(const hamming_lookup_entry *table, size_t cap, + uint64_t code, size_t len) { + if (table == NULL || cap == 0) return NULL; + size_t slot = code_hash_local(code, len, cap); + for (;;) { + const hamming_lookup_entry *entry = &table[slot]; + if (entry->target_index < 0) return NULL; + if (entry->code == code) return entry; + slot = (slot + 1) & (cap - 1); + } +} + +static hamming_lookup_entry *alloc_hamming_table(size_t cap) { + hamming_lookup_entry *table = (hamming_lookup_entry *)malloc(cap * sizeof(hamming_lookup_entry)); + if (table == NULL) return NULL; + for (size_t i = 0; i < cap; ++i) { + table[i].code = 0; + table[i].target_index = -1; + table[i].match_count = 0; + } + return table; +} + +static int hamming_seed_insert(hamming_lookup *lookup, unsigned char seed_id, uint64_t code, int target_index) { + if (lookup->n_seeds > (size_t)INT32_MAX) return -1; + size_t seed_len = seed_id == 0 ? lookup->seed0_len : lookup->target_len - lookup->seed0_len; + size_t slot = seed_hash_local(code, seed_len, seed_id, lookup->seed_hash_cap); + size_t e = lookup->n_seeds++; + lookup->seeds[e].code = code; + lookup->seeds[e].target_index = target_index; + lookup->seeds[e].seed_id = seed_id; + lookup->seeds[e].next = lookup->seed_heads[slot]; + lookup->seed_heads[slot] = (int)e; + return 0; +} + +static void free_hamming_lookup(hamming_lookup *lookup) { + if (lookup == NULL) return; + free(lookup->exact); + free(lookup->mismatch); + free(lookup->seeds); + free(lookup->seed_heads); + free(lookup->target_codes); + lookup->exact = NULL; + lookup->mismatch = NULL; + lookup->seeds = NULL; + lookup->seed_heads = NULL; + lookup->target_codes = NULL; + lookup->exact_cap = 0; + lookup->mismatch_cap = 0; + lookup->seed_hash_cap = 0; + lookup->n_seeds = 0; + lookup->target_len = 0; + lookup->seed0_len = 0; + lookup->ready = 0; + lookup->seed_ready = 0; +} + +static void free_levenshtein1_lookup(levenshtein1_lookup *lookup) { + if (lookup == NULL) return; + free(lookup->exact); + free(lookup->substitution); + free(lookup->target_deletion); + free(lookup->target_insertion); + memset(lookup, 0, sizeof(*lookup)); +} + +static uint64_t code_remove_base_local(uint64_t code, size_t pos, size_t len) { + uint64_t low = code & code_low_mask_local(pos); + uint64_t high = code >> (2 * (pos + 1)); + (void)len; + return low | (high << (2 * pos)); +} + +static uint64_t code_insert_base_local(uint64_t code, size_t pos, size_t len, uint64_t base) { + uint64_t low = code & code_low_mask_local(pos); + uint64_t high = (code >> (2 * pos)) & code_low_mask_local(len - pos); + return low | (base << (2 * pos)) | (high << (2 * (pos + 1))); +} + +static int build_levenshtein1_lookup(const seq_table *targets, size_t target_len, levenshtein1_lookup *lookup) { + memset(lookup, 0, sizeof(*lookup)); + if (target_len == 0 || target_len > 31) return 0; + for (size_t i = 0; i < targets->count; ++i) { + uint64_t code = 0; + if (targets->records[i].len != target_len || + !dna2_code_local(targets->records[i].seq, target_len, &code)) { + return 0; + } + } + + lookup->exact_cap = next_pow2_local(targets->count * 2 + 16); + lookup->substitution_cap = next_pow2_local(targets->count * target_len * 3 + 16); + lookup->target_deletion_cap = next_pow2_local(targets->count * target_len + 16); + lookup->target_insertion_cap = next_pow2_local(targets->count * (target_len + 1) * 4 + 16); + lookup->exact = alloc_hamming_table(lookup->exact_cap); + lookup->substitution = alloc_hamming_table(lookup->substitution_cap); + lookup->target_deletion = alloc_hamming_table(lookup->target_deletion_cap); + lookup->target_insertion = alloc_hamming_table(lookup->target_insertion_cap); + if (lookup->exact == NULL || lookup->substitution == NULL || lookup->target_deletion == NULL || + lookup->target_insertion == NULL) { + free_levenshtein1_lookup(lookup); + return -1; + } + lookup->target_len = target_len; + + for (size_t i = 0; i < targets->count; ++i) { + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { + free_levenshtein1_lookup(lookup); + return 0; + } + if (levenshtein1_lookup_insert(lookup->exact, lookup->exact_cap, code, target_len, (int)i) != 0) { + free_levenshtein1_lookup(lookup); + return -1; + } + for (size_t pos = 0; pos < target_len; ++pos) { + uint64_t shift = (uint64_t)2 * pos; + uint64_t old_base = (code >> shift) & 3ULL; + uint64_t mask = 3ULL << shift; + for (uint64_t b = 0; b < 4; ++b) { + if (b == old_base) continue; + uint64_t mutated = (code & ~mask) | (b << shift); + if (levenshtein1_lookup_insert(lookup->substitution, lookup->substitution_cap, mutated, + target_len, (int)i) != 0) { + free_levenshtein1_lookup(lookup); + return -1; + } + } + uint64_t deleted = code_remove_base_local(code, pos, target_len); + if (levenshtein1_lookup_insert(lookup->target_deletion, lookup->target_deletion_cap, deleted, + target_len - 1, (int)i) != 0) { + free_levenshtein1_lookup(lookup); + return -1; + } + } + for (size_t pos = 0; pos <= target_len; ++pos) { + for (uint64_t b = 0; b < 4; ++b) { + uint64_t inserted = code_insert_base_local(code, pos, target_len, b); + if (levenshtein1_lookup_insert(lookup->target_insertion, lookup->target_insertion_cap, inserted, + target_len + 1, (int)i) != 0) { + free_levenshtein1_lookup(lookup); + return -1; + } + } + } + } + lookup->ready = 1; + return 0; +} + +static int build_hamming_lookup(const seq_table *targets, size_t target_len, hamming_lookup *lookup) { + memset(lookup, 0, sizeof(*lookup)); + if (target_len == 0 || target_len > 32) return 0; + for (size_t i = 0; i < targets->count; ++i) { + if (targets->records[i].len != target_len) return 0; + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) return 0; + } + + size_t exact_need = targets->count * 2 + 16; + /* Tuned *3 (not *4) since exactly 3 mutations per position; smaller mismatch table + improves cache residency for hamming_lookup precompute k=1 guide/barcode counting. */ + size_t mismatch_need = targets->count * target_len * 3 + 16; + lookup->exact_cap = next_pow2_local(exact_need); + lookup->mismatch_cap = next_pow2_local(mismatch_need); + lookup->exact = alloc_hamming_table(lookup->exact_cap); + lookup->mismatch = alloc_hamming_table(lookup->mismatch_cap); + if (lookup->exact == NULL || lookup->mismatch == NULL) { + free_hamming_lookup(lookup); + return -1; + } + lookup->target_len = target_len; + + for (size_t i = 0; i < targets->count; ++i) { + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { + free_hamming_lookup(lookup); + return 0; + } + if (hamming_lookup_insert(lookup->exact, lookup->exact_cap, code, (int)i) != 0) { + free_hamming_lookup(lookup); + return -1; + } + for (size_t pos = 0; pos < target_len; ++pos) { + uint64_t shift = (uint64_t)2 * pos; + uint64_t old_base = (code >> shift) & 3ULL; + uint64_t mask = 3ULL << shift; + for (uint64_t b = 0; b < 4; ++b) { + if (b == old_base) continue; + uint64_t mutated = (code & ~mask) | (b << shift); + if (hamming_lookup_insert(lookup->mismatch, lookup->mismatch_cap, mutated, (int)i) != 0) { + free_hamming_lookup(lookup); + return -1; + } + } + } + } + lookup->ready = 1; + return 0; +} + +static int build_hamming_exact_lookup(const seq_table *targets, size_t target_len, hamming_lookup *lookup) { + memset(lookup, 0, sizeof(*lookup)); + if (target_len == 0 || target_len > 32) return 0; + for (size_t i = 0; i < targets->count; ++i) { + if (targets->records[i].len != target_len) return 0; + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) return 0; + } + + size_t exact_need = targets->count * 2 + 16; + lookup->exact_cap = next_pow2_local(exact_need); + lookup->exact = alloc_hamming_table(lookup->exact_cap); + if (lookup->exact == NULL) { + free_hamming_lookup(lookup); + return -1; + } + lookup->target_len = target_len; + + for (size_t i = 0; i < targets->count; ++i) { + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { + free_hamming_lookup(lookup); + return 0; + } + if (hamming_lookup_insert(lookup->exact, lookup->exact_cap, code, (int)i) != 0) { + free_hamming_lookup(lookup); + return -1; + } + } + lookup->ready = 1; + return 0; +} + +static int build_hamming_seed_lookup(const seq_table *targets, size_t target_len, hamming_lookup *lookup) { + memset(lookup, 0, sizeof(*lookup)); + if (target_len < 2 || target_len > 32) return build_hamming_lookup(targets, target_len, lookup); + for (size_t i = 0; i < targets->count; ++i) { + if (targets->records[i].len != target_len) return 0; + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) return 0; + } + + size_t exact_need = targets->count * 2 + 16; + size_t seed_need = targets->count * 2 + 16; + lookup->exact_cap = next_pow2_local(exact_need); + lookup->seed_hash_cap = next_pow2_local(seed_need * 2 + 1); + lookup->exact = alloc_hamming_table(lookup->exact_cap); + lookup->seeds = (hamming_seed_entry *)malloc(seed_need * sizeof(hamming_seed_entry)); + lookup->seed_heads = (int *)malloc(lookup->seed_hash_cap * sizeof(int)); + lookup->target_codes = (uint64_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(uint64_t)); + if (lookup->exact == NULL || lookup->seeds == NULL || lookup->seed_heads == NULL || lookup->target_codes == NULL) { + free_hamming_lookup(lookup); + return -1; + } + for (size_t i = 0; i < lookup->seed_hash_cap; ++i) lookup->seed_heads[i] = -1; + lookup->target_len = target_len; + lookup->seed0_len = target_len / 2; + + for (size_t i = 0; i < targets->count; ++i) { + uint64_t code = 0; + if (!dna2_code_local(targets->records[i].seq, target_len, &code)) { + free_hamming_lookup(lookup); + return 0; + } + lookup->target_codes[i] = code; + if (hamming_lookup_insert(lookup->exact, lookup->exact_cap, code, (int)i) != 0) { + free_hamming_lookup(lookup); + return -1; + } + uint64_t seed0 = code_segment_local(code, 0, lookup->seed0_len); + uint64_t seed1 = code_segment_local(code, lookup->seed0_len, target_len - lookup->seed0_len); + if (hamming_seed_insert(lookup, 0, seed0, (int)i) != 0 || + hamming_seed_insert(lookup, 1, seed1, (int)i) != 0) { + free_hamming_lookup(lookup); + return -1; + } + } + lookup->ready = 1; + lookup->seed_ready = 1; + return 0; +} + +static int cmp_ull_desc(const void *a, const void *b) { + unsigned long long aa = *(const unsigned long long *)a; + unsigned long long bb = *(const unsigned long long *)b; + return aa < bb ? 1 : (aa > bb ? -1 : 0); +} + +static int cmp_ull_asc(const void *a, const void *b) { + unsigned long long aa = *(const unsigned long long *)a; + unsigned long long bb = *(const unsigned long long *)b; + return aa > bb ? 1 : (aa < bb ? -1 : 0); +} + +static double gini_from_counts(const unsigned long long *values, size_t n) { + if (n == 0) return 0.0; + unsigned long long *tmp = (unsigned long long *)malloc(n * sizeof(unsigned long long)); + if (tmp == NULL) return 0.0; + unsigned long long sum = 0; + for (size_t i = 0; i < n; ++i) { + tmp[i] = values[i]; + sum += values[i]; + } + if (sum == 0) { + free(tmp); + return 0.0; + } + qsort(tmp, n, sizeof(unsigned long long), cmp_ull_asc); + long double weighted = 0.0; + for (size_t i = 0; i < n; ++i) weighted += (long double)(i + 1) * (long double)tmp[i]; + free(tmp); + long double gini = (2.0L * weighted / ((long double)n * (long double)sum)) - + (((long double)n + 1.0L) / (long double)n); + if (gini < 0.0L) return 0.0; + if (gini > 1.0L) return 1.0; + return (double)gini; +} + +static double top_fraction_from_counts(const unsigned long long *values, size_t n, double fraction) { + if (n == 0) return 0.0; + unsigned long long *tmp = (unsigned long long *)malloc(n * sizeof(unsigned long long)); + if (tmp == NULL) return 0.0; + unsigned long long sum = 0; + for (size_t i = 0; i < n; ++i) { + tmp[i] = values[i]; + sum += values[i]; + } + if (sum == 0) { + free(tmp); + return 0.0; + } + qsort(tmp, n, sizeof(unsigned long long), cmp_ull_desc); + size_t top_n = (size_t)((double)n * fraction); + if (top_n == 0) top_n = 1; + if (top_n > n) top_n = n; + unsigned long long top_sum = 0; + for (size_t i = 0; i < top_n; ++i) top_sum += tmp[i]; + free(tmp); + return (double)top_sum / (double)sum; +} + +static int compute_sample_qc_metrics(const seq_table *targets, const unsigned long long *counts, size_t sample_index, + const count_stats *stats, sample_qc_metrics *metrics_out) { + if (targets == NULL || counts == NULL || stats == NULL || metrics_out == NULL) return -1; + unsigned long long *target_totals = + (unsigned long long *)calloc(targets->count == 0 ? 1 : targets->count, sizeof(unsigned long long)); + if (target_totals == NULL) return -1; + unsigned long long observed_targets = 0; + for (size_t t = 0; t < targets->count; ++t) { + for (size_t kind = 0; kind < 5; ++kind) { + target_totals[t] += counts[((sample_index * targets->count + t) * 5) + kind]; + } + if (target_totals[t] != 0) ++observed_targets; + } + unsigned long long valid = stats->total >= stats->invalid ? stats->total - stats->invalid : 0; + double valid_denom = valid == 0 ? 1.0 : (double)valid; + metrics_out->assignment_rate = (double)stats->unique / valid_denom; + metrics_out->ambiguous_rate = (double)stats->ambiguous / valid_denom; + metrics_out->no_match_rate = (double)stats->unmatched / valid_denom; + metrics_out->invalid_rate = stats->total == 0 ? 0.0 : (double)stats->invalid / (double)stats->total; + metrics_out->coverage_fraction = + targets->count == 0 ? 0.0 : (double)observed_targets / (double)targets->count; + metrics_out->zero_count_fraction = + targets->count == 0 ? 0.0 : (double)(targets->count - observed_targets) / (double)targets->count; + metrics_out->gini_index = gini_from_counts(target_totals, targets->count); + metrics_out->top_1pct_fraction = top_fraction_from_counts(target_totals, targets->count, 0.01); + free(target_totals); + return 0; +} + +static void emit_sample_qc_review_warnings(const string_list *labels, const sample_qc_metrics *metrics, size_t count) { + if (labels == NULL || metrics == NULL || count == 0) return; + int any = 0; + for (size_t sample = 0; sample < count; ++sample) { + const sample_qc_metrics *m = &metrics[sample]; + int sample_warn = 0; + if (m->assignment_rate < 0.80) sample_warn = 1; + if (m->ambiguous_rate > 0.05) sample_warn = 1; + if (m->no_match_rate > 0.15) sample_warn = 1; + if (m->invalid_rate > 0.02) sample_warn = 1; + if (m->coverage_fraction < 0.90) sample_warn = 1; + if (m->zero_count_fraction > 0.10) sample_warn = 1; + if (m->gini_index > 0.50) sample_warn = 1; + if (m->top_1pct_fraction > 0.30) sample_warn = 1; + if (!sample_warn) continue; + any = 1; + fprintf(stderr, "dotmatch: QC review recommended for sample %s:", labels->items[sample]); + if (m->assignment_rate < 0.80) fprintf(stderr, " assignment_rate=%.1f%%", 100.0 * m->assignment_rate); + if (m->ambiguous_rate > 0.05) fprintf(stderr, " ambiguous_rate=%.1f%%", 100.0 * m->ambiguous_rate); + if (m->no_match_rate > 0.15) fprintf(stderr, " no_match_rate=%.1f%%", 100.0 * m->no_match_rate); + if (m->invalid_rate > 0.02) fprintf(stderr, " invalid_rate=%.1f%%", 100.0 * m->invalid_rate); + if (m->coverage_fraction < 0.90) fprintf(stderr, " coverage=%.1f%%", 100.0 * m->coverage_fraction); + if (m->zero_count_fraction > 0.10) fprintf(stderr, " zero_count_guides=%.1f%%", 100.0 * m->zero_count_fraction); + if (m->gini_index > 0.50) fprintf(stderr, " gini=%.2f", m->gini_index); + if (m->top_1pct_fraction > 0.30) fprintf(stderr, " top_1pct_fraction=%.1f%%", 100.0 * m->top_1pct_fraction); + fprintf(stderr, "\n"); + } + if (any) { + fprintf(stderr, + "dotmatch: review sample_qc.tsv and summary.json before downstream MAGeCK/BAGEL analysis; " + "thresholds are conservative diagnostics, not biological pass/fail rules\n"); + } +} + +typedef enum ambiguity_policy { + AMBIGUITY_POLICY_BEST = 0, + AMBIGUITY_POLICY_RADIUS = 1 +} ambiguity_policy; + +static const char *ambiguity_policy_name(ambiguity_policy policy) { + return policy == AMBIGUITY_POLICY_RADIUS ? "radius" : "best"; +} + +static int apply_ambiguity_policy(qdaln_match_result *result, ambiguity_policy policy) { + if (policy == AMBIGUITY_POLICY_RADIUS && result->status == QDALN_MATCH_UNIQUE && result->match_count > 1) { + result->status = QDALN_MATCH_AMBIGUOUS; + } + return 0; +} + +static void html_escape(FILE *out, const char *s); + +static void write_tsv_preview_table(FILE *out, const char *title, const char *path, size_t max_rows) { + FILE *in = fopen(path, "r"); + if (in == NULL) return; + fprintf(out, "

      "); + html_escape(out, title); + fprintf(out, "

      \n"); + char line[16384]; + size_t row = 0; + while (row <= max_rows && fgets(line, sizeof(line), in) != NULL) { + trim_line(line); + fprintf(out, ""); + char *fields[128]; + size_t n = split_fields(line, '\t', fields, 128); + for (size_t i = 0; i < n; ++i) { + fprintf(out, row == 0 ? ""); + } + fprintf(out, "\n"); + ++row; + } + fprintf(out, "
      " : ""); + html_escape(out, fields[i]); + fprintf(out, row == 0 ? "" : "
      \n"); + fclose(in); +} + +static int write_count_html_report(const char *path, const seq_table *targets, const string_list *reads, + const string_list *labels, const unsigned long long *counts, + const count_stats *stats_by_sample, const offset_list *selected_offsets, + int k, count_metric metric, ambiguity_policy policy, size_t target_len, + const char *audit_dir, const char *unmatched_report_path) { + FILE *out = open_output_file(path); + if (out == NULL) return -1; + + int needs_review = 0; + for (size_t sample = 0; sample < reads->count; ++sample) { + const count_stats *s = &stats_by_sample[sample]; + unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; + double denom = valid == 0 ? 1.0 : (double)valid; + if ((double)s->ambiguous / denom > 0.01 || (double)s->unmatched / denom > 0.10) needs_review = 1; + } + + fprintf(out, + "\nDotMatch Report" + "
      \n"); + fprintf(out, "

      DotMatch Report

      \n"); + fprintf(out, "

      Known-target assignment report for %zu target%s and %zu sample%s. Ambiguous reads are not silently counted.

      \n", + targets->count, targets->count == 1 ? "" : "s", reads->count, reads->count == 1 ? "" : "s"); + fprintf(out, "

      Run Status

      " + "
      Status
      %s
      " + "
      Targets
      %zu
      " + "
      Samples
      %zu
      " + "
      Target length
      %zu
      \n", + needs_review ? "warn" : "ok", needs_review ? "Needs Review" : "Ready", + targets->count, reads->count, target_len); + fprintf(out, "

      Inputs and Configuration

      k
      %d
      " + "
      Metric
      %s
      " + "
      Ambiguity policy
      %s
      " + "
      Assignment
      Known target
      \n", + k, metric_name(metric), ambiguity_policy_name(policy)); + + fprintf(out, "

      Target Assignment QC

      " + "" + "\n"); + for (size_t sample = 0; sample < reads->count; ++sample) { + const count_stats *s = &stats_by_sample[sample]; + unsigned long long covered = 0; + for (size_t t = 0; t < targets->count; ++t) { + unsigned long long total = 0; + for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * targets->count + t) * 5) + kind]; + if (total != 0) ++covered; + } + unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; + double denom = valid == 0 ? 1.0 : (double)valid; + fprintf(out, "" + "\n", + s->total, valid, 100.0 * (double)s->unique / denom, 100.0 * (double)s->exact / denom, + 100.0 * (double)s->corrected / denom, 100.0 * (double)s->ambiguous / denom, + 100.0 * (double)s->unmatched / denom, + targets->count == 0 ? 0.0 : 100.0 * (double)covered / (double)targets->count, + s->candidates_verified); + } + fprintf(out, "
      SampleTotal readsValid windowsAssignment rateExact rateRescue rateAmbiguous rateNo-match rateLibrary coverageCandidates verified
      "); + html_escape(out, labels->items[sample]); + fprintf(out, "%llu%llu%.2f%%%.2f%%%.2f%%%.2f%%%.2f%%%.2f%%%llu
      \n"); + + fprintf(out, "

      Warnings

        \n"); + if (!needs_review) { + fprintf(out, "
      • No high ambiguous or no-match warning thresholds were crossed.
        • \n"); + } + for (size_t sample = 0; sample < reads->count; ++sample) { + const count_stats *s = &stats_by_sample[sample]; + unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; + double denom = valid == 0 ? 1.0 : (double)valid; + if ((double)s->ambiguous / denom > 0.01) { + fprintf(out, "
      • Sample "); + html_escape(out, labels->items[sample]); + fprintf(out, " has ambiguous assignments above 1%% of valid extracted windows.
        • \n"); + } + if ((double)s->unmatched / denom > 0.10) { + fprintf(out, "
      • Sample "); + html_escape(out, labels->items[sample]); + fprintf(out, " has no-match reads above 10%% of valid extracted windows.
        • \n"); + } + } + fprintf(out, "
      • Ambiguous reads are not silently counted.
      \n"); + + fprintf(out, "

      Input Files

      \n"); + for (size_t sample = 0; sample < reads->count; ++sample) { + fprintf(out, "\n"); + } + fprintf(out, "
      SampleFASTQSelected start(s)
      "); + html_escape(out, labels->items[sample]); + fprintf(out, ""); + html_escape(out, reads->items[sample]); + fprintf(out, ""); + for (size_t i = 0; i < selected_offsets[sample].count; ++i) { + if (i != 0) fprintf(out, ", "); + fprintf(out, "%zu", selected_offsets[sample].items[i]); + } + fprintf(out, "
      \n"); + + if (audit_dir != NULL) { + char audit_path[4096]; + int n = snprintf(audit_path, sizeof(audit_path), "%s/%s", audit_dir, "audit_summary.tsv"); + if (n >= 0 && (size_t)n < sizeof(audit_path)) { + write_tsv_preview_table(out, "Library Audit", audit_path, 40); + } + } + if (unmatched_report_path != NULL) { + write_tsv_preview_table(out, "Top Unmatched", unmatched_report_path, 25); + } + + fprintf(out, "
      \n"); + fclose(out); + return 0; +} + +typedef struct count_dirty_slot { + size_t slot; + unsigned long long count; +} count_dirty_slot; + +typedef struct count_dirty_slots { + count_dirty_slot *items; + size_t count; + size_t cap; + size_t *table; + size_t table_cap; +} count_dirty_slots; + +typedef struct count_sample_job { + const qdaln_index *index; + const hamming_lookup *hlookup; + const levenshtein1_lookup *levlookup; + const seq_table *targets; + const char **target_ptrs; + const size_t *target_lens; + const char *reads_path; + const char *sample_label; + size_t sample_index; + offset_list *selected_offsets; + size_t target_len; + int k; + count_metric metric; + size_t indel_window; + unsigned long long *counts; + count_stats *stats; + FILE *assignments; + FILE *ambiguous_out; + FILE *unmatched_out; + const char *ambiguous_policy; + ambiguity_policy assignment_policy; + int direct_hamming_counts; + int metal_hamming_counts; + const uint64_t *metal_target_codes; + int fused_offset_detection; + size_t target_start; + size_t auto_offset; + size_t auto_offset_sample; + offset_mode offsets_mode; + double offset_min_fraction; + size_t read_threads; + int max_correction_qual; + int rc; + count_dirty_slots *dirty_slots; + count_progress *progress; +} count_sample_job; + +static void write_assignment_like_row(FILE *out, const seq_table *targets, const char *sample, const char *read_id, + const char *observed, qdaln_match_result r, const char *correction) { + const char *target_id = ""; + const char *target_seq = ""; + if (r.target_index >= 0) { + target_id = targets->records[r.target_index].id; + target_seq = targets->records[r.target_index].seq; + } + fprintf(out, "%s\t%s\t%s\t%d\t%s\t%s\t%d\t%d\t%d\t%s\t%s\n", + sample, read_id, observed, r.target_index, target_id, target_seq, r.best_distance, + r.second_best_distance, r.match_count, status_name(r.status), correction); +} + +static int find_observed_quality_window(const char *seq, size_t seq_len, const offset_list *offsets, + size_t fallback_offset, size_t target_len, count_metric metric, + size_t indel_window, int k, const char *observed, + size_t observed_len, size_t *start_out) { + size_t min_len = target_len; + size_t max_len = target_len; + if (metric == COUNT_METRIC_LEVENSHTEIN && indel_window != 0 && k == 1) { + min_len = target_len > indel_window ? target_len - indel_window : 0; + max_len = target_len + indel_window; + } + if (observed_len < min_len || observed_len > max_len) return 0; + + size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; + if (window_matches_observed_folded(seq, seq_len, offset, observed, observed_len)) { + *start_out = offset; + return 1; + } + } + return 0; +} + +static int quality_allows_unique_correction(const char *seq, size_t seq_len, const char *qual, + const offset_list *offsets, size_t fallback_offset, + size_t target_len, count_metric metric, size_t indel_window, + int k, const char *observed, const seq_record *target, + qdaln_match_result result, int max_correction_qual) { + if (max_correction_qual < 0 || qual == NULL) return 1; + if (result.status != QDALN_MATCH_UNIQUE || result.best_distance <= 0 || result.target_index < 0) return 1; + + size_t observed_len = strlen(observed); + size_t qual_start = 0; + if (!find_observed_quality_window(seq, seq_len, offsets, fallback_offset, target_len, metric, indel_window, + k, observed, observed_len, &qual_start)) { + return 0; + } + return observed_base_qualities_within_edit(observed, observed_len, target->seq, target->len, qual, + qual_start, max_correction_qual); +} + +static void html_escape(FILE *out, const char *s) { + for (; s != NULL && *s != '\0'; ++s) { + switch (*s) { + case '&': + fputs("&", out); + break; + case '<': + fputs("<", out); + break; + case '>': + fputs(">", out); + break; + case '"': + fputs(""", out); + break; + default: + fputc(*s, out); + break; + } + } +} + +static int build_target_arrays(const seq_table *targets, const char ***target_ptrs_out, size_t **target_lens_out) { + const char **target_ptrs = (const char **)malloc(targets->count * sizeof(char *)); + size_t *target_lens = (size_t *)malloc(targets->count * sizeof(size_t)); + if (targets->count != 0 && (target_ptrs == NULL || target_lens == NULL)) { + free(target_ptrs); + free(target_lens); + return -1; + } + for (size_t i = 0; i < targets->count; ++i) { + target_ptrs[i] = targets->records[i].seq; + target_lens[i] = targets->records[i].len; + } + *target_ptrs_out = target_ptrs; + *target_lens_out = target_lens; + return 0; +} + +static int all_targets_have_length(const seq_table *targets, size_t len) { + for (size_t i = 0; i < targets->count; ++i) { + if (targets->records[i].len != len) return 0; + } + return 1; +} + +static int cmp_size_asc(const void *a, const void *b) { + size_t aa = *(const size_t *)a; + size_t bb = *(const size_t *)b; + return aa > bb ? 1 : (aa < bb ? -1 : 0); +} + +static int collect_target_lengths(const seq_table *targets, size_t **lengths_out, size_t *count_out) { + size_t *lengths = (size_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(size_t)); + if (lengths == NULL) return -1; + size_t count = 0; + for (size_t i = 0; i < targets->count; ++i) { + size_t len = targets->records[i].len; + int seen = 0; + for (size_t j = 0; j < count; ++j) { + if (lengths[j] == len) { + seen = 1; + break; + } + } + if (!seen) lengths[count++] = len; + } + qsort(lengths, count, sizeof(size_t), cmp_size_asc); + *lengths_out = lengths; + *count_out = count; + return 0; +} + +typedef struct match_merge_hit { + int target_index; + int distance; +} match_merge_hit; + +typedef struct match_merge_state { + match_merge_hit inline_hits[16]; + match_merge_hit *hits; + size_t count; + size_t cap; + int next_synthetic_target; + int saw_none; +} match_merge_state; + +static void merge_state_init(match_merge_state *state) { + state->hits = state->inline_hits; + state->count = 0; + state->cap = sizeof(state->inline_hits) / sizeof(state->inline_hits[0]); + state->next_synthetic_target = -2; + state->saw_none = 0; +} + +static void merge_state_free(match_merge_state *state) { + if (state->hits != state->inline_hits) free(state->hits); + merge_state_init(state); +} + +static void copy_merge_observed(char *dst, size_t dst_cap, const char *src) { + if (dst_cap == 0) return; + size_t n = 0; + while (n + 1 < dst_cap && src[n] != '\0') ++n; + memcpy(dst, src, n); + dst[n] = '\0'; +} + +static int merge_state_grow(match_merge_state *state) { + size_t next_cap = state->cap * 2; + match_merge_hit *next = (match_merge_hit *)malloc(next_cap * sizeof(match_merge_hit)); + if (next == NULL) return -1; + memcpy(next, state->hits, state->count * sizeof(match_merge_hit)); + if (state->hits != state->inline_hits) free(state->hits); + state->hits = next; + state->cap = next_cap; + return 0; +} + +static int merge_state_add_hit(match_merge_state *state, int target_index, int distance) { + if (target_index >= 0) { + for (size_t i = 0; i < state->count; ++i) { + if (state->hits[i].target_index == target_index) { + if (distance < state->hits[i].distance) state->hits[i].distance = distance; + return 0; + } + } + } + if (state->count == state->cap && merge_state_grow(state) != 0) return -1; + state->hits[state->count].target_index = target_index; + state->hits[state->count].distance = distance; + ++state->count; + return 0; +} + +static int merge_state_best_distance(const match_merge_state *state) { + int best = -1; + for (size_t i = 0; i < state->count; ++i) { + int d = state->hits[i].distance; + if (best < 0 || d < best) best = d; + } + return best; +} + +static int merge_state_add_result(match_merge_state *state, char *best_observed, size_t best_observed_cap, + const char *observed, qdaln_match_result r) { + if (r.status == QDALN_MATCH_INVALID) return 0; + if (r.match_count == 0) { + if (state->count == 0 && !state->saw_none && best_observed_cap != 0) { + copy_merge_observed(best_observed, best_observed_cap, observed); + } + state->saw_none = 1; + return 0; + } + + int prior_best = merge_state_best_distance(state); + if ((prior_best < 0 || r.best_distance < prior_best) && best_observed_cap != 0) { + copy_merge_observed(best_observed, best_observed_cap, observed); + } + + if (r.target_index >= 0 && merge_state_add_hit(state, r.target_index, r.best_distance) != 0) return -1; + int remaining = r.match_count - (r.target_index >= 0 ? 1 : 0); + if (remaining <= 0) return 0; + + int synthetic_distance = r.best_distance; + if (r.status != QDALN_MATCH_AMBIGUOUS && r.second_best_distance >= 0) { + synthetic_distance = r.second_best_distance; + } + for (int i = 0; i < remaining; ++i) { + if (merge_state_add_hit(state, state->next_synthetic_target--, synthetic_distance) != 0) return -1; + } + return 0; +} + +static void merge_state_finish(const match_merge_state *state, qdaln_match_result *result) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (state->count == 0) { + result->status = state->saw_none ? QDALN_MATCH_NONE : QDALN_MATCH_INVALID; + return; + } + + int best_ties = 0; + for (size_t i = 0; i < state->count; ++i) { + int d = state->hits[i].distance; + ++result->match_count; + if (result->best_distance < 0 || d < result->best_distance) { + result->second_best_distance = result->best_distance; + result->best_distance = d; + result->target_index = state->hits[i].target_index >= 0 ? state->hits[i].target_index : -1; + best_ties = 1; + } else if (d == result->best_distance) { + if (state->hits[i].target_index >= 0 && + (result->target_index < 0 || state->hits[i].target_index < result->target_index)) { + result->target_index = state->hits[i].target_index; + } + ++best_ties; + } else if (result->second_best_distance < 0 || d < result->second_best_distance) { + result->second_best_distance = d; + } + } + result->status = best_ties > 1 ? QDALN_MATCH_AMBIGUOUS : QDALN_MATCH_UNIQUE; +} + +static void merge_summary_result(qdaln_match_result *best, char *best_observed, size_t best_observed_cap, + const char *observed, qdaln_match_result r) { + if (r.status == QDALN_MATCH_INVALID) return; + if (r.match_count == 0) { + if (best->status == QDALN_MATCH_INVALID) { + *best = r; + copy_merge_observed(best_observed, best_observed_cap, observed); + } + return; + } + if (best->match_count == 0 || best->best_distance < 0 || r.best_distance < best->best_distance) { + *best = r; + copy_merge_observed(best_observed, best_observed_cap, observed); + return; + } + if (r.best_distance == best->best_distance) { + if (r.target_index != best->target_index || r.status == QDALN_MATCH_AMBIGUOUS || + best->status == QDALN_MATCH_AMBIGUOUS) { + if (r.target_index >= 0 && (best->target_index < 0 || r.target_index < best->target_index)) { + best->target_index = r.target_index; + } + best->status = QDALN_MATCH_AMBIGUOUS; + best->match_count += r.match_count; + } + } else if (best->second_best_distance < 0 || r.best_distance < best->second_best_distance) { + best->second_best_distance = r.best_distance; + best->match_count += r.match_count; + } +} + +static int assign_count_window(const qdaln_index *index, const char *seq, size_t seq_len, size_t target_start, + size_t target_len, int k, count_metric metric, size_t indel_window, + qdaln_match_result *result, qdaln_index_stats *stats, char *observed, + size_t observed_cap, int best_exact_shortcut) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (stats != NULL) { + stats->candidates_considered = 0; + stats->candidates_verified = 0; + } + if (observed_cap != 0) observed[0] = '\0'; + + match_merge_state merge; + merge_state_init(&merge); + int rc = 0; + size_t min_len = target_len; + size_t max_len = target_len; + if (metric == COUNT_METRIC_LEVENSHTEIN && indel_window != 0 && k == 1) { + min_len = target_len > indel_window ? target_len - indel_window : 0; + max_len = target_len + indel_window; + } + + for (size_t len = min_len; len <= max_len; ++len) { + if (len >= observed_cap) continue; + if (target_start > seq_len || len > seq_len - target_start) continue; + + if (metric == COUNT_METRIC_HAMMING && k == 0) { + qdaln_match_result r; + qdaln_index_stats s = {0, 0}; + int exact_rc = qdaln_index_lookup_exact_ascii_stats(index, seq + target_start, len, &r, &s); + if (exact_rc != 0) { + rc = -1; + goto done; + } + if (stats != NULL) { + stats->candidates_considered += s.candidates_considered; + stats->candidates_verified += s.candidates_verified; + } + char candidate[8192]; + if (len >= sizeof(candidate)) continue; + memcpy(candidate, seq + target_start, len); + candidate[len] = '\0'; + uppercase_ascii(candidate); + if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { + rc = -1; + goto done; + } + continue; + } + + char candidate[8192]; + if (len >= sizeof(candidate)) continue; + memcpy(candidate, seq + target_start, len); + candidate[len] = '\0'; + uppercase_ascii(candidate); + + const char *read_ptr = candidate; + size_t read_len = len; + qdaln_match_result r; + qdaln_index_stats s = {0, 0}; + if (best_exact_shortcut && metric == COUNT_METRIC_HAMMING && k == 1) { + int exact_rc = qdaln_index_lookup_exact_stats(index, read_ptr, read_len, &r, &s); + if (exact_rc != 0) { + rc = -1; + goto done; + } + if (stats != NULL) { + stats->candidates_considered += s.candidates_considered; + stats->candidates_verified += s.candidates_verified; + } + if (r.status == QDALN_MATCH_UNIQUE || r.status == QDALN_MATCH_AMBIGUOUS || r.status == QDALN_MATCH_INVALID) { + if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { + rc = -1; + goto done; + } + continue; + } + s.candidates_considered = 0; + s.candidates_verified = 0; + } + int assign_rc = metric == COUNT_METRIC_HAMMING + ? qdaln_index_assign_hamming_stats(index, &read_ptr, &read_len, 1, k, &r, &s) + : best_exact_shortcut + ? qdaln_index_assign_status_stats(index, &read_ptr, &read_len, 1, k, &r, &s) + : qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &r, &s); + if (assign_rc != 0) { + rc = -1; + goto done; + } + if (stats != NULL) { + stats->candidates_considered += s.candidates_considered; + stats->candidates_verified += s.candidates_verified; + } + if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { + rc = -1; + goto done; + } + } + +done: + merge_state_finish(&merge, result); + merge_state_free(&merge); + return rc; +} + +static int assign_count_length_set(const qdaln_index *index, const char *seq, size_t seq_len, size_t target_start, + const size_t *lengths, size_t n_lengths, int k, count_metric metric, + size_t indel_window, qdaln_match_result *result, qdaln_index_stats *stats, + char *observed, size_t observed_cap, int best_exact_shortcut) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (stats != NULL) { + stats->candidates_considered = 0; + stats->candidates_verified = 0; + } + if (observed_cap != 0) observed[0] = '\0'; + for (size_t i = 0; i < n_lengths; ++i) { + qdaln_match_result r = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_index_stats s = {0, 0}; + char candidate[8192]; + if (assign_count_window(index, seq, seq_len, target_start, lengths[i], k, metric, indel_window, + &r, &s, candidate, sizeof(candidate), best_exact_shortcut) != 0) { + return -1; + } + if (stats != NULL) { + stats->candidates_considered += s.candidates_considered; + stats->candidates_verified += s.candidates_verified; + } + merge_summary_result(result, observed, observed_cap, candidate, r); + } + return 0; +} + +static int assign_count_offsets(const qdaln_index *index, const char *seq, size_t seq_len, + const offset_list *offsets, size_t fallback_offset, size_t target_len, + int k, count_metric metric, size_t indel_window, + qdaln_match_result *result, qdaln_index_stats *stats, + char *observed, size_t observed_cap, int best_exact_shortcut) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (stats != NULL) { + stats->candidates_considered = 0; + stats->candidates_verified = 0; + } + if (observed_cap != 0) observed[0] = '\0'; + + size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; + if (best_exact_shortcut && k == 1 && target_len < observed_cap) { + match_merge_state exact_merge; + merge_state_init(&exact_merge); + qdaln_match_result exact_result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_index_stats exact_stats_total = {0, 0}; + char exact_observed[8192]; + int exact_rc_total = 0; + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; + if (offset > seq_len || target_len > seq_len - offset || target_len >= sizeof(exact_observed)) continue; + memcpy(exact_observed, seq + offset, target_len); + exact_observed[target_len] = '\0'; + uppercase_ascii(exact_observed); + const char *read_ptr = exact_observed; + size_t read_len = target_len; + qdaln_match_result exact_one; + qdaln_index_stats exact_stats = {0, 0}; + if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, 0, &exact_one, &exact_stats) != 0) { + merge_state_free(&exact_merge); + return -1; + } + exact_stats_total.candidates_considered += exact_stats.candidates_considered; + exact_stats_total.candidates_verified += exact_stats.candidates_verified; + if (merge_state_add_result(&exact_merge, observed, observed_cap, exact_observed, exact_one) != 0) { + exact_rc_total = -1; + break; + } + } + merge_state_finish(&exact_merge, &exact_result); + merge_state_free(&exact_merge); + if (exact_rc_total != 0) return -1; + if (stats != NULL) { + stats->candidates_considered += exact_stats_total.candidates_considered; + stats->candidates_verified += exact_stats_total.candidates_verified; + } + if (exact_result.match_count > 0) { + *result = exact_result; + return 0; + } + } + match_merge_state merge; + merge_state_init(&merge); + int rc = 0; + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; + qdaln_match_result candidate = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_index_stats local_stats = {0, 0}; + char local_observed[8192]; + if (assign_count_window(index, seq, seq_len, offset, target_len, k, metric, indel_window, + &candidate, &local_stats, local_observed, sizeof(local_observed), + best_exact_shortcut) != 0) { + rc = -1; + break; + } + if (stats != NULL) { + stats->candidates_considered += local_stats.candidates_considered; + stats->candidates_verified += local_stats.candidates_verified; + } + if (merge_state_add_result(&merge, observed, observed_cap, local_observed, candidate) != 0) { + rc = -1; + break; + } + } + + merge_state_finish(&merge, result); + merge_state_free(&merge); + return rc; +} + +static int hamming_distance_within_k_cli(const char *a, size_t a_len, const char *b, size_t b_len, int k) { + if (a_len != b_len) return -1; + int d = 0; + for (size_t i = 0; i < a_len; ++i) { + if (a[i] != b[i] && ++d > k) return -1; + } + return d; +} + +static int hamming_distance_cli(const char *a, size_t a_len, const char *b, size_t b_len) { + if (a_len != b_len) return -1; + int d = 0; + for (size_t i = 0; i < a_len; ++i) { + if (a[i] != b[i]) ++d; + } + return d; +} + +static int scan_assign_metric(const char *read, size_t read_len, const char *const *targets, + const size_t *target_lens, size_t n_targets, int k, + count_metric metric, qdaln_match_result *result) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; + int best_ties = 0; + for (size_t i = 0; i < n_targets; ++i) { + int d = metric == COUNT_METRIC_HAMMING + ? hamming_distance_within_k_cli(read, read_len, targets[i], target_lens[i], k) + : qdaln_edit_distance_leq(read, read_len, targets[i], target_lens[i], k) > 0 + ? qdaln_edit_distance(read, read_len, targets[i], target_lens[i]) + : -1; + if (d < 0 || d > k) continue; + ++result->match_count; + if (result->best_distance < 0 || d < result->best_distance) { + result->second_best_distance = result->best_distance; + result->best_distance = d; + result->target_index = (int)i; + best_ties = 1; + } else if (d == result->best_distance) { + if (result->target_index < 0 || (int)i < result->target_index) result->target_index = (int)i; + ++best_ties; + } else if (result->second_best_distance < 0 || d < result->second_best_distance) { + result->second_best_distance = d; + } + } + if (result->match_count == 0) { + result->status = QDALN_MATCH_NONE; + } else if (best_ties > 1) { + result->status = QDALN_MATCH_AMBIGUOUS; + } else { + result->status = QDALN_MATCH_UNIQUE; + } + return 0; +} + +static int scan_count_window(const char *const *targets, const size_t *target_lens, size_t n_targets, + const char *seq, size_t seq_len, size_t target_start, size_t target_len, + int k, count_metric metric, size_t indel_window, + qdaln_match_result *result, char *observed, size_t observed_cap) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (observed_cap != 0) observed[0] = '\0'; + + match_merge_state merge; + merge_state_init(&merge); + int rc = 0; + size_t min_len = target_len; + size_t max_len = target_len; + if (metric == COUNT_METRIC_LEVENSHTEIN && indel_window != 0 && k == 1) { + min_len = target_len > indel_window ? target_len - indel_window : 0; + max_len = target_len + indel_window; + } + + for (size_t len = min_len; len <= max_len; ++len) { + if (len >= observed_cap) continue; + if (target_start > seq_len || len > seq_len - target_start) continue; + char candidate[8192]; + if (len >= sizeof(candidate)) continue; + memcpy(candidate, seq + target_start, len); + candidate[len] = '\0'; + uppercase_ascii(candidate); + qdaln_match_result r; + if (scan_assign_metric(candidate, len, targets, target_lens, n_targets, k, metric, &r) != 0) { + rc = -1; + break; + } + if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { + rc = -1; + break; + } + } + merge_state_finish(&merge, result); + merge_state_free(&merge); + return rc; +} + +static int scan_count_offsets(const char *const *targets, const size_t *target_lens, size_t n_targets, + const char *seq, size_t seq_len, const offset_list *offsets, + size_t fallback_offset, size_t target_len, int k, count_metric metric, + size_t indel_window, qdaln_match_result *result, + char *observed, size_t observed_cap) { + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (observed_cap != 0) observed[0] = '\0'; + size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; + match_merge_state merge; + merge_state_init(&merge); + int rc = 0; + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; + qdaln_match_result candidate = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + char local_observed[8192]; + if (scan_count_window(targets, target_lens, n_targets, seq, seq_len, offset, target_len, k, metric, + indel_window, &candidate, local_observed, sizeof(local_observed)) != 0) { + rc = -1; + break; + } + if (merge_state_add_result(&merge, observed, observed_cap, local_observed, candidate) != 0) { + rc = -1; + break; + } + } + merge_state_finish(&merge, result); + merge_state_free(&merge); + return rc; +} + +static void hamming_lookup_result_from_entry(const hamming_lookup_entry *entry, int distance, + qdaln_match_result *result) { + result->target_index = entry->target_index; + result->best_distance = distance; + result->second_best_distance = -1; + result->match_count = entry->match_count; + result->status = entry->match_count > 1 ? QDALN_MATCH_AMBIGUOUS : QDALN_MATCH_UNIQUE; +} + +static void levenshtein1_lookup_result_from_entry(const hamming_lookup_entry *entry, int distance, + qdaln_match_result *result) { + result->target_index = entry->target_index; + result->best_distance = distance; + result->second_best_distance = -1; + result->match_count = entry->match_count; + result->status = entry->match_count > 1 ? QDALN_MATCH_AMBIGUOUS : QDALN_MATCH_UNIQUE; +} + +static int assign_levenshtein1_lookup_offset(const levenshtein1_lookup *lookup, const char *seq, size_t seq_len, + size_t offset, qdaln_match_result *result, qdaln_index_stats *stats, + char *observed, size_t observed_cap) { + if (lookup == NULL || !lookup->ready || lookup->target_len == 0 || lookup->target_len > 31) return 0; + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (stats != NULL) { + stats->candidates_considered = 0; + stats->candidates_verified = 0; + } + if (observed_cap != 0) observed[0] = '\0'; + + size_t target_len = lookup->target_len; + if (offset > seq_len || target_len - 1 > seq_len - offset) return 1; + + uint64_t code = 0; + const hamming_lookup_entry *entry = NULL; + qdaln_match_result local = {-1, -1, -1, 0, QDALN_MATCH_NONE}; + int have_len_l = 0; + uint64_t code_len_l = 0; + if (target_len < observed_cap && target_len <= seq_len - offset && + dna2_code_local_fold(seq + offset, target_len, &code_len_l)) { + have_len_l = 1; + code = code_len_l; + entry = levenshtein1_lookup_find(lookup->exact, lookup->exact_cap, code, target_len); + if (entry != NULL) { + levenshtein1_lookup_result_from_entry(entry, 0, &local); + if (stats != NULL) { + stats->candidates_considered += (size_t)entry->match_count; + stats->candidates_verified += (size_t)entry->match_count; + } + copy_upper_ascii_window(observed, observed_cap, seq + offset, target_len); + *result = local; + return 1; + } + } + + match_merge_state merge; + merge_state_init(&merge); + int rc = 0; + char candidate_observed[128]; + + if (have_len_l) { + entry = levenshtein1_lookup_find(lookup->substitution, lookup->substitution_cap, code_len_l, target_len); + if (entry != NULL) { + levenshtein1_lookup_result_from_entry(entry, 1, &local); + if (stats != NULL) { + stats->candidates_considered += (size_t)entry->match_count; + stats->candidates_verified += (size_t)entry->match_count; + } + copy_upper_ascii_window(candidate_observed, sizeof(candidate_observed), seq + offset, target_len); + rc = merge_state_add_result(&merge, observed, observed_cap, candidate_observed, local); + } + } + + if (rc == 0 && target_len - 1 < observed_cap && dna2_code_local_fold(seq + offset, target_len - 1, &code)) { + entry = levenshtein1_lookup_find(lookup->target_deletion, lookup->target_deletion_cap, code, target_len - 1); + if (entry != NULL) { + levenshtein1_lookup_result_from_entry(entry, 1, &local); + if (stats != NULL) { + stats->candidates_considered += (size_t)entry->match_count; + stats->candidates_verified += (size_t)entry->match_count; + } + copy_upper_ascii_window(candidate_observed, sizeof(candidate_observed), seq + offset, target_len - 1); + rc = merge_state_add_result(&merge, observed, observed_cap, candidate_observed, local); + } + } + + if (rc == 0 && target_len + 1 < observed_cap && target_len + 1 <= seq_len - offset && + dna2_code_local_fold(seq + offset, target_len + 1, &code)) { + entry = levenshtein1_lookup_find(lookup->target_insertion, lookup->target_insertion_cap, code, target_len + 1); + if (entry != NULL) { + levenshtein1_lookup_result_from_entry(entry, 1, &local); + if (stats != NULL) { + stats->candidates_considered += (size_t)entry->match_count; + stats->candidates_verified += (size_t)entry->match_count; + } + copy_upper_ascii_window(candidate_observed, sizeof(candidate_observed), seq + offset, target_len + 1); + rc = merge_state_add_result(&merge, observed, observed_cap, candidate_observed, local); + } + } + + if (rc != 0) { + merge_state_free(&merge); + return -1; + } + merge_state_finish(&merge, result); + merge_state_free(&merge); + if (result->match_count > 0) { + return 1; + } + + if (target_len <= seq_len - offset && target_len < observed_cap) { + copy_upper_ascii_window(observed, observed_cap, seq + offset, target_len); + } else if (target_len - 1 < observed_cap) { + copy_upper_ascii_window(observed, observed_cap, seq + offset, target_len - 1); + } + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; + return 1; +} + +static int hamming_lookup_counts_eligible(int count_only, int max_correction_qual, count_metric metric, + size_t indel_window, int k, size_t target_len, + hamming_index_strategy hamming_strategy) { + if (!count_only || max_correction_qual >= 0) return 0; + if (metric != COUNT_METRIC_HAMMING || indel_window != 0) return 0; + if (k != 0 && k != 1) return 0; + if (target_len > 32) return 0; + return hamming_strategy == HAMMING_INDEX_PRECOMPUTE || hamming_strategy == HAMMING_INDEX_AUTO; +} + +static int levenshtein1_lookup_counts_eligible(int count_only, int max_correction_qual, count_metric metric, + size_t indel_window, int k, size_t target_len, + size_t max_selected_offsets, FILE *assignments, + FILE *ambiguous_out, FILE *unmatched_out, + ambiguity_policy assignment_policy) { + if (!count_only || max_correction_qual >= 0) return 0; + if (assignments != NULL || ambiguous_out != NULL || unmatched_out != NULL) return 0; + if (assignment_policy != AMBIGUITY_POLICY_BEST) return 0; + if (metric != COUNT_METRIC_LEVENSHTEIN || indel_window != 1 || k != 1) return 0; + if (max_selected_offsets > 1 || target_len == 0 || target_len > 31) return 0; + return 1; +} + +static int hamming_direct_worker_eligible(int lookup_eligible, ambiguity_policy assignment_policy, int k) { + if (!lookup_eligible) return 0; + if (k == 0) return 1; + return assignment_policy == AMBIGUITY_POLICY_BEST || assignment_policy == AMBIGUITY_POLICY_RADIUS; +} + +static const char *count_backend_mode_name(count_backend_mode mode) { + switch (mode) { + case COUNT_BACKEND_CPU: + return "cpu"; + case COUNT_BACKEND_METAL: + return "gpu-metal-experimental"; + case COUNT_BACKEND_AUTO: + default: + return "auto"; + } +} + +static int parse_count_backend_mode(const char *value, count_backend_mode *mode_out) { + if (strcmp(value, "auto") == 0) { + *mode_out = COUNT_BACKEND_AUTO; + return 0; + } + if (strcmp(value, "cpu") == 0) { + *mode_out = COUNT_BACKEND_CPU; + return 0; + } + if (strcmp(value, "gpu-metal-experimental") == 0 || strcmp(value, "metal") == 0) { + *mode_out = COUNT_BACKEND_METAL; + return 0; + } + return -1; +} + +static int metal_hamming_count_eligible(count_backend_mode backend, int hamming_lookup_eligible, + ambiguity_policy assignment_policy, int k, size_t max_selected_offsets, + int fused_offset_detection) { + if (backend != COUNT_BACKEND_METAL) return 0; + if (!hamming_lookup_eligible || !qdmetal_available()) return 0; + if (fused_offset_detection) return 0; + if (max_selected_offsets > 1) return 0; + if (k == 1 && assignment_policy != AMBIGUITY_POLICY_BEST) return 0; + return 1; +} + +static int build_packed_target_codes(const seq_table *targets, size_t target_len, uint64_t **codes_out) { + if (targets == NULL || codes_out == NULL || target_len == 0 || target_len > 32) return 0; + uint64_t *codes = (uint64_t *)calloc(targets->count == 0 ? 1 : targets->count, sizeof(uint64_t)); + if (codes == NULL) return -1; + for (size_t i = 0; i < targets->count; ++i) { + if (targets->records[i].len != target_len || !dna2_code_local(targets->records[i].seq, target_len, &codes[i])) { + free(codes); + return 0; + } + } + *codes_out = codes; + return 1; +} + +static const char *metal_count_engine_name(size_t n_targets) { + return n_targets >= 1024 ? "hamming_metal_seed_index" : "hamming_metal_brute_force"; +} + +static int direct_hamming_merge_lookup_entry(match_merge_state *merge, const hamming_lookup_entry *entry, + int distance) { + if (entry == NULL) return 0; + qdaln_match_result r; + hamming_lookup_result_from_entry(entry, distance, &r); + return merge_state_add_result(merge, NULL, 0, "", r); +} + +static int direct_hamming_collect_seed_hits(count_sample_job *job, match_merge_state *merge, unsigned char seed_id, + uint64_t seed_code, uint64_t read_code) { + const hamming_lookup *lookup = job->hlookup; + size_t seed_len = seed_id == 0 ? lookup->seed0_len : lookup->target_len - lookup->seed0_len; + size_t slot = seed_hash_local(seed_code, seed_len, seed_id, lookup->seed_hash_cap); + for (int e = lookup->seed_heads[slot]; e >= 0; e = lookup->seeds[e].next) { + const hamming_seed_entry *entry = &lookup->seeds[e]; + if (entry->seed_id != seed_id || entry->code != seed_code || entry->target_index < 0) continue; + job->stats->candidates_considered += 1; + job->stats->candidates_verified += 1; + if (hamming_code_distance_local(read_code, lookup->target_codes[entry->target_index], lookup->target_len) > 1) { + continue; + } + qdaln_match_result r = {entry->target_index, 1, -1, 1, QDALN_MATCH_UNIQUE}; + if (merge_state_add_result(merge, NULL, 0, "", r) != 0) return -1; + } + return 0; +} + +static int assign_hamming_lookup_offsets(const hamming_lookup *lookup, const char *seq, size_t seq_len, + const offset_list *offsets, size_t fallback_offset, int k, + qdaln_match_result *result, qdaln_index_stats *stats, + char *observed, size_t observed_cap, int exact_merge) { + if (lookup == NULL || !lookup->ready || lookup->target_len >= observed_cap || (k != 0 && k != 1)) return 0; + *result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (stats != NULL) { + stats->candidates_considered = 0; + stats->candidates_verified = 0; + } + if (observed_cap != 0) observed[0] = '\0'; + + size_t n_offsets = offsets == NULL || offsets->count == 0 ? 1 : offsets->count; + match_merge_state merge; + merge_state_init(&merge); + qdaln_match_result fast_result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + int fast_saw_window = 0; + int rc = 1; + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; + if (offset > seq_len || lookup->target_len > seq_len - offset) continue; + if (!exact_merge) fast_saw_window = 1; + char candidate[8192]; + uint64_t code = 0; + const char *window = seq + offset; + if (!dna2_code_local_fold(window, lookup->target_len, &code)) { + rc = 0; + break; + } + + const hamming_lookup_entry *entry = hamming_lookup_find(lookup->exact, lookup->exact_cap, code); + qdaln_match_result r = {-1, -1, -1, 0, QDALN_MATCH_NONE}; + if (entry != NULL) { + hamming_lookup_result_from_entry(entry, 0, &r); + if (stats != NULL) { + stats->candidates_considered += (size_t)entry->match_count; + stats->candidates_verified += (size_t)entry->match_count; + } + copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); + if (exact_merge ? merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0 + : (merge_summary_result(&fast_result, observed, observed_cap, candidate, r), 0)) { + rc = -1; + break; + } + if (!(exact_merge && k == 1)) continue; + } + if (k == 1) { + if (!exact_merge && fast_result.best_distance == 0) continue; + entry = hamming_lookup_find(lookup->mismatch, lookup->mismatch_cap, code); + if (entry != NULL) { + hamming_lookup_result_from_entry(entry, 1, &r); + if (stats != NULL) { + stats->candidates_considered += (size_t)entry->match_count; + stats->candidates_verified += (size_t)entry->match_count; + } + copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); + if (exact_merge ? merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0 + : (merge_summary_result(&fast_result, observed, observed_cap, candidate, r), 0)) { + rc = -1; + break; + } + } else { + if (!exact_merge) continue; + copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); + if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { + rc = -1; + break; + } + } + } else { + if (!exact_merge) continue; + copy_upper_ascii_window(candidate, sizeof(candidate), window, lookup->target_len); + if (merge_state_add_result(&merge, observed, observed_cap, candidate, r) != 0) { + rc = -1; + break; + } + } + } + + if (exact_merge) merge_state_finish(&merge, result); + else { + if (fast_result.status == QDALN_MATCH_INVALID && fast_saw_window) { + fast_result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; + } + *result = fast_result; + } + merge_state_free(&merge); + return rc; +} + +typedef struct seq_buffer { + char **items; + size_t *lens; + size_t count; + size_t cap; + char *fixed_items; + size_t fixed_len; + size_t fixed_cap; + int fixed_active; +} seq_buffer; + +static int seq_buffer_ptr_in_fixed(const seq_buffer *buffer, const char *ptr) { + if (buffer->fixed_items == NULL || buffer->fixed_cap == 0) return 0; + uintptr_t p = (uintptr_t)ptr; + uintptr_t start = (uintptr_t)buffer->fixed_items; + uintptr_t end = start + buffer->fixed_cap * (buffer->fixed_len + 1); + return p >= start && p < end; +} + +static void free_seq_buffer(seq_buffer *buffer) { + if (buffer == NULL) return; + for (size_t i = 0; i < buffer->count; ++i) { + if (!seq_buffer_ptr_in_fixed(buffer, buffer->items[i])) free(buffer->items[i]); + } + free(buffer->fixed_items); + free(buffer->items); + free(buffer->lens); + buffer->items = NULL; + buffer->lens = NULL; + buffer->fixed_items = NULL; + buffer->count = 0; + buffer->cap = 0; + buffer->fixed_len = 0; + buffer->fixed_cap = 0; + buffer->fixed_active = 0; +} + +/* reset_seq_buffer reclaims per-sequence mallocs (for variable-length batches) but + * retains the items/lens arrays and fixed block (when active for uniform lengths) + * so subsequent batches reuse the same allocations without per-batch malloc/free. + * This is used in the high-throughput batched reader paths for large FASTQ. + */ +static void reset_seq_buffer(seq_buffer *buffer) { + if (buffer == NULL) return; + for (size_t i = 0; i < buffer->count; ++i) { + if (!seq_buffer_ptr_in_fixed(buffer, buffer->items[i])) free(buffer->items[i]); + } + if (!buffer->fixed_active) { + free(buffer->fixed_items); + buffer->fixed_items = NULL; + buffer->fixed_len = 0; + buffer->fixed_cap = 0; + } + /* keep items/lens (and fixed block + active if still set) and cap for reuse */ + buffer->count = 0; +} + +static int grow_seq_buffer(seq_buffer *buffer) { + size_t old_cap = buffer->cap; + size_t next_cap = buffer->cap == 0 ? 1024 : buffer->cap * 2; + char **next_items = (char **)realloc(buffer->items, next_cap * sizeof(char *)); + if (next_items == NULL) return -1; + buffer->items = next_items; + size_t *next_lens = (size_t *)realloc(buffer->lens, next_cap * sizeof(size_t)); + if (next_lens == NULL) return -1; + buffer->lens = next_lens; + buffer->cap = next_cap; + if (buffer->fixed_active) { + char *next_fixed = (char *)realloc(buffer->fixed_items, next_cap * (buffer->fixed_len + 1)); + if (next_fixed == NULL) { + buffer->cap = old_cap; + return -1; + } + buffer->fixed_items = next_fixed; + buffer->fixed_cap = next_cap; + for (size_t i = 0; i < buffer->count; ++i) { + buffer->items[i] = buffer->fixed_items + i * (buffer->fixed_len + 1); + } + } + return 0; +} + +static int reserve_seq_buffer(seq_buffer *buffer, size_t requested_cap) { + if (requested_cap <= buffer->cap) return 0; + char **next_items = (char **)realloc(buffer->items, requested_cap * sizeof(char *)); + if (next_items == NULL) return -1; + buffer->items = next_items; + size_t *next_lens = (size_t *)realloc(buffer->lens, requested_cap * sizeof(size_t)); + if (next_lens == NULL) return -1; + buffer->lens = next_lens; + buffer->cap = requested_cap; + if (buffer->fixed_active) { + char *next_fixed = (char *)realloc(buffer->fixed_items, requested_cap * (buffer->fixed_len + 1)); + if (next_fixed == NULL) return -1; + buffer->fixed_items = next_fixed; + buffer->fixed_cap = requested_cap; + for (size_t i = 0; i < buffer->count; ++i) { + buffer->items[i] = buffer->fixed_items + i * (buffer->fixed_len + 1); + } + } + return 0; +} + +static int push_seq_buffer(seq_buffer *buffer, const char *seq, size_t len) { + if (buffer->count == buffer->cap && grow_seq_buffer(buffer) != 0) return -1; + if (buffer->count == 0 && buffer->fixed_items == NULL && len <= 8191) { + buffer->fixed_items = (char *)malloc(buffer->cap * (len + 1)); + if (buffer->fixed_items != NULL) { + buffer->fixed_len = len; + buffer->fixed_cap = buffer->cap; + buffer->fixed_active = 1; + } + } + if (buffer->fixed_active && len == buffer->fixed_len) { + char *dst = buffer->fixed_items + buffer->count * (buffer->fixed_len + 1); + memcpy(dst, seq, len); + dst[len] = '\0'; + buffer->items[buffer->count] = dst; + } else { + if (buffer->fixed_active && len != buffer->fixed_len) buffer->fixed_active = 0; + buffer->items[buffer->count] = xstrndup(seq, len); + if (buffer->items[buffer->count] == NULL) return -1; + } + buffer->lens[buffer->count] = len; + ++buffer->count; + return 0; +} + +static void direct_hamming_record_hit(int target_index, int match_count, int *best_target, int *ambiguous) { + if (match_count > 1) *ambiguous = 1; + if (*best_target < 0) { + *best_target = target_index; + } else if (target_index != *best_target) { + if (target_index >= 0 && target_index < *best_target) *best_target = target_index; + *ambiguous = 1; + } +} + +static void merge_count_stats(count_stats *dst, const count_stats *src) { + dst->total += src->total; + dst->unique += src->unique; + dst->exact += src->exact; + dst->corrected += src->corrected; + dst->ambiguous += src->ambiguous; + dst->unmatched += src->unmatched; + dst->invalid += src->invalid; + dst->candidates_considered += src->candidates_considered; + dst->candidates_verified += src->candidates_verified; +} + +static void free_count_dirty_slots(count_dirty_slots *dirty) { + if (dirty == NULL) return; + free(dirty->items); + free(dirty->table); + dirty->items = NULL; + dirty->table = NULL; + dirty->count = 0; + dirty->cap = 0; + dirty->table_cap = 0; +} + +static size_t count_dirty_slot_hash(size_t slot) { + uint64_t x = (uint64_t)slot; + x ^= x >> 30; + x *= UINT64_C(0xbf58476d1ce4e5b9); + x ^= x >> 27; + x *= UINT64_C(0x94d049bb133111eb); + x ^= x >> 31; + return (size_t)x; +} + +static int count_dirty_slots_rehash(count_dirty_slots *dirty, size_t min_cap) { + size_t cap = 32; + while (cap < min_cap) cap *= 2; + size_t *table = (size_t *)calloc(cap, sizeof(size_t)); + if (table == NULL) return -1; + + for (size_t i = 0; i < dirty->count; ++i) { + size_t mask = cap - 1; + size_t pos = count_dirty_slot_hash(dirty->items[i].slot) & mask; + while (table[pos] != 0) pos = (pos + 1) & mask; + table[pos] = i + 1; + } + + free(dirty->table); + dirty->table = table; + dirty->table_cap = cap; + return 0; +} + +static int mark_count_dirty_slot(count_dirty_slots *dirty, size_t slot) { + if (dirty == NULL) return 0; + + if (dirty->table_cap == 0 || ((dirty->count + 1) * 2) > dirty->table_cap) { + if (count_dirty_slots_rehash(dirty, (dirty->count + 1) * 4) != 0) return -1; + } + + size_t mask = dirty->table_cap - 1; + size_t pos = count_dirty_slot_hash(slot) & mask; + while (dirty->table[pos] != 0) { + count_dirty_slot *item = &dirty->items[dirty->table[pos] - 1]; + if (item->slot == slot) { + ++item->count; + return 0; + } + pos = (pos + 1) & mask; + } + + if (dirty->count == dirty->cap) { + size_t next_cap = dirty->cap == 0 ? 16 : dirty->cap * 2; + count_dirty_slot *next = (count_dirty_slot *)realloc(dirty->items, next_cap * sizeof(count_dirty_slot)); + if (next == NULL) return -1; + dirty->items = next; + dirty->cap = next_cap; + } + dirty->items[dirty->count].slot = slot; + dirty->items[dirty->count].count = 1; + dirty->table[pos] = dirty->count + 1; + ++dirty->count; + return 0; +} + +static int increment_count_slot(count_sample_job *job, size_t slot) { + if (job->dirty_slots != NULL) return mark_count_dirty_slot(job->dirty_slots, slot); + ++job->counts[slot]; + return 0; +} + +static int direct_hamming_apply_match_result(count_sample_job *job, qdaln_match_result result, int saw_window) { + apply_ambiguity_policy(&result, job->assignment_policy); + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { + int kind = result.best_distance == 0 ? 0 : 1; + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + (size_t)kind) != 0) { + return -1; + } + ++job->stats->unique; + if (result.best_distance == 0) ++job->stats->exact; + else ++job->stats->corrected; + } else if (result.status == QDALN_MATCH_AMBIGUOUS) { + ++job->stats->ambiguous; + } else if (result.status == QDALN_MATCH_NONE) { + ++job->stats->unmatched; + } else if (saw_window) { + ++job->stats->unmatched; + } else { + ++job->stats->invalid; + } + return 0; +} + +static void direct_hamming_visit_seed(const count_sample_job *job, unsigned char seed_id, uint64_t seed_code, + uint64_t read_code, int *best_target, int *ambiguous) { + const hamming_lookup *lookup = job->hlookup; + size_t seed_len = seed_id == 0 ? lookup->seed0_len : lookup->target_len - lookup->seed0_len; + size_t slot = seed_hash_local(seed_code, seed_len, seed_id, lookup->seed_hash_cap); + for (int e = lookup->seed_heads[slot]; e >= 0; ) { + int nexte = lookup->seeds[e].next; + const hamming_seed_entry *entry = &lookup->seeds[e]; + if (entry->seed_id != seed_id || entry->code != seed_code) { + e = nexte; + continue; + } + int target_index = entry->target_index; + if (target_index < 0) { + e = nexte; + continue; + } + job->stats->candidates_considered += 1; + job->stats->candidates_verified += 1; + if (hamming_code_distance_local(read_code, lookup->target_codes[target_index], lookup->target_len) > 1) { + e = nexte; + continue; + } + direct_hamming_record_hit(target_index, 1, best_target, ambiguous); + e = nexte; + } +} + +static size_t selected_offset_at(const offset_list *offsets, size_t fallback_offset, size_t i) { + return offsets == NULL || offsets->count == 0 ? fallback_offset : offsets->items[i]; +} + +static int selected_offsets_are_sorted(const offset_list *offsets) { + if (offsets == NULL || offsets->count < 2) return 1; + for (size_t i = 1; i < offsets->count; ++i) { + if (offsets->items[i] < offsets->items[i - 1]) return 0; + } + return 1; +} + +static void fill_direct_hamming_codes(const char *seq, size_t seq_len, const offset_list *offsets, + size_t fallback_offset, size_t target_len, uint64_t *codes, + unsigned char *valid, unsigned char *invalid_counts, + unsigned char *bad_positions, size_t n_offsets, + int *saw_window, int *saw_non_acgt_window) { + *saw_window = 0; + *saw_non_acgt_window = 0; + memset(valid, 0, n_offsets); + memset(invalid_counts, 0, n_offsets); + memset(bad_positions, 0, n_offsets); + if (target_len == 0 || target_len > 32) return; + + if (!selected_offsets_are_sorted(offsets)) { + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = selected_offset_at(offsets, fallback_offset, i); + if (offset > seq_len || target_len > seq_len - offset) continue; + *saw_window = 1; + uint64_t code = 0; + unsigned char n_bad = 0; + unsigned char bad_pos = 0; + for (size_t j = 0; j < target_len; ++j) { + uint64_t value = 0; + if (!dna2_base_fold_value(seq[offset + j], &value)) { + if (n_bad < 255) ++n_bad; + bad_pos = (unsigned char)j; + } + code |= value << (2 * j); + } + if (n_bad != 0) { + *saw_non_acgt_window = 1; + codes[i] = code; + invalid_counts[i] = n_bad; + bad_positions[i] = bad_pos; + continue; + } + valid[i] = 1; + codes[i] = code; + } + return; + } + + uint64_t code = 0; + size_t invalid_count = 0; + size_t current_offset = 0; + int have_window = 0; + for (size_t i = 0; i < n_offsets; ++i) { + size_t offset = selected_offset_at(offsets, fallback_offset, i); + if (offset > seq_len || target_len > seq_len - offset) break; + + if (!have_window) { + code = 0; + invalid_count = 0; + size_t last_bad = 0; + for (size_t j = 0; j < target_len; ++j) { + uint64_t value = 0; + if (!dna2_base_fold_value(seq[offset + j], &value)) { + ++invalid_count; + last_bad = j; + } + code |= value << (2 * j); + } + current_offset = offset; + have_window = 1; + if (invalid_count == 1) bad_positions[i] = (unsigned char)last_bad; + } else { + while (current_offset < offset) { + uint64_t outgoing = 0; + if (!dna2_base_fold_value(seq[current_offset], &outgoing) && invalid_count != 0) --invalid_count; + (void)outgoing; + code >>= 2; + uint64_t incoming = 0; + if (!dna2_base_fold_value(seq[current_offset + target_len], &incoming)) ++invalid_count; + code |= incoming << (2 * (target_len - 1)); + ++current_offset; + } + if (invalid_count == 1) { + for (size_t j = 0; j < target_len; ++j) { + uint64_t value = 0; + if (!dna2_base_fold_value(seq[current_offset + j], &value)) { + bad_positions[i] = (unsigned char)j; + break; + } + } + } + } + + *saw_window = 1; + if (invalid_count == 0) { + valid[i] = 1; + codes[i] = code; + } else { + *saw_non_acgt_window = 1; + codes[i] = code; + invalid_counts[i] = invalid_count > 255 ? 255 : (unsigned char)invalid_count; + } + } +} + +static int direct_hamming_count_seq(count_sample_job *job, const char *seq, size_t seq_len) { + if (job->hlookup == NULL || !job->hlookup->ready) return 0; + ++job->stats->total; + count_progress_tick(job->progress); + + size_t n_offsets = job->selected_offsets == NULL || job->selected_offsets->count == 0 + ? 1 : job->selected_offsets->count; + if (n_offsets == 1) { + size_t offset = job->selected_offsets == NULL || job->selected_offsets->count == 0 + ? job->target_start : job->selected_offsets->items[0]; + if (offset > seq_len || job->hlookup->target_len > seq_len - offset) { + ++job->stats->invalid; + return 0; + } + + uint64_t code = 0; + unsigned char invalid_count = 0; + unsigned char bad_position = 0; + for (size_t j = 0; j < job->hlookup->target_len; ++j) { + uint64_t value = 0; + if (!dna2_base_fold_value(seq[offset + j], &value)) { + if (invalid_count < 255) ++invalid_count; + bad_position = (unsigned char)j; + } + code |= value << (2 * j); + } + + int exact_target = -1; + int exact_ambiguous = 0; + if (invalid_count == 0) { + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, code); + if (entry != NULL) { + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + if (job->k == 0) { + if (entry->match_count > 1) { + ++job->stats->ambiguous; + } else { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + + (size_t)entry->target_index) * 5) + 0) != 0) { + return -1; + } + ++job->stats->unique; + ++job->stats->exact; + } + return 0; + } + direct_hamming_record_hit(entry->target_index, entry->match_count, &exact_target, + &exact_ambiguous); + } + } + + if (job->k == 0) { + ++job->stats->unmatched; + return 0; + } + + if (exact_target >= 0) { + if (exact_ambiguous) { + ++job->stats->ambiguous; + } else { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)exact_target) * 5) + 0) != 0) { + return -1; + } + ++job->stats->unique; + ++job->stats->exact; + } + return 0; + } + + int mismatch_target = -1; + int mismatch_ambiguous = 0; + if (job->k == 1) { + if (invalid_count == 0) { + if (job->hlookup->seed_ready) { + uint64_t seed0 = code_segment_local(code, 0, job->hlookup->seed0_len); + uint64_t seed1 = code_segment_local(code, job->hlookup->seed0_len, + job->hlookup->target_len - job->hlookup->seed0_len); + direct_hamming_visit_seed(job, 0, seed0, code, &mismatch_target, &mismatch_ambiguous); + direct_hamming_visit_seed(job, 1, seed1, code, &mismatch_target, &mismatch_ambiguous); + } else { + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->mismatch, job->hlookup->mismatch_cap, code); + if (entry != NULL) { + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, + &mismatch_ambiguous); + } + } + } else if (invalid_count == 1) { + uint64_t shift = (uint64_t)2 * bad_position; + uint64_t mask = 3ULL << shift; + for (uint64_t b = 0; b < 4; ++b) { + uint64_t patched = (code & ~mask) | (b << shift); + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, patched); + if (entry == NULL) continue; + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, + &mismatch_ambiguous); + } + } + } + + if (mismatch_target >= 0) { + if (mismatch_ambiguous) { + ++job->stats->ambiguous; + } else { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)mismatch_target) * 5) + 1) != 0) { + return -1; + } + ++job->stats->unique; + ++job->stats->corrected; + } + } else { + ++job->stats->unmatched; + } + return 0; + } + + uint64_t inline_codes[64]; + unsigned char inline_valid[64]; + unsigned char inline_invalid_counts[64]; + unsigned char inline_bad_positions[64]; + uint64_t *codes = inline_codes; + unsigned char *valid = inline_valid; + unsigned char *invalid_counts = inline_invalid_counts; + unsigned char *bad_positions = inline_bad_positions; + if (n_offsets > sizeof(inline_codes) / sizeof(inline_codes[0])) { + codes = (uint64_t *)malloc(n_offsets * sizeof(uint64_t)); + valid = (unsigned char *)malloc(n_offsets); + invalid_counts = (unsigned char *)malloc(n_offsets); + bad_positions = (unsigned char *)malloc(n_offsets); + if (codes == NULL || valid == NULL || invalid_counts == NULL || bad_positions == NULL) { + free(codes); + free(valid); + free(invalid_counts); + free(bad_positions); + ++job->stats->invalid; + return -1; + } + } + + int saw_window = 0; + int saw_non_acgt_window = 0; + fill_direct_hamming_codes(seq, seq_len, job->selected_offsets, job->target_start, + job->hlookup->target_len, codes, valid, invalid_counts, + bad_positions, n_offsets, + &saw_window, &saw_non_acgt_window); + (void)saw_non_acgt_window; + + if (job->k == 1 && job->assignment_policy == AMBIGUITY_POLICY_RADIUS) { + match_merge_state merge; + merge_state_init(&merge); + int rc = 0; + for (size_t i = 0; i < n_offsets && rc == 0; ++i) { + if (valid[i]) { + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, codes[i]); + if (entry != NULL) { + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + rc = direct_hamming_merge_lookup_entry(&merge, entry, 0); + } + if (rc == 0) { + if (job->hlookup->seed_ready) { + uint64_t seed0 = code_segment_local(codes[i], 0, job->hlookup->seed0_len); + uint64_t seed1 = code_segment_local(codes[i], job->hlookup->seed0_len, + job->hlookup->target_len - job->hlookup->seed0_len); + if (direct_hamming_collect_seed_hits(job, &merge, 0, seed0, codes[i]) != 0 || + direct_hamming_collect_seed_hits(job, &merge, 1, seed1, codes[i]) != 0) { + rc = -1; + } + } else { + entry = hamming_lookup_find(job->hlookup->mismatch, job->hlookup->mismatch_cap, codes[i]); + if (entry != NULL) { + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + rc = direct_hamming_merge_lookup_entry(&merge, entry, 1); + } + } + } + } else if (invalid_counts[i] == 1) { + uint64_t shift = (uint64_t)2 * bad_positions[i]; + uint64_t mask = 3ULL << shift; + for (uint64_t b = 0; b < 4 && rc == 0; ++b) { + uint64_t patched = (codes[i] & ~mask) | (b << shift); + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, patched); + if (entry == NULL) continue; + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + rc = direct_hamming_merge_lookup_entry(&merge, entry, 1); + } + } + } + qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + if (rc == 0) { + merge_state_finish(&merge, &result); + if (result.status == QDALN_MATCH_INVALID && saw_window) { + result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; + } + rc = direct_hamming_apply_match_result(job, result, saw_window); + } + merge_state_free(&merge); + if (codes != inline_codes) { + free(codes); + free(valid); + free(invalid_counts); + free(bad_positions); + } + return rc; + } + + int exact_target = -1; + int exact_ambiguous = 0; + for (size_t i = 0; i < n_offsets; ++i) { + if (!valid[i]) continue; + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, codes[i]); + if (entry == NULL) continue; + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + direct_hamming_record_hit(entry->target_index, entry->match_count, &exact_target, &exact_ambiguous); + } + + if (exact_target >= 0) { + if (exact_ambiguous) { + ++job->stats->ambiguous; + } else { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)exact_target) * 5) + 0) != 0) { + if (codes != inline_codes) { + free(codes); + free(valid); + free(invalid_counts); + free(bad_positions); + } + return -1; + } + ++job->stats->unique; + ++job->stats->exact; + } + if (codes != inline_codes) { + free(codes); + free(valid); + free(invalid_counts); + free(bad_positions); + } + return 0; + } + + int mismatch_target = -1; + int mismatch_ambiguous = 0; + if (job->k == 1) { + for (size_t i = 0; i < n_offsets; ++i) { + if (valid[i]) { + if (job->hlookup->seed_ready) { + uint64_t seed0 = code_segment_local(codes[i], 0, job->hlookup->seed0_len); + uint64_t seed1 = code_segment_local(codes[i], job->hlookup->seed0_len, + job->hlookup->target_len - job->hlookup->seed0_len); + direct_hamming_visit_seed(job, 0, seed0, codes[i], &mismatch_target, &mismatch_ambiguous); + direct_hamming_visit_seed(job, 1, seed1, codes[i], &mismatch_target, &mismatch_ambiguous); + } else { + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->mismatch, job->hlookup->mismatch_cap, codes[i]); + if (entry == NULL) continue; + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, + &mismatch_ambiguous); + } + } else if (invalid_counts[i] == 1) { + uint64_t shift = (uint64_t)2 * bad_positions[i]; + uint64_t mask = 3ULL << shift; + for (uint64_t b = 0; b < 4; ++b) { + uint64_t patched = (codes[i] & ~mask) | (b << shift); + const hamming_lookup_entry *entry = + hamming_lookup_find(job->hlookup->exact, job->hlookup->exact_cap, patched); + if (entry == NULL) continue; + job->stats->candidates_considered += (unsigned long long)entry->match_count; + job->stats->candidates_verified += (unsigned long long)entry->match_count; + direct_hamming_record_hit(entry->target_index, entry->match_count, &mismatch_target, + &mismatch_ambiguous); + } + } + } + } + + if (mismatch_target >= 0) { + if (mismatch_ambiguous) { + ++job->stats->ambiguous; + } else { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)mismatch_target) * 5) + 1) != 0) { + if (codes != inline_codes) { + free(codes); + free(valid); + free(invalid_counts); + free(bad_positions); + } + return -1; + } + ++job->stats->unique; + ++job->stats->corrected; + } + } else if (saw_window) { + ++job->stats->unmatched; + } else { + ++job->stats->invalid; + } + + if (codes != inline_codes) { + free(codes); + free(valid); + free(invalid_counts); + free(bad_positions); + } + return 0; +} + +typedef struct direct_hamming_batch_job { + count_sample_job job; + char **items; + size_t *lens; + size_t start; + size_t end; + count_dirty_slots dirty_slots; + count_stats local_stats; + int rc; +} direct_hamming_batch_job; + +static void *direct_hamming_batch_worker(void *arg) { + direct_hamming_batch_job *batch = (direct_hamming_batch_job *)arg; + batch->job.sample_index = 0; + batch->job.stats = &batch->local_stats; + batch->job.dirty_slots = &batch->dirty_slots; + batch->rc = 0; + for (size_t i = batch->start; i < batch->end; ++i) { + if (direct_hamming_count_seq(&batch->job, batch->items[i], batch->lens[i]) != 0) { + batch->rc = 1; + break; + } + } + return NULL; +} + +static int process_direct_hamming_buffer(count_sample_job *job, const seq_buffer *buffer) { + if (buffer->count == 0) return 0; + if (job->k == 0 && job->index != NULL && job->hlookup != NULL && job->hlookup->ready && + job->hlookup->target_len == job->target_len && (job->selected_offsets == NULL || job->selected_offsets->count <= 1)) { + qdaln_match_result *results = (qdaln_match_result *)calloc(buffer->count, sizeof(qdaln_match_result)); + if (results == NULL) return 1; + qdaln_index_stats batch_stats = {0, 0}; + if (qdaln_index_lookup_exact_ascii_many_stats(job->index, (const char *const *)buffer->items, buffer->lens, + buffer->count, results, &batch_stats) != 0) { + free(results); + return 1; + } + job->stats->candidates_considered += batch_stats.candidates_considered; + job->stats->candidates_verified += batch_stats.candidates_verified; + for (size_t i = 0; i < buffer->count; ++i) { + qdaln_match_result result = results[i]; + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + 0) != 0) { + free(results); + return 1; + } + ++job->stats->unique; + ++job->stats->exact; + } else if (result.status == QDALN_MATCH_AMBIGUOUS) { + ++job->stats->ambiguous; + } else if (result.status == QDALN_MATCH_NONE) { + ++job->stats->unmatched; + } else { + ++job->stats->invalid; + } + } + free(results); + return 0; + } + size_t read_threads = job->read_threads; + if (read_threads <= 1 || buffer->count < 1024) { + for (size_t i = 0; i < buffer->count; ++i) { + if (direct_hamming_count_seq(job, buffer->items[i], buffer->lens[i]) != 0) return 1; + } + return 0; + } + if (read_threads > buffer->count) read_threads = buffer->count; + + size_t target_slots = job->targets->count * 5; + pthread_t *thread_ids = (pthread_t *)calloc(read_threads, sizeof(pthread_t)); + direct_hamming_batch_job *jobs = (direct_hamming_batch_job *)calloc(read_threads, sizeof(direct_hamming_batch_job)); + if (thread_ids == NULL || jobs == NULL) { + free(thread_ids); + free(jobs); + return 1; + } + + size_t launched = 0; + int rc = 0; + for (size_t t = 0; t < read_threads; ++t) { + size_t start = (buffer->count * t) / read_threads; + size_t end = (buffer->count * (t + 1)) / read_threads; + jobs[t].job = *job; + jobs[t].items = buffer->items; + jobs[t].lens = buffer->lens; + jobs[t].start = start; + jobs[t].end = end; + if (pthread_create(&thread_ids[t], NULL, direct_hamming_batch_worker, &jobs[t]) != 0) { + rc = 1; + break; + } + ++launched; + } + + for (size_t t = 0; t < launched; ++t) { + pthread_join(thread_ids[t], NULL); + if (jobs[t].rc != 0) rc = 1; + } + if (rc == 0) { + size_t dst_offset = job->sample_index * target_slots; + for (size_t t = 0; t < launched; ++t) { + merge_count_stats(job->stats, &jobs[t].local_stats); + for (size_t i = 0; i < jobs[t].dirty_slots.count; ++i) { + size_t slot = jobs[t].dirty_slots.items[i].slot; + job->counts[dst_offset + slot] += jobs[t].dirty_slots.items[i].count; + } + } + } + + for (size_t t = 0; t < read_threads; ++t) free_count_dirty_slots(&jobs[t].dirty_slots); + free(thread_ids); + free(jobs); + return rc; +} + +static void score_offsets_for_seq(const hamming_lookup *lookup, const char *seq, size_t seq_len, + size_t target_start, size_t target_len, size_t range, + unsigned long long *scores) { + if (lookup == NULL || !lookup->ready || lookup->target_len != target_len) return; + size_t n_offsets = 0; + if (offset_count_for_range(range, &n_offsets) != 0) return; + for (size_t oi = 0; oi < n_offsets; ++oi) { + long delta = (long)oi - (long)range; + if (delta < 0 && target_start < (size_t)(-delta)) continue; + size_t offset = delta < 0 ? target_start - (size_t)(-delta) : target_start + (size_t)delta; + if (offset > seq_len || target_len > seq_len - offset) continue; + uint64_t code = 0; + if (!dna2_code_local_fold(seq + offset, target_len, &code)) continue; + const hamming_lookup_entry *entry = hamming_lookup_find(lookup->exact, lookup->exact_cap, code); + if (entry != NULL && entry->match_count == 1) ++scores[oi]; + } +} + +static int select_offsets_from_scores(size_t target_start, size_t range, const unsigned long long *scores, + size_t checked, offset_mode mode, double min_fraction, + offset_list *selected_offsets); + +static int count_sample_worker_direct_hamming(count_sample_job *job) { + fastq_reader reader = {0}; + if (fastq_reader_open(&reader, job->reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + return 1; + } + + char seq[8192]; + int got = 0; + + if (job->fused_offset_detection) { + size_t n_offsets = 0; + if (offset_count_for_range(job->auto_offset, &n_offsets) != 0) { + fastq_reader_close(&reader); + return 1; + } + unsigned long long *scores = (unsigned long long *)calloc(n_offsets, sizeof(unsigned long long)); + if (scores == NULL) { + fastq_reader_close(&reader); + return 1; + } + seq_buffer buffered = {0}; + if (reserve_seq_buffer(&buffered, job->auto_offset_sample) != 0) { + free_seq_buffer(&buffered); + free(scores); + fastq_reader_close(&reader); + return 1; + } + size_t checked = 0; + size_t seq_len = 0; + while (checked < job->auto_offset_sample && + (got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { + score_offsets_for_seq(job->hlookup, seq, seq_len, job->target_start, job->target_len, + job->auto_offset, scores); + if (push_seq_buffer(&buffered, seq, seq_len) != 0) { + free_seq_buffer(&buffered); + free(scores); + fastq_reader_close(&reader); + return 1; + } + ++checked; + } + if (got < 0 || + select_offsets_from_scores(job->target_start, job->auto_offset, scores, checked, job->offsets_mode, + job->offset_min_fraction, job->selected_offsets) != 0) { + free_seq_buffer(&buffered); + free(scores); + fastq_reader_close(&reader); + return 1; + } + free(scores); + if (process_direct_hamming_buffer(job, &buffered) != 0) { + free_seq_buffer(&buffered); + fastq_reader_close(&reader); + return 1; + } + free_seq_buffer(&buffered); + } + + size_t seq_len = 0; + if (job->read_threads <= 1) { + while ((got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { + if (direct_hamming_count_seq(job, seq, seq_len) != 0) { + fastq_reader_close(&reader); + return 1; + } + } + } else { + const size_t batch_reads = 1048576; + seq_buffer batch = {0}; + if (reserve_seq_buffer(&batch, batch_reads) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + return 1; + } + while ((got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { + if (push_seq_buffer(&batch, seq, seq_len) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + return 1; + } + if (batch.count == batch_reads) { + if (process_direct_hamming_buffer(job, &batch) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + return 1; + } + reset_seq_buffer(&batch); + if (reserve_seq_buffer(&batch, batch_reads) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + return 1; + } + } + } + if (got >= 0 && batch.count != 0 && process_direct_hamming_buffer(job, &batch) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + return 1; + } + free_seq_buffer(&batch); + } + fastq_reader_close(&reader); + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + return 1; + } + return 0; +} + +static int pack_read_window_code(const char *seq, size_t seq_len, size_t offset, size_t target_len, uint64_t *code_out) { + if (offset > seq_len || target_len > seq_len - offset) return 0; + return dna2_code_local(seq + offset, target_len, code_out); +} + +static int apply_metal_match_to_counts(count_sample_job *job, const qdmetal_match_result *metal_result) { + qdaln_match_result result = {metal_result->target_index, metal_result->best_distance, + metal_result->second_best_distance, metal_result->match_count, + metal_result->status}; + return direct_hamming_apply_match_result(job, result, 1); +} + +static int count_sample_worker_metal_hamming(count_sample_job *job) { + if (job->metal_target_codes == NULL) return 1; + fastq_reader reader = {0}; + if (fastq_reader_open(&reader, job->reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + return 1; + } + + size_t window_offset = job->selected_offsets == NULL || job->selected_offsets->count == 0 + ? job->target_start + : job->selected_offsets->items[0]; + const size_t batch_cap = 262144; + uint64_t *read_codes = (uint64_t *)malloc(batch_cap * sizeof(uint64_t)); + qdmetal_match_result *metal_results = + (qdmetal_match_result *)malloc(batch_cap * sizeof(qdmetal_match_result)); + if (read_codes == NULL || metal_results == NULL) { + free(read_codes); + free(metal_results); + fastq_reader_close(&reader); + return 1; + } + + char seq[8192]; + size_t seq_len = 0; + size_t batch_count = 0; + int got = 0; + int rc = 0; + + while (rc == 0 && (got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { + ++job->stats->total; + count_progress_tick(job->progress); + uint64_t code = 0; + if (!pack_read_window_code(seq, seq_len, window_offset, job->target_len, &code)) { + if (window_offset > seq_len || job->target_len > seq_len - window_offset) ++job->stats->invalid; + else ++job->stats->unmatched; + continue; + } + read_codes[batch_count++] = code; + if (batch_count == batch_cap) { + qdmetal_assign_stats mstats = {0, 0, NULL, NULL}; + if (qdmetal_hamming_assign(read_codes, batch_count, job->metal_target_codes, job->targets->count, + job->target_len, job->k, metal_results, &mstats) != 0) { + rc = 1; + break; + } + job->stats->candidates_considered += (unsigned long long)mstats.candidates_considered; + job->stats->candidates_verified += (unsigned long long)mstats.candidates_verified; + for (size_t i = 0; i < batch_count; ++i) { + if (apply_metal_match_to_counts(job, &metal_results[i]) != 0) { + rc = 1; + break; + } + } + batch_count = 0; + } + } + + if (rc == 0 && got >= 0 && batch_count != 0) { + qdmetal_assign_stats mstats = {0, 0, NULL, NULL}; + if (qdmetal_hamming_assign(read_codes, batch_count, job->metal_target_codes, job->targets->count, + job->target_len, job->k, metal_results, &mstats) != 0) { + rc = 1; + } else { + job->stats->candidates_considered += (unsigned long long)mstats.candidates_considered; + job->stats->candidates_verified += (unsigned long long)mstats.candidates_verified; + for (size_t i = 0; i < batch_count; ++i) { + if (apply_metal_match_to_counts(job, &metal_results[i]) != 0) { + rc = 1; + break; + } + } + } + } + + free(read_codes); + free(metal_results); + fastq_reader_close(&reader); + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + return 1; + } + return rc; +} + +static int count_sample_sequence(count_sample_job *job, const char *seq, size_t seq_len, const char *qual, + const char *read_id) { + char observed[8192]; + qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_index_stats istats = {0, 0}; + observed[0] = '\0'; + ++job->stats->total; + count_progress_tick(job->progress); + int best_exact_shortcut = job->k == 1 && + job->assignment_policy == AMBIGUITY_POLICY_BEST && job->assignments == NULL && + job->ambiguous_out == NULL && job->unmatched_out == NULL; + int handled = 0; + if (job->metric == COUNT_METRIC_LEVENSHTEIN && job->k == 1 && job->indel_window == 1 && + job->levlookup != NULL && job->levlookup->ready && + (job->selected_offsets == NULL || job->selected_offsets->count <= 1)) { + size_t offset = job->selected_offsets == NULL || job->selected_offsets->count == 0 + ? job->target_start : job->selected_offsets->items[0]; + int lookup_rc = assign_levenshtein1_lookup_offset(job->levlookup, seq, seq_len, offset, &result, + &istats, observed, sizeof(observed)); + if (lookup_rc < 0) return -1; + handled = lookup_rc; + } + if (job->metric == COUNT_METRIC_HAMMING && job->indel_window == 0 && job->hlookup != NULL && job->hlookup->ready) { + int exact_merge = job->assignment_policy == AMBIGUITY_POLICY_RADIUS || + job->assignments != NULL || job->ambiguous_out != NULL || + job->unmatched_out != NULL; + int lookup_rc = assign_hamming_lookup_offsets(job->hlookup, seq, seq_len, job->selected_offsets, 0, + job->k, &result, &istats, observed, sizeof(observed), + exact_merge); + if (lookup_rc < 0) return -1; + handled = lookup_rc; + } + if (!handled && + assign_count_offsets(job->index, seq, seq_len, job->selected_offsets, 0, job->target_len, job->k, + job->metric, job->indel_window, &result, &istats, observed, + sizeof(observed), best_exact_shortcut) != 0) { + return -1; + } + apply_ambiguity_policy(&result, job->assignment_policy); + if (result.status != QDALN_MATCH_INVALID) { + job->stats->candidates_considered += (unsigned long long)istats.candidates_considered; + job->stats->candidates_verified += (unsigned long long)istats.candidates_verified; + } + + const char *correction = "invalid"; + int quality_rejected = 0; + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0 && result.best_distance > 0) { + seq_record *target = &job->targets->records[result.target_index]; + if (!quality_allows_unique_correction(seq, seq_len, qual, job->selected_offsets, 0, job->target_len, + job->metric, job->indel_window, job->k, observed, target, result, + job->max_correction_qual)) { + result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; + quality_rejected = 1; + } + } + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { + seq_record *target = &job->targets->records[result.target_index]; + int kind = correction_kind(observed, strlen(observed), target->seq, target->len, result.best_distance); + size_t slot = ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + (size_t)kind; + if (increment_count_slot(job, slot) != 0) return -1; + ++job->stats->unique; + if (result.best_distance == 0) ++job->stats->exact; + else ++job->stats->corrected; + correction = correction_name(kind); + } else if (result.status == QDALN_MATCH_AMBIGUOUS) { + ++job->stats->ambiguous; + correction = "ambiguous"; + } else if (result.status == QDALN_MATCH_NONE) { + ++job->stats->unmatched; + correction = quality_rejected ? "quality_rejected" : "none"; + } else { + ++job->stats->invalid; + } + + const char *id = read_id == NULL ? "" : read_id; + if (job->assignments != NULL && + (result.status != QDALN_MATCH_AMBIGUOUS || strcmp(job->ambiguous_policy, "report") == 0)) { + write_assignment_like_row(job->assignments, job->targets, job->sample_label, id, observed, result, + correction); + } + if (job->ambiguous_out != NULL && result.status == QDALN_MATCH_AMBIGUOUS) { + write_assignment_like_row(job->ambiguous_out, job->targets, job->sample_label, id, observed, result, + correction); + } + if (job->unmatched_out != NULL && + (result.status == QDALN_MATCH_NONE || result.status == QDALN_MATCH_INVALID)) { + write_assignment_like_row(job->unmatched_out, job->targets, job->sample_label, id, observed, result, + correction); + } + return 0; +} + +typedef struct count_batch_job { + count_sample_job job; + char **items; + size_t *lens; + size_t start; + size_t end; + count_dirty_slots dirty_slots; + count_stats local_stats; + int rc; +} count_batch_job; + +static void *count_batch_worker(void *arg) { + count_batch_job *batch = (count_batch_job *)arg; + batch->job.sample_index = 0; + batch->job.stats = &batch->local_stats; + batch->job.dirty_slots = &batch->dirty_slots; + batch->job.assignments = NULL; + batch->job.ambiguous_out = NULL; + batch->job.unmatched_out = NULL; + batch->rc = 0; + for (size_t i = batch->start; i < batch->end; ++i) { + if (count_sample_sequence(&batch->job, batch->items[i], batch->lens[i], NULL, NULL) != 0) { + batch->rc = 1; + break; + } + } + return NULL; +} + +static int process_count_buffer(count_sample_job *job, const seq_buffer *buffer) { + if (buffer->count == 0) return 0; + if (job->k == 0 && job->metric == COUNT_METRIC_HAMMING && job->indel_window == 0 && job->index != NULL && + job->hlookup != NULL && job->hlookup->ready && job->hlookup->target_len == job->target_len && + job->assignments == NULL && job->ambiguous_out == NULL && job->unmatched_out == NULL && + job->max_correction_qual < 0 && job->assignment_policy == AMBIGUITY_POLICY_BEST) { + qdaln_match_result *results = (qdaln_match_result *)calloc(buffer->count, sizeof(qdaln_match_result)); + if (results == NULL) return 1; + qdaln_index_stats batch_stats = {0, 0}; + if (qdaln_index_lookup_exact_ascii_many_stats(job->index, (const char *const *)buffer->items, buffer->lens, + buffer->count, results, &batch_stats) != 0) { + free(results); + return 1; + } + job->stats->candidates_considered += batch_stats.candidates_considered; + job->stats->candidates_verified += batch_stats.candidates_verified; + for (size_t i = 0; i < buffer->count; ++i) { + qdaln_match_result result = results[i]; + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { + if (increment_count_slot(job, ((job->sample_index * job->targets->count + (size_t)result.target_index) * 5) + 0) != 0) { + free(results); + return 1; + } + ++job->stats->unique; + ++job->stats->exact; + } else if (result.status == QDALN_MATCH_AMBIGUOUS) { + ++job->stats->ambiguous; + } else if (result.status == QDALN_MATCH_NONE) { + ++job->stats->unmatched; + } else { + ++job->stats->invalid; + } + } + free(results); + return 0; + } + size_t read_threads = job->read_threads; + if (read_threads <= 1 || buffer->count < 1024) { + for (size_t i = 0; i < buffer->count; ++i) { + if (count_sample_sequence(job, buffer->items[i], buffer->lens[i], NULL, NULL) != 0) return 1; + } + return 0; + } + if (read_threads > buffer->count) read_threads = buffer->count; + + size_t target_slots = job->targets->count * 5; + pthread_t *thread_ids = (pthread_t *)calloc(read_threads, sizeof(pthread_t)); + count_batch_job *jobs = (count_batch_job *)calloc(read_threads, sizeof(count_batch_job)); + if (thread_ids == NULL || jobs == NULL) { + free(thread_ids); + free(jobs); + return 1; + } + + size_t launched = 0; + int rc = 0; + for (size_t t = 0; t < read_threads; ++t) { + size_t start = (buffer->count * t) / read_threads; + size_t end = (buffer->count * (t + 1)) / read_threads; + jobs[t].job = *job; + jobs[t].items = buffer->items; + jobs[t].lens = buffer->lens; + jobs[t].start = start; + jobs[t].end = end; + if (pthread_create(&thread_ids[t], NULL, count_batch_worker, &jobs[t]) != 0) { + rc = 1; + break; + } + ++launched; + } + + for (size_t t = 0; t < launched; ++t) { + pthread_join(thread_ids[t], NULL); + if (jobs[t].rc != 0) rc = 1; + } + if (rc == 0) { + size_t dst_offset = job->sample_index * target_slots; + for (size_t t = 0; t < launched; ++t) { + merge_count_stats(job->stats, &jobs[t].local_stats); + for (size_t i = 0; i < jobs[t].dirty_slots.count; ++i) { + size_t slot = jobs[t].dirty_slots.items[i].slot; + job->counts[dst_offset + slot] += jobs[t].dirty_slots.items[i].count; + } + } + } + + for (size_t t = 0; t < read_threads; ++t) free_count_dirty_slots(&jobs[t].dirty_slots); + free(thread_ids); + free(jobs); + return rc; +} + +static void *count_sample_worker(void *arg); + +typedef struct count_samples_args { + const qdaln_index *index; + const hamming_lookup *hlookup; + const levenshtein1_lookup *levlookup; + const seq_table *targets; + const char **target_ptrs; + const size_t *target_lens; + const string_list *reads; + const string_list *labels; + offset_list *selected_offsets; + size_t target_len; + int k; + count_metric metric; + size_t indel_window; + unsigned long long *counts; + count_stats *stats_by_sample; + FILE *assignments; + FILE *ambiguous_out; + FILE *unmatched_out; + const char *ambiguous_policy; + ambiguity_policy assignment_policy; + int direct_hamming_counts; + int metal_hamming_counts; + const uint64_t *metal_target_codes; + int fused_offset_detection; + size_t target_start; + size_t auto_offset; + size_t auto_offset_sample; + offset_mode offsets_mode; + double offset_min_fraction; + size_t effective_read_threads; + int max_correction_qual; + size_t sample_threads; + count_progress *progress_by_sample; +} count_samples_args; + +static int run_count_samples_phase(const count_samples_args *args) { + if (args == NULL || args->reads == NULL || args->labels == NULL || args->counts == NULL || + args->stats_by_sample == NULL) { + return -1; + } + size_t sample_threads = args->sample_threads; + if (sample_threads <= 1 || args->reads->count <= 1) { + for (size_t sample = 0; sample < args->reads->count; ++sample) { + count_sample_job job = { + args->index, args->hlookup, args->levlookup, args->targets, args->target_ptrs, args->target_lens, + args->reads->items[sample], args->labels->items[sample], sample, &args->selected_offsets[sample], + args->target_len, args->k, args->metric, args->indel_window, args->counts, &args->stats_by_sample[sample], + args->assignments, args->ambiguous_out, args->unmatched_out, args->ambiguous_policy, + args->assignment_policy, args->direct_hamming_counts, args->metal_hamming_counts, + args->metal_target_codes, args->fused_offset_detection, args->target_start, args->auto_offset, + args->auto_offset_sample, args->offsets_mode, args->offset_min_fraction, args->effective_read_threads, + args->max_correction_qual, 1, NULL, + args->progress_by_sample != NULL ? &args->progress_by_sample[sample] : NULL}; + count_sample_worker(&job); + if (job.rc != 0) return job.rc; + } + return 0; + } + + pthread_t *thread_ids = (pthread_t *)calloc(sample_threads, sizeof(pthread_t)); + count_sample_job *jobs = (count_sample_job *)calloc(args->reads->count, sizeof(count_sample_job)); + if (thread_ids == NULL || jobs == NULL) { + free(thread_ids); + free(jobs); + fprintf(stderr, "out of memory\n"); + return -1; + } + int rc = 0; + size_t next_sample = 0; + while (next_sample < args->reads->count && rc == 0) { + size_t batch = args->reads->count - next_sample; + if (batch > sample_threads) batch = sample_threads; + for (size_t i = 0; i < batch; ++i) { + size_t sample = next_sample + i; + jobs[sample] = (count_sample_job){ + args->index, args->hlookup, args->levlookup, args->targets, args->target_ptrs, args->target_lens, + args->reads->items[sample], args->labels->items[sample], sample, &args->selected_offsets[sample], + args->target_len, args->k, args->metric, args->indel_window, args->counts, &args->stats_by_sample[sample], + NULL, NULL, NULL, args->ambiguous_policy, args->assignment_policy, args->direct_hamming_counts, + args->metal_hamming_counts, args->metal_target_codes, args->fused_offset_detection, args->target_start, + args->auto_offset, args->auto_offset_sample, args->offsets_mode, args->offset_min_fraction, 1, + args->max_correction_qual, 1, NULL, + args->progress_by_sample != NULL ? &args->progress_by_sample[sample] : NULL}; + if (pthread_create(&thread_ids[i], NULL, count_sample_worker, &jobs[sample]) != 0) { + fprintf(stderr, "failed to create worker thread\n"); + batch = i; + rc = -1; + break; + } + } + for (size_t i = 0; i < batch; ++i) { + pthread_join(thread_ids[i], NULL); + if (rc == 0 && jobs[next_sample + i].rc != 0) rc = jobs[next_sample + i].rc; + } + next_sample += batch; + } + free(thread_ids); + free(jobs); + return rc; +} + +static unsigned long long count_matrix_cell_total(const unsigned long long *counts, size_t n_targets, size_t sample, + size_t target) { + unsigned long long total = 0; + for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * n_targets + target) * 5) + kind]; + return total; +} + +static int validate_metal_counts_against_cpu(const unsigned long long *metal_counts, const unsigned long long *cpu_counts, + size_t n_samples, size_t n_targets, size_t *diff_guides_out, + long long *delta_reads_out, char *example_guide, size_t example_cap) { + if (metal_counts == NULL || cpu_counts == NULL || diff_guides_out == NULL || delta_reads_out == NULL) return -1; + *diff_guides_out = 0; + *delta_reads_out = 0; + if (example_guide != NULL && example_cap > 0) example_guide[0] = '\0'; + for (size_t t = 0; t < n_targets; ++t) { + int guide_diff = 0; + for (size_t s = 0; s < n_samples; ++s) { + unsigned long long metal_total = count_matrix_cell_total(metal_counts, n_targets, s, t); + unsigned long long cpu_total = count_matrix_cell_total(cpu_counts, n_targets, s, t); + if (metal_total != cpu_total) { + guide_diff = 1; + *delta_reads_out += (long long)metal_total - (long long)cpu_total; + if (example_guide != NULL && example_guide[0] == '\0' && example_cap > 0) { + snprintf(example_guide, example_cap, "sample_index=%zu target_index=%zu metal=%llu cpu=%llu", s, t, + metal_total, cpu_total); + } + } + } + if (guide_diff) ++*diff_guides_out; + } + return *diff_guides_out == 0 ? 0 : 1; +} + +static int env_truthy(const char *value) { + if (value == NULL || value[0] == '\0') return 0; + if (strcmp(value, "0") == 0) return 0; + if (strcmp(value, "false") == 0 || strcmp(value, "FALSE") == 0) return 0; + if (strcmp(value, "no") == 0 || strcmp(value, "NO") == 0) return 0; + return 1; +} + +static void *count_sample_worker(void *arg) { + count_sample_job *job = (count_sample_job *)arg; + if (job->metal_hamming_counts) { + job->rc = count_sample_worker_metal_hamming(job); + return NULL; + } + if (job->direct_hamming_counts) { + job->rc = count_sample_worker_direct_hamming(job); + return NULL; + } + + fastq_reader reader = {0}; + if (fastq_reader_open(&reader, job->reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + job->rc = 1; + return NULL; + } + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + char read_id[8192]; + int got = 0; + int need_read_id = job->assignments != NULL || job->ambiguous_out != NULL || job->unmatched_out != NULL; + int need_quality = job->max_correction_qual >= 0; + int need_full_record = need_read_id || need_quality; + size_t seq_len = 0; + if (job->read_threads > 1 && !need_full_record) { + const size_t batch_reads = 1048576; + seq_buffer batch = {0}; + if (reserve_seq_buffer(&batch, batch_reads) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + job->rc = 1; + return NULL; + } + while ((got = fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { + if (push_seq_buffer(&batch, seq, seq_len) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + job->rc = 1; + return NULL; + } + if (batch.count == batch_reads) { + if (process_count_buffer(job, &batch) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + fprintf(stderr, "FASTQ assignment failed\n"); + job->rc = 1; + return NULL; + } + reset_seq_buffer(&batch); + if (reserve_seq_buffer(&batch, batch_reads) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + job->rc = 1; + return NULL; + } + } + } + if (got >= 0 && batch.count != 0 && process_count_buffer(job, &batch) != 0) { + free_seq_buffer(&batch); + fastq_reader_close(&reader); + fprintf(stderr, "FASTQ assignment failed\n"); + job->rc = 1; + return NULL; + } + free_seq_buffer(&batch); + } else { + while ((got = need_full_record + ? fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len) + : fastq_read_sequence_record_len(&reader, seq, sizeof(seq), &seq_len)) == 1) { + read_id[0] = '\0'; + if (need_read_id) fastq_read_id(header, read_id, sizeof(read_id)); + if (count_sample_sequence(job, seq, seq_len, job->max_correction_qual >= 0 ? qual : NULL, + read_id) != 0) { + fastq_reader_close(&reader); + fprintf(stderr, "FASTQ assignment failed\n"); + job->rc = 1; + return NULL; + } + } + } + fastq_reader_close(&reader); + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + job->rc = 1; + return NULL; + } + job->rc = 0; + return NULL; +} + +static int select_offsets_from_scores(size_t target_start, size_t range, const unsigned long long *scores, + size_t checked, offset_mode mode, double min_fraction, + offset_list *selected_offsets) { + free_offset_list(selected_offsets); + if (range == 0) return push_offset_unique(selected_offsets, target_start); + + size_t n_offsets = 0; + if (offset_count_for_range(range, &n_offsets) != 0) return -1; + + size_t best_i = range; + for (size_t oi = 0; oi < n_offsets; ++oi) { + size_t best_dist = best_i > range ? best_i - range : range - best_i; + size_t this_dist = oi > range ? oi - range : range - oi; + if (scores[oi] > scores[best_i] || (scores[oi] == scores[best_i] && this_dist < best_dist)) { + best_i = oi; + } + } + + int rc = 0; + if (mode == OFFSET_MODE_MULTI && checked != 0) { + for (size_t oi = 0; oi < n_offsets; ++oi) { + double fraction = (double)scores[oi] / (double)checked; + if (fraction + 1e-12 < min_fraction) continue; + long delta = (long)oi - (long)range; + if (delta < 0 && target_start < (size_t)(-delta)) continue; + size_t offset = delta < 0 ? target_start - (size_t)(-delta) : target_start + (size_t)delta; + if (push_offset_unique(selected_offsets, offset) != 0) { + rc = -1; + break; + } + } + } + if (rc == 0 && selected_offsets->count == 0 && scores[best_i] != 0) { + long best_delta = (long)best_i - (long)range; + size_t offset = best_delta < 0 ? target_start - (size_t)(-best_delta) : target_start + (size_t)best_delta; + rc = push_offset_unique(selected_offsets, offset); + } + if (rc == 0 && selected_offsets->count == 0) { + rc = push_offset_unique(selected_offsets, target_start); + } + return rc; +} + +static int detect_offsets(const qdaln_index *index, const hamming_lookup *exact_lookup, const char *reads_path, size_t target_start, + size_t target_len, size_t range, size_t sample_limit, offset_mode mode, + double min_fraction, offset_list *selected_offsets) { + if (range == 0) { + free_offset_list(selected_offsets); + return push_offset_unique(selected_offsets, target_start); + } + + size_t n_offsets = 0; + if (offset_count_for_range(range, &n_offsets) != 0) return -1; + unsigned long long *scores = (unsigned long long *)calloc(n_offsets, sizeof(unsigned long long)); + if (scores == NULL) return -1; + + fastq_reader reader = {0}; + if (fastq_reader_open(&reader, reads_path) != 0) { + free(scores); + return -1; + } + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + size_t checked = 0; + int got = 0; + size_t seq_len = 0; + while (checked < sample_limit && + (got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { + if (exact_lookup != NULL && exact_lookup->ready && exact_lookup->target_len == target_len) { + score_offsets_for_seq(exact_lookup, seq, seq_len, target_start, target_len, range, scores); + } else { + for (size_t oi = 0; oi < n_offsets; ++oi) { + long delta = (long)oi - (long)range; + if (delta < 0 && target_start < (size_t)(-delta)) continue; + size_t offset = delta < 0 ? target_start - (size_t)(-delta) : target_start + (size_t)delta; + if (offset > seq_len || target_len > seq_len - offset) continue; + char observed[8192]; + if (target_len >= sizeof(observed)) continue; + memcpy(observed, seq + offset, target_len); + observed[target_len] = '\0'; + uppercase_ascii(observed); + const char *read_ptr = observed; + size_t read_len = target_len; + qdaln_match_result r; + if (qdaln_index_assign(index, &read_ptr, &read_len, 1, 0, &r) != 0) { + fastq_reader_close(&reader); + free(scores); + return -1; + } + if (r.status == QDALN_MATCH_UNIQUE) ++scores[oi]; + } + } + ++checked; + } + + fastq_reader_close(&reader); + if (got < 0) { + free(scores); + return -1; + } + + int rc = select_offsets_from_scores(target_start, range, scores, checked, mode, min_fraction, selected_offsets); + free(scores); + return rc; +} + +typedef struct offset_detect_job { + const qdaln_index *index; + const hamming_lookup *exact_lookup; + const char *reads_path; + size_t target_start; + size_t target_len; + size_t range; + size_t sample_limit; + offset_mode mode; + double min_fraction; + offset_list *selected_offsets; + int rc; +} offset_detect_job; + +static void *detect_offsets_worker(void *arg) { + offset_detect_job *job = (offset_detect_job *)arg; + job->rc = detect_offsets(job->index, job->exact_lookup, job->reads_path, job->target_start, job->target_len, + job->range, job->sample_limit, job->mode, job->min_fraction, job->selected_offsets); + return NULL; +} + +static int detect_offsets_for_samples(const qdaln_index *index, const hamming_lookup *exact_lookup, + const string_list *reads, size_t target_start, size_t target_len, + size_t range, size_t sample_limit, offset_mode mode, double min_fraction, + offset_list *selected_offsets, size_t threads) { + if (reads->count == 0) return 0; + if (threads > reads->count) threads = reads->count; + if (threads <= 1 || reads->count <= 1) { + for (size_t sample = 0; sample < reads->count; ++sample) { + if (detect_offsets(index, exact_lookup, reads->items[sample], target_start, target_len, range, + sample_limit, mode, min_fraction, &selected_offsets[sample]) != 0) { + return -1; + } + } + return 0; + } + + pthread_t *thread_ids = (pthread_t *)calloc(threads, sizeof(pthread_t)); + offset_detect_job *jobs = (offset_detect_job *)calloc(reads->count, sizeof(offset_detect_job)); + if (thread_ids == NULL || jobs == NULL) { + free(thread_ids); + free(jobs); + return -1; + } + + int rc = 0; + size_t next_sample = 0; + while (next_sample < reads->count && rc == 0) { + size_t batch = reads->count - next_sample; + if (batch > threads) batch = threads; + for (size_t i = 0; i < batch; ++i) { + size_t sample = next_sample + i; + jobs[sample] = (offset_detect_job){index, exact_lookup, reads->items[sample], target_start, target_len, + range, sample_limit, mode, min_fraction, + &selected_offsets[sample], 0}; + if (pthread_create(&thread_ids[i], NULL, detect_offsets_worker, &jobs[sample]) != 0) { + batch = i; + rc = -1; + break; + } + } + for (size_t i = 0; i < batch; ++i) { + pthread_join(thread_ids[i], NULL); + if (jobs[next_sample + i].rc != 0) rc = -1; + } + next_sample += batch; + } + + free(thread_ids); + free(jobs); + return rc; +} + +static int run_count(const char *argv0, int argc, char **argv) { + const char *targets_path = NULL; + const char *samples_path = NULL; + const char *out_path = NULL; + const char *assignments_path = NULL; + const char *summary_path = NULL; + const char *report_path = NULL; + const char *report_audit_dir = NULL; + const char *report_unmatched_path = NULL; + const char *ambiguous_path = NULL; + const char *unmatched_path = NULL; + const char *sample_qc_path = NULL; + const char *target_counts_long_path = NULL; + const int crispr_mode = strcmp(argv[1], "crispr-count") == 0; + const char *format = crispr_mode ? "mageck" : "dotmatch"; + const char *ambiguous_policy = "discard"; + ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; + count_metric metric = COUNT_METRIC_LEVENSHTEIN; + hamming_index_strategy hamming_strategy = HAMMING_INDEX_AUTO; + size_t target_start = 0; + size_t target_len = 0; + size_t indel_window = 0; + size_t auto_offset = 0; + size_t auto_offset_sample = 1000; + offset_mode offsets_mode = OFFSET_MODE_BEST; + double offset_min_fraction = 0.005; + size_t threads = 0; + int max_correction_qual = -1; + int k = -1; + count_backend_mode backend_mode = COUNT_BACKEND_AUTO; + string_list reads = {0}; + string_list labels = {0}; + int show_progress = isatty(STDERR_FILENO); + size_t progress_interval_reads = 250000; + char default_sample_qc_path[PATH_MAX]; + count_progress *progress_by_sample = NULL; + int metal_validate = env_truthy(getenv("DOTMATCH_METAL_VALIDATE")); + const char *metal_validation_status = NULL; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if ((strcmp(arg, "--targets") == 0 || strcmp(arg, "--library") == 0) && i < argc) { + targets_path = argv[i++]; + } else if (strcmp(arg, "--samples") == 0 && i < argc) { + samples_path = argv[i++]; + } else if (strcmp(arg, "--reads") == 0 && i < argc) { + if (push_string(&reads, argv[i++]) != 0) { + fprintf(stderr, "out of memory\n"); + goto fail_args; + } + } else if (strcmp(arg, "--sample-label") == 0 && i < argc) { + if (split_string_list(&labels, argv[i++], ',') != 0) { + fprintf(stderr, "out of memory\n"); + goto fail_args; + } + } else if ((strcmp(arg, "--target-start") == 0 || strcmp(arg, "--guide-start") == 0) && i < argc) { + if (parse_size_value(argv[i++], &target_start) != 0) { + usage(argv0); + goto fail_args; + } + } else if ((strcmp(arg, "--target-length") == 0 || strcmp(arg, "--guide-length") == 0) && i < argc) { + if (parse_size_value(argv[i++], &target_len) != 0 || target_len == 0) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 3) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--metric") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "hamming") == 0) { + metric = COUNT_METRIC_HAMMING; + } else if (strcmp(value, "levenshtein") == 0) { + metric = COUNT_METRIC_LEVENSHTEIN; + } else { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--hamming-index") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "auto") == 0) { + hamming_strategy = HAMMING_INDEX_AUTO; + } else if (strcmp(value, "query") == 0) { + hamming_strategy = HAMMING_INDEX_QUERY; + } else if (strcmp(value, "precompute") == 0) { + hamming_strategy = HAMMING_INDEX_PRECOMPUTE; + } else { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--indel-window") == 0 && i < argc) { + if (parse_size_value(argv[i++], &indel_window) != 0 || indel_window > 1) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--auto-offset") == 0 && i < argc) { + if (parse_size_value(argv[i++], &auto_offset) != 0) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--auto-offset-sample") == 0 && i < argc) { + if (parse_size_value(argv[i++], &auto_offset_sample) != 0 || auto_offset_sample == 0) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--offset-mode") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "best") == 0) { + offsets_mode = OFFSET_MODE_BEST; + } else if (strcmp(value, "multi") == 0) { + offsets_mode = OFFSET_MODE_MULTI; + } else { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--offset-min-fraction") == 0 && i < argc) { + if (parse_double_value(argv[i++], &offset_min_fraction) != 0 || + offset_min_fraction < 0.0 || offset_min_fraction > 1.0) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--backend") == 0 && i < argc) { + if (parse_count_backend_mode(argv[i++], &backend_mode) != 0) { + fprintf(stderr, "--backend must be auto, cpu, or gpu-metal-experimental\n"); + goto fail_args; + } + } else if (strcmp(arg, "--progress") == 0) { + show_progress = 1; + } else if (strcmp(arg, "--no-progress") == 0) { + show_progress = 0; + } else if (strcmp(arg, "--progress-interval") == 0 && i < argc) { + if (parse_size_value(argv[i++], &progress_interval_reads) != 0 || progress_interval_reads == 0) { + fprintf(stderr, "--progress-interval must be a positive integer\n"); + goto fail_args; + } + } else if (strcmp(arg, "--metal-validate") == 0) { + metal_validate = 1; + } else if (strcmp(arg, "--threads") == 0 && i < argc) { + if (parse_size_value(argv[i++], &threads) != 0) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--max-correction-qual") == 0 && i < argc) { + if (parse_int_value(argv[i++], &max_correction_qual) != 0 || + max_correction_qual < 0 || max_correction_qual > 93) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--out") == 0 && i < argc) { + out_path = argv[i++]; + } else if (strcmp(arg, "--assignments") == 0 && i < argc) { + assignments_path = argv[i++]; + } else if (strcmp(arg, "--summary") == 0 && i < argc) { + summary_path = argv[i++]; + } else if (strcmp(arg, "--report") == 0 && i < argc) { + report_path = argv[i++]; + } else if (strcmp(arg, "--report-audit-dir") == 0 && i < argc) { + report_audit_dir = argv[i++]; + } else if (strcmp(arg, "--report-unmatched") == 0 && i < argc) { + report_unmatched_path = argv[i++]; + } else if (strcmp(arg, "--qc") == 0 && i < argc) { + sample_qc_path = argv[i++]; + } else if (strcmp(arg, "--sample-qc") == 0 && i < argc) { + sample_qc_path = argv[i++]; + } else if (strcmp(arg, "--target-counts-long") == 0 && i < argc) { + target_counts_long_path = argv[i++]; + } else if (strcmp(arg, "--ambiguous-out") == 0 && i < argc) { + ambiguous_path = argv[i++]; + } else if (strcmp(arg, "--unmatched-out") == 0 && i < argc) { + unmatched_path = argv[i++]; + } else if (strcmp(arg, "--ambiguous") == 0 && i < argc) { + ambiguous_policy = argv[i++]; + if (strcmp(ambiguous_policy, "discard") != 0 && strcmp(ambiguous_policy, "report") != 0) { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "best") == 0) { + assignment_policy = AMBIGUITY_POLICY_BEST; + } else if (strcmp(value, "radius") == 0) { + assignment_policy = AMBIGUITY_POLICY_RADIUS; + } else { + usage(argv0); + goto fail_args; + } + } else if (strcmp(arg, "--format") == 0 && i < argc) { + format = argv[i++]; + if (strcmp(format, "dotmatch") != 0 && strcmp(format, "mageck") != 0) { + usage(argv0); + goto fail_args; + } + } else { + usage(argv0); + goto fail_args; + } + } + + if (samples_path != NULL && read_samples_file(samples_path, &labels, &reads) != 0) { + fprintf(stderr, "failed to read samples file\n"); + goto fail_args; + } + if (targets_path == NULL || reads.count == 0 || out_path == NULL || target_len == 0 || k < 0) { + usage(argv0); + goto fail_args; + } + if (crispr_mode && sample_qc_path == NULL && + derive_output_sibling_path(out_path, "sample_qc.tsv", default_sample_qc_path, + sizeof(default_sample_qc_path)) == 0) { + sample_qc_path = default_sample_qc_path; + } + if (auto_offset > MAX_AUTO_OFFSET) { + fprintf(stderr, "--auto-offset must be <= %d\n", MAX_AUTO_OFFSET); + goto fail_args; + } + if (metric == COUNT_METRIC_HAMMING && indel_window != 0) { + fprintf(stderr, "--indel-window is only valid with --metric levenshtein\n"); + goto fail_args; + } + if (metric == COUNT_METRIC_LEVENSHTEIN && k > 2) { + fprintf(stderr, "--metric levenshtein supports --k up to 2\n"); + goto fail_args; + } + if (indel_window != 0 && k != 1) { + fprintf(stderr, "--indel-window requires --k 1\n"); + goto fail_args; + } + if (labels.count == 0) { + for (size_t i = 0; i < reads.count; ++i) { + if (push_string(&labels, path_basename(reads.items[i])) != 0) { + fprintf(stderr, "out of memory\n"); + goto fail_args; + } + } + } + if (labels.count != reads.count) { + fprintf(stderr, "--sample-label count must match --reads count\n"); + goto fail_args; + } + if (validate_unique_sample_labels(&labels, crispr_mode ? "--samples sample" : "--sample-label") != 0) { + goto fail_args; + } + if (threads > 1 && (assignments_path != NULL || ambiguous_path != NULL || unmatched_path != NULL)) { + fprintf(stderr, "--threads > 1 is not supported with row-level diagnostic outputs\n"); + goto fail_args; + } + if (threads == 0) { + size_t auto_t = get_cpu_count(); + if (auto_t > 1 && (assignments_path != NULL || ambiguous_path != NULL || unmatched_path != NULL)) { + threads = 1; /* fall back for safe ordered diagnostic outputs */ + } else { + threads = auto_t; + } + } + int count_only = assignments_path == NULL && ambiguous_path == NULL && unmatched_path == NULL; + + seq_table targets = {0}; + qdaln_index *index = NULL; + hamming_lookup hlookup = {0}; + hamming_lookup offset_lookup = {0}; + levenshtein1_lookup levlookup = {0}; + const char **target_ptrs = NULL; + size_t *target_lens = NULL; + unsigned char *ambiguous_nearby = NULL; + unsigned long long *counts = NULL; + count_stats *stats_by_sample = NULL; + offset_list *selected_offsets = NULL; + FILE *out = NULL; + FILE *assignments = NULL; + FILE *ambiguous_out = NULL; + FILE *unmatched_out = NULL; + int rc = 1; + double run_start_seconds = seconds_now(); + double target_index_seconds = 0.0; + double offset_detection_seconds = 0.0; + double hamming_precompute_seconds = 0.0; + double counting_seconds = 0.0; + const char *offset_detection_strategy = auto_offset == 0 ? "none" : "prepass"; + const char *count_engine = "generic_indexed"; + size_t effective_read_threads = threads; + const char *backend_effective = "cpu"; + int metal_hamming_counts = 0; + uint64_t *metal_target_codes = NULL; + + double phase_start_seconds = seconds_now(); + if (read_target_table(targets_path, &targets) != 0) { + fprintf(stderr, "failed to read targets\n"); + goto done; + } + int target_id_check = validate_unique_seq_ids(&targets, crispr_mode ? "guide" : "target"); + if (target_id_check != 0) { + if (target_id_check == -1) fprintf(stderr, "out of memory\n"); + goto done; + } + if (metric == COUNT_METRIC_HAMMING && !all_targets_have_length(&targets, target_len)) { + fprintf(stderr, "--metric hamming requires every target to have --target-length bases\n"); + goto done; + } + target_index_seconds = seconds_now() - phase_start_seconds; + + int hamming_lookup_eligible = hamming_lookup_counts_eligible( + count_only, max_correction_qual, metric, indel_window, k, target_len, hamming_strategy); + int direct_hamming_counts = hamming_direct_worker_eligible(hamming_lookup_eligible, assignment_policy, k); + int may_use_metal = backend_mode == COUNT_BACKEND_METAL && hamming_lookup_eligible && qdmetal_available() && + (k == 0 || assignment_policy == AMBIGUITY_POLICY_BEST); + if (direct_hamming_counts) { + phase_start_seconds = seconds_now(); + int use_mismatch_precompute_now = k == 1 && + (hamming_strategy == HAMMING_INDEX_PRECOMPUTE || + (hamming_strategy == HAMMING_INDEX_AUTO && auto_offset != 0 && + offsets_mode == OFFSET_MODE_MULTI)); + int lookup_rc = 0; + if (k == 0) { + lookup_rc = build_hamming_exact_lookup(&targets, target_len, &hlookup); + } else if (use_mismatch_precompute_now) { + lookup_rc = build_hamming_lookup(&targets, target_len, &hlookup); + } else { + lookup_rc = build_hamming_seed_lookup(&targets, target_len, &hlookup); + } + if (lookup_rc != 0) { + fprintf(stderr, "failed to build Hamming lookup\n"); + goto done; + } + hamming_precompute_seconds = seconds_now() - phase_start_seconds; + if (hlookup.ready) { + count_engine = "hamming_lookup_direct"; + } else { + direct_hamming_counts = 0; + } + } + + int need_general_index = !direct_hamming_counts || strcmp(format, "dotmatch") == 0; + if (need_general_index) { + phase_start_seconds = seconds_now(); + if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + index = qdaln_index_build(target_ptrs, target_lens, targets.count); + if (index == NULL) { + fprintf(stderr, "failed to build target index\n"); + goto done; + } + target_index_seconds += seconds_now() - phase_start_seconds; + } + + size_t sample_target_slots = 0; + size_t total_slots = 0; + if (checked_mul_size(reads.count, targets.count, &sample_target_slots) != 0 || + checked_mul_size(sample_target_slots, 5, &total_slots) != 0) { + fprintf(stderr, "count matrix is too large\n"); + goto done; + } + counts = (unsigned long long *)calloc(alloc_count_or_one(total_slots), sizeof(unsigned long long)); + stats_by_sample = (count_stats *)calloc(alloc_count_or_one(reads.count), sizeof(count_stats)); + ambiguous_nearby = (unsigned char *)calloc(alloc_count_or_one(targets.count), sizeof(unsigned char)); + selected_offsets = (offset_list *)calloc(alloc_count_or_one(reads.count), sizeof(offset_list)); + if (counts == NULL || stats_by_sample == NULL || ambiguous_nearby == NULL || selected_offsets == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t sample = 0; sample < reads.count; ++sample) { + if (push_offset_unique(&selected_offsets[sample], target_start) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + } + + if (strcmp(format, "dotmatch") == 0) { + for (size_t i = 0; i < targets.count; ++i) { + qdaln_match_result r; + qdaln_index_stats s; + const char *seq_ptr = targets.records[i].seq; + size_t seq_len = targets.records[i].len; + int assign_rc = metric == COUNT_METRIC_HAMMING + ? qdaln_index_assign_hamming_stats(index, &seq_ptr, &seq_len, 1, k, &r, &s) + : qdaln_index_assign_stats(index, &seq_ptr, &seq_len, 1, k, &r, &s); + if (assign_rc != 0) { + fprintf(stderr, "target ambiguity check failed\n"); + goto done; + } + ambiguous_nearby[i] = r.match_count > 1 ? 1 : 0; + } + } + + if (assignments_path != NULL) { + assignments = open_output_file(assignments_path); + if (assignments == NULL) { + fprintf(stderr, "failed to open assignments output\n"); + goto done; + } + fprintf(assignments, "sample\tread_id\tobserved_seq\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\tcorrection\n"); + } + if (ambiguous_path != NULL) { + ambiguous_out = open_output_file(ambiguous_path); + if (ambiguous_out == NULL) { + fprintf(stderr, "failed to open ambiguous output\n"); + goto done; + } + fprintf(ambiguous_out, "sample\tread_id\tobserved_seq\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\tcorrection\n"); + } + if (unmatched_path != NULL) { + unmatched_out = open_output_file(unmatched_path); + if (unmatched_out == NULL) { + fprintf(stderr, "failed to open unmatched output\n"); + goto done; + } + fprintf(unmatched_out, "sample\tread_id\tobserved_seq\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\tcorrection\n"); + } + + int metal_blocks_fused_offset = may_use_metal && offsets_mode != OFFSET_MODE_MULTI; + int fused_offset_detection = direct_hamming_counts && auto_offset != 0 && !metal_blocks_fused_offset; + if (fused_offset_detection) { + offset_detection_strategy = "fused"; + } + + if (auto_offset != 0 && !fused_offset_detection) { + phase_start_seconds = seconds_now(); + const hamming_lookup *offset_lookup_ptr = hlookup.ready ? &hlookup : NULL; + if (metric == COUNT_METRIC_HAMMING && offset_lookup_ptr == NULL) { + int lookup_rc = build_hamming_exact_lookup(&targets, target_len, &offset_lookup); + if (lookup_rc != 0) { + fprintf(stderr, "failed to build offset detection lookup\n"); + goto done; + } + if (offset_lookup.ready) offset_lookup_ptr = &offset_lookup; + } + if (detect_offsets_for_samples(index, offset_lookup_ptr, &reads, target_start, target_len, auto_offset, + auto_offset_sample, offsets_mode, offset_min_fraction, selected_offsets, + threads) != 0) { + fprintf(stderr, "automatic offset detection failed\n"); + goto done; + } + offset_detection_seconds = seconds_now() - phase_start_seconds; + free_hamming_lookup(&offset_lookup); + } + + size_t max_selected_offsets = 0; + for (size_t sample = 0; sample < reads.count; ++sample) { + if (selected_offsets[sample].count > max_selected_offsets) max_selected_offsets = selected_offsets[sample].count; + } + int direct_levenshtein_counts = levenshtein1_lookup_counts_eligible( + count_only, max_correction_qual, metric, indel_window, k, target_len, max_selected_offsets, + assignments, ambiguous_out, unmatched_out, assignment_policy); + if (direct_levenshtein_counts) { + phase_start_seconds = seconds_now(); + int lookup_rc = build_levenshtein1_lookup(&targets, target_len, &levlookup); + if (lookup_rc != 0) { + fprintf(stderr, "failed to build Levenshtein k=1 lookup\n"); + goto done; + } + if (levlookup.ready) { + count_engine = "levenshtein_k1_lookup_direct"; + target_index_seconds += seconds_now() - phase_start_seconds; + } else { + direct_levenshtein_counts = 0; + } + } + if (metal_hamming_count_eligible(backend_mode, hamming_lookup_eligible, assignment_policy, k, + max_selected_offsets, fused_offset_detection)) { + int pack_rc = build_packed_target_codes(&targets, target_len, &metal_target_codes); + if (pack_rc == 1) { + metal_hamming_counts = 1; + direct_hamming_counts = 0; + count_engine = metal_count_engine_name(targets.count); + backend_effective = "gpu-metal-experimental"; + } else if (pack_rc < 0) { + goto done; + } else if (backend_mode == COUNT_BACKEND_METAL) { + fprintf(stderr, + "Metal backend requires packable A/C/G/T targets, count-only output, single offset, and " + "best-distance policy for k=1\n"); + goto done; + } + } else if (backend_mode == COUNT_BACKEND_METAL) { + fprintf(stderr, + "Metal backend unavailable for this workload; requires Darwin Metal, --metric hamming, k 0|1, " + "count-only output, and single offset"); + if (k == 1) fprintf(stderr, " with --ambiguity-policy best"); + fprintf(stderr, "\n"); + goto done; + } else if (direct_hamming_counts && !fused_offset_detection && max_selected_offsets <= 1) { + count_engine = "hamming_lookup_direct_single_offset"; + } + int use_precomputed_hamming = metric == COUNT_METRIC_HAMMING && k == 1 && + (hamming_strategy == HAMMING_INDEX_PRECOMPUTE || + (hamming_strategy == HAMMING_INDEX_AUTO && max_selected_offsets > 1)); + if (use_precomputed_hamming && (!hlookup.ready || hlookup.mismatch == NULL)) { + phase_start_seconds = seconds_now(); + free_hamming_lookup(&hlookup); + int lookup_rc = build_hamming_lookup(&targets, target_len, &hlookup); + if (lookup_rc != 0) { + fprintf(stderr, "failed to build Hamming lookup\n"); + goto done; + } + hamming_precompute_seconds = seconds_now() - phase_start_seconds; + } + + if (show_progress) { + progress_by_sample = (count_progress *)calloc(reads.count, sizeof(count_progress)); + if (progress_by_sample == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t sample = 0; sample < reads.count; ++sample) { + count_progress_init(&progress_by_sample[sample], labels.items[sample], progress_interval_reads); + } + } + + phase_start_seconds = seconds_now(); + size_t sample_threads = threads; + if ((direct_hamming_counts || metal_hamming_counts) && reads.count == 1 && threads > 1) { + effective_read_threads = threads; + sample_threads = 1; + } else if (count_only && reads.count == 1 && threads > 1) { + effective_read_threads = threads; + sample_threads = 1; + } else if (sample_threads > reads.count) { + sample_threads = reads.count; + } + count_samples_args sample_args = { + index, &hlookup, &levlookup, &targets, target_ptrs, target_lens, &reads, &labels, selected_offsets, target_len, k, metric, + indel_window, counts, stats_by_sample, assignments, ambiguous_out, unmatched_out, ambiguous_policy, + assignment_policy, direct_hamming_counts, metal_hamming_counts, metal_target_codes, fused_offset_detection, + target_start, auto_offset, auto_offset_sample, offsets_mode, offset_min_fraction, effective_read_threads, + max_correction_qual, sample_threads, progress_by_sample}; + if (run_count_samples_phase(&sample_args) != 0) goto done; + + if (metal_validate && metal_hamming_counts) { + size_t count_slots = total_slots == 0 ? 1 : total_slots; + size_t sample_slots = reads.count == 0 ? 1 : reads.count; + unsigned long long *metal_counts_snapshot = + (unsigned long long *)calloc(count_slots, sizeof(unsigned long long)); + count_stats *metal_stats_snapshot = (count_stats *)calloc(sample_slots, sizeof(count_stats)); + if (metal_counts_snapshot == NULL || metal_stats_snapshot == NULL) { + free(metal_counts_snapshot); + free(metal_stats_snapshot); + fprintf(stderr, "out of memory\n"); + goto done; + } + memcpy(metal_counts_snapshot, counts, total_slots * sizeof(unsigned long long)); + memcpy(metal_stats_snapshot, stats_by_sample, reads.count * sizeof(count_stats)); + memset(counts, 0, count_slots * sizeof(unsigned long long)); + memset(stats_by_sample, 0, sample_slots * sizeof(count_stats)); + + int cpu_direct = hamming_direct_worker_eligible(hamming_lookup_eligible, assignment_policy, k) && hlookup.ready; + if (!cpu_direct) { + fprintf(stderr, "Metal validation requires a CPU Hamming direct lookup for this workload\n"); + free(metal_counts_snapshot); + free(metal_stats_snapshot); + goto done; + } + count_samples_args cpu_args = sample_args; + cpu_args.direct_hamming_counts = 1; + cpu_args.metal_hamming_counts = 0; + cpu_args.metal_target_codes = NULL; + cpu_args.assignments = NULL; + cpu_args.ambiguous_out = NULL; + cpu_args.unmatched_out = NULL; + cpu_args.progress_by_sample = NULL; + if (run_count_samples_phase(&cpu_args) != 0) { + free(metal_counts_snapshot); + free(metal_stats_snapshot); + goto done; + } + + size_t diff_guides = 0; + long long delta_reads = 0; + char example[256]; + example[0] = '\0'; + if (validate_metal_counts_against_cpu(metal_counts_snapshot, counts, reads.count, targets.count, &diff_guides, + &delta_reads, example, sizeof(example)) != 0) { + metal_validation_status = "failed"; + fprintf(stderr, + "dotmatch: Metal validation failed: %zu guides differ across samples (net read delta %lld); %s\n", + diff_guides, delta_reads, example[0] == '\0' ? "no example" : example); + fprintf(stderr, "dotmatch: rerun with --backend cpu for authoritative counts\n"); + free(metal_counts_snapshot); + free(metal_stats_snapshot); + goto done; + } + metal_validation_status = "passed"; + fprintf(stderr, "dotmatch: Metal validation passed (%zu guides, CPU authority check)\n", targets.count); + memcpy(counts, metal_counts_snapshot, total_slots * sizeof(unsigned long long)); + memcpy(stats_by_sample, metal_stats_snapshot, reads.count * sizeof(count_stats)); + free(metal_counts_snapshot); + free(metal_stats_snapshot); + } else if (metal_validate && backend_mode == COUNT_BACKEND_METAL) { + fprintf(stderr, "dotmatch: --metal-validate requires an active Metal counting backend for this workload\n"); + goto done; + } + + counting_seconds = seconds_now() - phase_start_seconds; + if (show_progress && progress_by_sample != NULL) { + for (size_t sample = 0; sample < reads.count; ++sample) { + count_progress_finish(&progress_by_sample[sample]); + } + } + + out = open_output_file(out_path); + if (out == NULL) { + fprintf(stderr, "failed to open count output\n"); + goto done; + } + if (strcmp(format, "mageck") == 0) { + fprintf(out, "sgRNA\tGene"); + for (size_t sample = 0; sample < reads.count; ++sample) fprintf(out, "\t%s", labels.items[sample]); + fprintf(out, "\n"); + for (size_t t = 0; t < targets.count; ++t) { + fprintf(out, "%s\t%s", targets.records[t].id, targets.records[t].gene); + for (size_t sample = 0; sample < reads.count; ++sample) { + unsigned long long total = 0; + for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * targets.count + t) * 5) + kind]; + fprintf(out, "\t%llu", total); + } + fprintf(out, "\n"); + } + } else { + fprintf(out, "target_id\ttarget_seq\tgene\tambiguous_nearby"); + for (size_t sample = 0; sample < reads.count; ++sample) { + fprintf(out, "\t%s_count_exact\t%s_count_corrected_substitution\t%s_count_corrected_insertion\t%s_count_corrected_deletion\t%s_count_corrected_other\t%s_count_total", + labels.items[sample], labels.items[sample], labels.items[sample], labels.items[sample], labels.items[sample], labels.items[sample]); + } + fprintf(out, "\n"); + for (size_t t = 0; t < targets.count; ++t) { + fprintf(out, "%s\t%s\t%s\t%d", targets.records[t].id, targets.records[t].seq, targets.records[t].gene, (int)ambiguous_nearby[t]); + for (size_t sample = 0; sample < reads.count; ++sample) { + unsigned long long exact = counts[((sample * targets.count + t) * 5) + 0]; + unsigned long long sub = counts[((sample * targets.count + t) * 5) + 1]; + unsigned long long ins = counts[((sample * targets.count + t) * 5) + 2]; + unsigned long long del = counts[((sample * targets.count + t) * 5) + 3]; + unsigned long long other = counts[((sample * targets.count + t) * 5) + 4]; + fprintf(out, "\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu", exact, sub, ins, del, other, exact + sub + ins + del + other); + } + fprintf(out, "\n"); + } + } + + if (target_counts_long_path != NULL) { + FILE *long_out = open_output_file(target_counts_long_path); + if (long_out == NULL) { + fprintf(stderr, "failed to open long target-count output\n"); + goto done; + } + fprintf(long_out, "sample_id\ttarget_id\tgroup\tsequence\texact_count\tk1_sub_count\tk1_ins_count\tk1_del_count\tother_count\ttotal_count\tambiguous_nearby\n"); + for (size_t sample = 0; sample < reads.count; ++sample) { + for (size_t t = 0; t < targets.count; ++t) { + unsigned long long exact = counts[((sample * targets.count + t) * 5) + 0]; + unsigned long long sub = counts[((sample * targets.count + t) * 5) + 1]; + unsigned long long ins = counts[((sample * targets.count + t) * 5) + 2]; + unsigned long long del = counts[((sample * targets.count + t) * 5) + 3]; + unsigned long long other = counts[((sample * targets.count + t) * 5) + 4]; + fprintf(long_out, "%s\t%s\t%s\t%s\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%d\n", + labels.items[sample], targets.records[t].id, targets.records[t].gene, targets.records[t].seq, + exact, sub, ins, del, other, exact + sub + ins + del + other, (int)ambiguous_nearby[t]); + } + } + fclose(long_out); + } + + if (sample_qc_path != NULL) { + FILE *qc = open_output_file(sample_qc_path); + if (qc == NULL) { + fprintf(stderr, "failed to open sample QC output\n"); + goto done; + } + fprintf(qc, "sample_id\tfastq\ttotal_reads\tvalid_extracted_reads\tassigned_reads\texact_reads\tk1_rescued_reads\tk1_sub_reads\tk1_ins_reads\tk1_del_reads\tambiguous_reads\tno_match_reads\tinvalid_reads\tassignment_rate\texact_rate\trescue_rate\tambiguous_rate\tno_match_rate\ttargets_observed\tzero_count_targets\tgini_index\ttop_1pct_read_fraction\tcandidates_verified\n"); + for (size_t sample = 0; sample < reads.count; ++sample) { + unsigned long long *target_totals = (unsigned long long *)calloc(targets.count == 0 ? 1 : targets.count, sizeof(unsigned long long)); + if (target_totals == NULL) { + fclose(qc); + fprintf(stderr, "out of memory\n"); + goto done; + } + unsigned long long sub = 0; + unsigned long long ins = 0; + unsigned long long del = 0; + unsigned long long observed_targets = 0; + for (size_t t = 0; t < targets.count; ++t) { + unsigned long long exact = counts[((sample * targets.count + t) * 5) + 0]; + sub += counts[((sample * targets.count + t) * 5) + 1]; + ins += counts[((sample * targets.count + t) * 5) + 2]; + del += counts[((sample * targets.count + t) * 5) + 3]; + target_totals[t] = exact + counts[((sample * targets.count + t) * 5) + 1] + + counts[((sample * targets.count + t) * 5) + 2] + + counts[((sample * targets.count + t) * 5) + 3] + + counts[((sample * targets.count + t) * 5) + 4]; + if (target_totals[t] != 0) ++observed_targets; + } + count_stats *s = &stats_by_sample[sample]; + unsigned long long valid = s->total >= s->invalid ? s->total - s->invalid : 0; + double valid_denom = valid == 0 ? 1.0 : (double)valid; + fprintf(qc, "%s\t%s\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%llu\t%.8f\t%.8f\t%.8f\t%.8f\t%.8f\t%llu\t%llu\t%.8f\t%.8f\t%llu\n", + labels.items[sample], reads.items[sample], s->total, valid, s->unique, s->exact, s->corrected, + sub, ins, del, s->ambiguous, s->unmatched, s->invalid, + (double)s->unique / valid_denom, (double)s->exact / valid_denom, (double)s->corrected / valid_denom, + (double)s->ambiguous / valid_denom, (double)s->unmatched / valid_denom, + observed_targets, (unsigned long long)(targets.count - observed_targets), + gini_from_counts(target_totals, targets.count), + top_fraction_from_counts(target_totals, targets.count, 0.01), + s->candidates_verified); + free(target_totals); + } + fclose(qc); + } + + if (summary_path != NULL) { + FILE *summary = open_output_file(summary_path); + if (summary == NULL) { + fprintf(stderr, "failed to open summary output\n"); + goto done; + } + double total_before_summary_seconds = seconds_now() - run_start_seconds; + fprintf(summary, + "{\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"max_correction_qual\": ", + k, metric_name(metric), ambiguity_policy_name(assignment_policy), qdaln_alphabet_policy()); + if (max_correction_qual >= 0) { + fprintf(summary, "%d", max_correction_qual); + } else { + fprintf(summary, "null"); + } + fprintf(summary, + ",\n \"indel_window\": %zu,\n \"target_start\": %zu,\n \"auto_offset\": %zu,\n \"offset_mode\": \"%s\",\n \"offset_min_fraction\": %.8f,\n \"offset_detection_strategy\": \"%s\",\n \"backend_requested\": \"%s\",\n \"backend_effective\": \"%s\",\n \"metal_device\": ", + indel_window, target_start, auto_offset, offset_mode_name(offsets_mode), offset_min_fraction, + offset_detection_strategy, count_backend_mode_name(backend_mode), backend_effective); + if (metal_hamming_counts && qdmetal_device_name() != NULL) { + fprintf(summary, "\"%s\"", qdmetal_device_name()); + } else { + fprintf(summary, "null"); + } + fprintf(summary, ",\n \"metal_validation\": "); + if (metal_validation_status != NULL) { + fprintf(summary, "\"%s\"", metal_validation_status); + } else { + fprintf(summary, "null"); + } + fprintf(summary, + ",\n \"count_engine\": \"%s\",\n \"hamming_index\": \"%s\",\n \"target_length\": %zu,\n \"n_targets\": %zu,\n \"read_threads\": %zu,\n \"phase_seconds\": {\"target_index\": %.6f, \"offset_detection\": %.6f, \"hamming_precompute\": %.6f, \"counting\": %.6f, \"total_before_summary\": %.6f},\n \"samples\": [\n", + count_engine, hamming_lookup_kind(&hlookup), target_len, targets.count, effective_read_threads, + target_index_seconds, offset_detection_seconds, hamming_precompute_seconds, counting_seconds, + total_before_summary_seconds); + for (size_t sample = 0; sample < reads.count; ++sample) { + count_stats *s = &stats_by_sample[sample]; + unsigned long long covered = 0; + unsigned long long top_count = 0; + size_t top_target = 0; + for (size_t t = 0; t < targets.count; ++t) { + unsigned long long total = 0; + for (size_t kind = 0; kind < 5; ++kind) total += counts[((sample * targets.count + t) * 5) + kind]; + if (total != 0) ++covered; + if (total > top_count) { + top_count = total; + top_target = t; + } + } + double rescued_percent = s->total == 0 ? 0.0 : 100.0 * (double)s->corrected / (double)s->total; + double ambiguous_percent = s->total == 0 ? 0.0 : 100.0 * (double)s->ambiguous / (double)s->total; + double unmatched_percent = s->total == 0 ? 0.0 : 100.0 * (double)s->unmatched / (double)s->total; + fprintf(summary, + " {\"sample\": \"%s\", \"selected_target_start\": %zu, \"selected_target_starts\": [", + labels.items[sample], first_selected_offset(&selected_offsets[sample], target_start)); + for (size_t oi = 0; oi < selected_offsets[sample].count; ++oi) { + if (oi != 0) fprintf(summary, ", "); + fprintf(summary, "%zu", selected_offsets[sample].items[oi]); + } + fprintf(summary, + "], \"total_reads\": %llu, \"assigned_unique\": %llu, \"assigned_exact\": %llu, \"assigned_corrected\": %llu, \"k1_rescued_reads\": %llu, \"percent_rescued_by_k1\": %.6f, \"ambiguous\": %llu, \"percent_ambiguous\": %.6f, \"unmatched\": %llu, \"percent_unmatched\": %.6f, \"invalid\": %llu, \"library_covered_targets\": %llu, \"library_coverage_fraction\": %.6f, \"top_target_id\": \"%s\", \"top_target_count\": %llu, \"candidates_considered\": %llu, \"candidates_verified\": %llu}%s\n", + s->total, s->unique, s->exact, s->corrected, + s->corrected, rescued_percent, s->ambiguous, ambiguous_percent, s->unmatched, unmatched_percent, + s->invalid, + covered, targets.count == 0 ? 0.0 : (double)covered / (double)targets.count, + targets.count == 0 ? "" : targets.records[top_target].id, top_count, s->candidates_considered, + s->candidates_verified, sample + 1 == reads.count ? "" : ","); + } + fprintf(summary, " ]\n}\n"); + fclose(summary); + } + if (report_path != NULL) { + if (write_count_html_report(report_path, &targets, &reads, &labels, counts, stats_by_sample, selected_offsets, + k, metric, assignment_policy, target_len, report_audit_dir, + report_unmatched_path) != 0) { + fprintf(stderr, "failed to write HTML report\n"); + goto done; + } + } + { + sample_qc_metrics *qc_metrics = (sample_qc_metrics *)calloc(reads.count, sizeof(sample_qc_metrics)); + if (qc_metrics != NULL) { + int metrics_ok = 1; + for (size_t sample = 0; sample < reads.count; ++sample) { + if (compute_sample_qc_metrics(&targets, counts, sample, &stats_by_sample[sample], + &qc_metrics[sample]) != 0) { + metrics_ok = 0; + break; + } + } + if (metrics_ok) { + int enough_reads = 0; + for (size_t sample = 0; sample < reads.count; ++sample) { + if (stats_by_sample[sample].total >= 1000) enough_reads = 1; + } + if (enough_reads || crispr_mode) emit_sample_qc_review_warnings(&labels, qc_metrics, reads.count); + } + free(qc_metrics); + } + } + rc = 0; + +done: + if (progress_by_sample != NULL) { + for (size_t sample = 0; sample < reads.count; ++sample) count_progress_fini(&progress_by_sample[sample]); + free(progress_by_sample); + } + if (out != NULL) fclose(out); + if (assignments != NULL) fclose(assignments); + if (ambiguous_out != NULL) fclose(ambiguous_out); + if (unmatched_out != NULL) fclose(unmatched_out); + qdaln_index_free(index); + free_hamming_lookup(&hlookup); + free_hamming_lookup(&offset_lookup); + free_levenshtein1_lookup(&levlookup); + free(metal_target_codes); + free(target_ptrs); + free(target_lens); + free(ambiguous_nearby); + free(counts); + free(stats_by_sample); + if (selected_offsets != NULL) { + for (size_t sample = 0; sample < reads.count; ++sample) free_offset_list(&selected_offsets[sample]); + } + free(selected_offsets); + free_table(&targets); + free_string_list(&reads); + free_string_list(&labels); + return rc; + +fail_args: + free_string_list(&reads); + free_string_list(&labels); + return 2; +} + +static int string_list_contains_exact(const string_list *list, const char *s) { + if (list == NULL || s == NULL) return 0; + for (size_t i = 0; i < list->count; ++i) { + if (strcmp(list->items[i], s) == 0) return 1; + } + return 0; +} + +static int read_first_column_values(const char *path, string_list *values) { + if (path == NULL) return 0; + fastq_reader reader = {0}; + if (fastq_reader_open(&reader, path) != 0) return -1; + char buf[8192]; + size_t len = 0; + int rc = 0; + while ((rc = fastq_getline_len(&reader, buf, sizeof(buf), &len)) > 0) { + (void)len; + trim_line(buf); + if (buf[0] == '\0' || buf[0] == '#') continue; + char *tab = strchr(buf, '\t'); + if (tab != NULL) *tab = '\0'; + if (push_string(values, buf) != 0) { + fastq_reader_close(&reader); + return -1; + } + } + fastq_reader_close(&reader); + return rc < 0 ? -1 : 0; +} + +static int guide_counter_push_sample_name(string_list *labels, const char *path, size_t idx) { + const char *base = path_basename(path); + char fallback[32]; + if (base == NULL || base[0] == '\0') { + int n = snprintf(fallback, sizeof(fallback), "s%zu", idx + 1); + if (n < 0 || (size_t)n >= sizeof(fallback)) return -1; + return push_string(labels, fallback); + } + char *name = xstrndup(base, strlen(base)); + if (name == NULL) return -1; + if (ends_with(name, ".gz")) name[strlen(name) - 3] = '\0'; + if (ends_with(name, ".fastq")) { + name[strlen(name) - 6] = '\0'; + } else if (ends_with(name, ".fq")) { + name[strlen(name) - 3] = '\0'; + } + int rc = push_string(labels, name); + free(name); + return rc; +} + +static const char *guide_counter_type_for_target(const seq_record *target, const string_list *essential_genes, + const string_list *nonessential_genes, + const string_list *control_guides, + regex_t *control_re) { + if (string_list_contains_exact(essential_genes, target->gene)) return "Essential"; + if (string_list_contains_exact(nonessential_genes, target->gene)) return "Nonessential"; + if (string_list_contains_exact(control_guides, target->id)) return "Control"; + if (control_re != NULL && + (regexec(control_re, target->id, 0, NULL, 0) == 0 || + regexec(control_re, target->gene, 0, NULL, 0) == 0)) { + return "Control"; + } + return "Other"; +} + +static int parse_ull_value(const char *s, unsigned long long *out) { + if (s == NULL || s[0] == '-' || s[0] == '\0') return -1; + char *end = NULL; + errno = 0; + unsigned long long v = strtoull(s, &end, 10); + if (errno == ERANGE || end == s || *end != '\0') return -1; + *out = v; + return 0; +} + +static double round_positive_dp(double value, int places) { + double factor = 1.0; + for (int i = 0; i < places; ++i) factor *= 10.0; + unsigned long long scaled = (unsigned long long)(value * factor + 0.5); + return (double)scaled / factor; +} + +static int guide_counter_write_outputs(const char *output_prefix, const char *tmp_counts_path, + const char *tmp_qc_path, const seq_table *targets, + const string_list *reads, const string_list *labels, + const string_list *essential_genes, + const string_list *nonessential_genes, + const string_list *control_guides, regex_t *control_re) { + char counts_path[4096]; + char extended_path[4096]; + char stats_path[4096]; + int n = snprintf(counts_path, sizeof(counts_path), "%s.counts.txt", output_prefix); + if (n < 0 || (size_t)n >= sizeof(counts_path)) return -1; + n = snprintf(extended_path, sizeof(extended_path), "%s.extended-counts.txt", output_prefix); + if (n < 0 || (size_t)n >= sizeof(extended_path)) return -1; + n = snprintf(stats_path, sizeof(stats_path), "%s.stats.txt", output_prefix); + if (n < 0 || (size_t)n >= sizeof(stats_path)) return -1; + + size_t matrix_slots = 0; + if (checked_mul_size(targets->count, labels->count, &matrix_slots) != 0) return -1; + unsigned long long *matrix = (unsigned long long *)calloc(alloc_count_or_one(matrix_slots), + sizeof(unsigned long long)); + const char **types = (const char **)calloc(alloc_count_or_one(targets->count), sizeof(const char *)); + if (matrix == NULL || types == NULL) { + free(matrix); + free(types); + return -1; + } + for (size_t t = 0; t < targets->count; ++t) { + types[t] = guide_counter_type_for_target(&targets->records[t], essential_genes, nonessential_genes, + control_guides, control_re); + } + + FILE *in = fopen(tmp_counts_path, "r"); + FILE *counts = open_output_file(counts_path); + FILE *extended = open_output_file(extended_path); + if (in == NULL || counts == NULL || extended == NULL) { + if (in != NULL) fclose(in); + if (counts != NULL) fclose(counts); + if (extended != NULL) fclose(extended); + free(matrix); + free(types); + return -1; + } + + fprintf(counts, "guide\tgene"); + fprintf(extended, "guide\tgene\tguide_type"); + for (size_t sample = 0; sample < labels->count; ++sample) { + fprintf(counts, "\t%s", labels->items[sample]); + fprintf(extended, "\t%s", labels->items[sample]); + } + fprintf(counts, "\n"); + fprintf(extended, "\n"); + + char buf[65536]; + size_t row = 0; + int first = 1; + while (fgets(buf, sizeof(buf), in) != NULL) { + trim_line(buf); + if (first) { + first = 0; + continue; + } + char *fields[1024]; + size_t nf = split_fields(buf, '\t', fields, sizeof(fields) / sizeof(fields[0])); + if (nf < 2 + labels->count || row >= targets->count) { + fclose(in); + fclose(counts); + fclose(extended); + free(matrix); + free(types); + return -1; + } + fprintf(counts, "%s\t%s", targets->records[row].id, targets->records[row].gene); + fprintf(extended, "%s\t%s\t%s", targets->records[row].id, targets->records[row].gene, types[row]); + for (size_t sample = 0; sample < labels->count; ++sample) { + unsigned long long value = 0; + if (parse_ull_value(fields[2 + sample], &value) != 0) { + fclose(in); + fclose(counts); + fclose(extended); + free(matrix); + free(types); + return -1; + } + matrix[row * labels->count + sample] = value; + fprintf(counts, "\t%llu", value); + fprintf(extended, "\t%llu", value); + } + fprintf(counts, "\n"); + fprintf(extended, "\n"); + ++row; + } + int matrix_ok = !ferror(in) && row == targets->count; + fclose(in); + fclose(counts); + fclose(extended); + if (!matrix_ok) { + free(matrix); + free(types); + return -1; + } + + unsigned long long *total_reads = (unsigned long long *)calloc(alloc_count_or_one(labels->count), + sizeof(unsigned long long)); + if (total_reads == NULL) { + free(matrix); + free(types); + return -1; + } + FILE *qc = fopen(tmp_qc_path, "r"); + if (qc == NULL) { + free(total_reads); + free(matrix); + free(types); + return -1; + } + int total_reads_col = -1; + size_t qc_row = 0; + first = 1; + while (fgets(buf, sizeof(buf), qc) != NULL) { + trim_line(buf); + char *fields[64]; + size_t nf = split_fields(buf, '\t', fields, sizeof(fields) / sizeof(fields[0])); + if (first) { + total_reads_col = find_column(fields, nf, "total_reads", NULL, NULL); + first = 0; + continue; + } + if (total_reads_col < 0 || (size_t)total_reads_col >= nf || qc_row >= labels->count || + parse_ull_value(fields[total_reads_col], &total_reads[qc_row]) != 0) { + fclose(qc); + free(total_reads); + free(matrix); + free(types); + return -1; + } + ++qc_row; + } + int qc_ok = !ferror(qc) && qc_row == labels->count; + fclose(qc); + if (!qc_ok) { + free(total_reads); + free(matrix); + free(types); + return -1; + } + + FILE *stats = open_output_file(stats_path); + if (stats == NULL) { + free(total_reads); + free(matrix); + free(types); + return -1; + } + fprintf(stats, "file\tlabel\ttotal_guides\ttotal_reads\tmapped_reads\tfrac_mapped\tmean_reads_per_guide\tmean_reads_essential\tmean_reads_nonessential\tmean_reads_control\tmean_reads_other\tzero_read_guides\n"); + for (size_t sample = 0; sample < labels->count; ++sample) { + unsigned long long mapped = 0; + unsigned long long zero = 0; + double essential_sum = 0.0; + double nonessential_sum = 0.0; + double control_sum = 0.0; + double other_sum = 0.0; + size_t essential_count = 0; + size_t nonessential_count = 0; + size_t control_count = 0; + size_t other_count = 0; + for (size_t t = 0; t < targets->count; ++t) { + unsigned long long value = matrix[t * labels->count + sample]; + mapped += value; + if (value == 0) ++zero; + if (strcmp(types[t], "Essential") == 0) { + essential_sum += (double)value; + ++essential_count; + } else if (strcmp(types[t], "Nonessential") == 0) { + nonessential_sum += (double)value; + ++nonessential_count; + } else if (strcmp(types[t], "Control") == 0) { + control_sum += (double)value; + ++control_count; + } else { + other_sum += (double)value; + ++other_count; + } + } + double total = (double)total_reads[sample]; + double frac = total == 0.0 ? 0.0 : round_positive_dp((double)mapped / total, 4); + double mean_all = targets->count == 0 ? 0.0 : round_positive_dp((double)mapped / (double)targets->count, 2); + double mean_essential = essential_count == 0 ? 0.0 : round_positive_dp(essential_sum / (double)essential_count, 2); + double mean_nonessential = nonessential_count == 0 ? 0.0 : round_positive_dp(nonessential_sum / (double)nonessential_count, 2); + double mean_control = control_count == 0 ? 0.0 : round_positive_dp(control_sum / (double)control_count, 2); + double mean_other = other_count == 0 ? 0.0 : round_positive_dp(other_sum / (double)other_count, 2); + fprintf(stats, "%s\t%s\t%zu\t%llu\t%llu\t%.4f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%llu\n", + reads->items[sample], labels->items[sample], targets->count, total_reads[sample], mapped, + frac, mean_all, mean_essential, mean_nonessential, mean_control, mean_other, zero); + } + fclose(stats); + free(total_reads); + free(matrix); + free(types); + return 0; +} + +static int push_count_arg(string_list *args, const char *s) { + return push_string(args, s); +} + +static int run_guide_counter_compatible(const char *argv0, int argc, char **argv) { + int start = 2; + if (strcmp(argv[1], "guide-counter") == 0) { + if (argc < 3 || strcmp(argv[2], "count") != 0) { + usage(argv0); + return 2; + } + start = 3; + } + + const char *library_path = NULL; + const char *output_prefix = NULL; + const char *essential_path = NULL; + const char *nonessential_path = NULL; + const char *control_guides_path = NULL; + const char *control_pattern = NULL; + size_t offset_sample_size = 100000; + double offset_min_fraction = 0.0025; + int exact_match = 0; + string_list reads = {0}; + string_list labels = {0}; + + int i = start; + while (i < argc) { + const char *arg = argv[i++]; + if ((strcmp(arg, "--input") == 0 || strcmp(arg, "-i") == 0) && i < argc) { + while (i < argc && argv[i][0] != '-') { + if (push_string(&reads, argv[i++]) != 0) goto oom; + } + } else if ((strcmp(arg, "--samples") == 0 || strcmp(arg, "-s") == 0) && i < argc) { + while (i < argc && argv[i][0] != '-') { + if (push_string(&labels, argv[i++]) != 0) goto oom; + } + } else if ((strcmp(arg, "--library") == 0 || strcmp(arg, "-l") == 0) && i < argc) { + library_path = argv[i++]; + } else if ((strcmp(arg, "--output") == 0 || strcmp(arg, "-o") == 0) && i < argc) { + output_prefix = argv[i++]; + } else if ((strcmp(arg, "--essential-genes") == 0 || strcmp(arg, "-e") == 0) && i < argc) { + essential_path = argv[i++]; + } else if ((strcmp(arg, "--nonessential-genes") == 0 || strcmp(arg, "-n") == 0) && i < argc) { + nonessential_path = argv[i++]; + } else if ((strcmp(arg, "--control-guides") == 0 || strcmp(arg, "-c") == 0) && i < argc) { + control_guides_path = argv[i++]; + } else if ((strcmp(arg, "--control-pattern") == 0 || strcmp(arg, "-C") == 0) && i < argc) { + control_pattern = argv[i++]; + } else if ((strcmp(arg, "--offset-sample-size") == 0 || strcmp(arg, "-N") == 0) && i < argc) { + if (parse_size_value(argv[i++], &offset_sample_size) != 0 || offset_sample_size == 0) goto bad_args; + } else if ((strcmp(arg, "--offset-min-fraction") == 0 || strcmp(arg, "-f") == 0) && i < argc) { + if (parse_double_value(argv[i++], &offset_min_fraction) != 0 || + offset_min_fraction < 0.0 || offset_min_fraction > 1.0) { + goto bad_args; + } + } else if (strcmp(arg, "--exact-match") == 0 || strcmp(arg, "-x") == 0) { + exact_match = 1; + } else if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) { + usage(argv0); + free_string_list(&reads); + free_string_list(&labels); + return 0; + } else { + goto bad_args; + } + } + + if (library_path == NULL || output_prefix == NULL || reads.count == 0) goto bad_args; + if (labels.count == 0) { + for (size_t sample = 0; sample < reads.count; ++sample) { + if (guide_counter_push_sample_name(&labels, reads.items[sample], sample) != 0) goto oom; + } + } + if (labels.count != reads.count) { + fprintf(stderr, "--samples count must match --input count\n"); + free_string_list(&reads); + free_string_list(&labels); + return 2; + } + if (validate_unique_sample_labels(&labels, "--samples") != 0) { + free_string_list(&reads); + free_string_list(&labels); + return 2; + } + + seq_table targets = {0}; + string_list essential_genes = {0}; + string_list nonessential_genes = {0}; + string_list control_guides = {0}; + regex_t control_re; + int have_control_re = 0; + string_list count_args = {0}; + int rc = 1; + char target_len_s[32]; + char k_s[8]; + char auto_offset_s[16]; + char offset_sample_s[32]; + char offset_min_s[64]; + char label_csv[8192]; + char tmp_counts_path[4096] = ""; + char tmp_qc_path[4096] = ""; + + if (read_target_table(library_path, &targets) != 0 || targets.count == 0) { + fprintf(stderr, "failed to read guide library\n"); + goto done; + } + int guide_id_check = validate_unique_seq_ids(&targets, "guide"); + if (guide_id_check != 0) { + if (guide_id_check == -1) fprintf(stderr, "out of memory\n"); + goto done; + } + size_t guide_len = targets.records[0].len; + for (size_t t = 1; t < targets.count; ++t) { + if (targets.records[t].len != guide_len) { + fprintf(stderr, "GuideCounter compatibility requires one guide length\n"); + goto done; + } + } + if (read_first_column_values(essential_path, &essential_genes) != 0 || + read_first_column_values(nonessential_path, &nonessential_genes) != 0 || + read_first_column_values(control_guides_path, &control_guides) != 0) { + fprintf(stderr, "failed to read guide annotation files\n"); + goto done; + } + if (control_pattern != NULL) { + if (regcomp(&control_re, control_pattern, REG_EXTENDED | REG_ICASE | REG_NOSUB) != 0) { + fprintf(stderr, "failed to compile --control-pattern\n"); + goto done; + } + have_control_re = 1; + } + + label_csv[0] = '\0'; + for (size_t sample = 0; sample < labels.count; ++sample) { + size_t used = strlen(label_csv); + int n = snprintf(label_csv + used, sizeof(label_csv) - used, "%s%s", + sample == 0 ? "" : ",", labels.items[sample]); + if (n < 0 || (size_t)n >= sizeof(label_csv) - used) { + fprintf(stderr, "too many sample labels for GuideCounter compatibility wrapper\n"); + goto done; + } + } + int n = snprintf(tmp_counts_path, sizeof(tmp_counts_path), "%s.dotmatch-counts.tmp", output_prefix); + if (n < 0 || (size_t)n >= sizeof(tmp_counts_path)) goto done; + n = snprintf(tmp_qc_path, sizeof(tmp_qc_path), "%s.dotmatch-qc.tmp", output_prefix); + if (n < 0 || (size_t)n >= sizeof(tmp_qc_path)) goto done; + snprintf(target_len_s, sizeof(target_len_s), "%zu", guide_len); + snprintf(k_s, sizeof(k_s), "%d", exact_match ? 0 : 1); + snprintf(auto_offset_s, sizeof(auto_offset_s), "%d", 499); + snprintf(offset_sample_s, sizeof(offset_sample_s), "%zu", offset_sample_size); + snprintf(offset_min_s, sizeof(offset_min_s), "%.8g", offset_min_fraction); + + if (push_count_arg(&count_args, argv0) != 0 || + push_count_arg(&count_args, "count") != 0 || + push_count_arg(&count_args, "--targets") != 0 || + push_count_arg(&count_args, library_path) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t sample = 0; sample < reads.count; ++sample) { + if (push_count_arg(&count_args, "--reads") != 0 || + push_count_arg(&count_args, reads.items[sample]) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + } + const char *fixed_args[] = { + "--sample-label", label_csv, + "--target-start", "0", + "--target-length", target_len_s, + "--k", k_s, + "--metric", "hamming", + "--ambiguity-policy", "best", + "--format", "mageck", + "--auto-offset", auto_offset_s, + "--auto-offset-sample", offset_sample_s, + "--offset-mode", "multi", + "--offset-min-fraction", offset_min_s, + "--out", tmp_counts_path, + "--sample-qc", tmp_qc_path + }; + for (size_t ai = 0; ai < sizeof(fixed_args) / sizeof(fixed_args[0]); ++ai) { + if (push_count_arg(&count_args, fixed_args[ai]) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + } + + rc = run_count(argv0, (int)count_args.count, count_args.items); + if (rc != 0) goto done; + if (guide_counter_write_outputs(output_prefix, tmp_counts_path, tmp_qc_path, &targets, &reads, &labels, + &essential_genes, &nonessential_genes, &control_guides, + have_control_re ? &control_re : NULL) != 0) { + fprintf(stderr, "failed to write GuideCounter-compatible outputs\n"); + rc = 1; + goto done; + } + rc = 0; + +done: + if (have_control_re) regfree(&control_re); + unlink(tmp_counts_path); + unlink(tmp_qc_path); + free_string_list(&count_args); + free_string_list(&essential_genes); + free_string_list(&nonessential_genes); + free_string_list(&control_guides); + free_table(&targets); + free_string_list(&reads); + free_string_list(&labels); + return rc; + +oom: + fprintf(stderr, "out of memory\n"); + free_string_list(&reads); + free_string_list(&labels); + return 1; + +bad_args: + usage(argv0); + free_string_list(&reads); + free_string_list(&labels); + return 2; +} + +static int run_fastq_assign(const char *argv0, int argc, char **argv) { + const char *barcodes_path = NULL; + const char *reads_path = NULL; + const char *out_path = NULL; + ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; + size_t barcode_start = 0; + size_t barcode_len = 0; + int k = -1; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if (strcmp(arg, "--barcodes") == 0 && i < argc) { + barcodes_path = argv[i++]; + } else if (strcmp(arg, "--reads") == 0 && i < argc) { + reads_path = argv[i++]; + } else if (strcmp(arg, "--barcode-start") == 0 && i < argc) { + if (parse_size_value(argv[i++], &barcode_start) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--barcode-length") == 0 && i < argc) { + if (parse_size_value(argv[i++], &barcode_len) != 0 || barcode_len == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || (k != 0 && k != 1)) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "best") == 0) { + assignment_policy = AMBIGUITY_POLICY_BEST; + } else if (strcmp(value, "radius") == 0) { + assignment_policy = AMBIGUITY_POLICY_RADIUS; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--out") == 0 && i < argc) { + out_path = argv[i++]; + } else { + usage(argv0); + return 2; + } + } + + if (barcodes_path == NULL || reads_path == NULL || out_path == NULL || barcode_len == 0 || k < 0) { + usage(argv0); + return 2; + } + + seq_table targets = {0}; + fastq_reader reader = {0}; + FILE *out = NULL; + qdaln_index *index = NULL; + int rc = 1; + + if (read_table(barcodes_path, &targets) != 0) { + fprintf(stderr, "failed to read barcode file\n"); + goto done; + } + int barcode_id_check = validate_unique_seq_ids(&targets, "barcode"); + if (barcode_id_check != 0) { + if (barcode_id_check == -1) fprintf(stderr, "out of memory\n"); + goto done; + } + + const char **target_ptrs = (const char **)malloc(targets.count * sizeof(char *)); + size_t *target_lens = (size_t *)malloc(targets.count * sizeof(size_t)); + if (targets.count != 0 && (target_ptrs == NULL || target_lens == NULL)) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t i = 0; i < targets.count; ++i) { + target_ptrs[i] = targets.records[i].seq; + target_lens[i] = targets.records[i].len; + } + index = qdaln_index_build(target_ptrs, target_lens, targets.count); + free(target_ptrs); + free(target_lens); + if (index == NULL) { + fprintf(stderr, "failed to build barcode index\n"); + goto done; + } + + if (fastq_reader_open(&reader, reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + goto done; + } + out = open_output_file(out_path); + if (out == NULL) { + fprintf(stderr, "failed to open output file\n"); + goto done; + } + + fprintf(out, "read_id\tobserved_barcode\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\n"); + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + char read_id[8192]; + char observed[8192]; + int got = 0; + size_t seq_len = 0; + while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { + fastq_read_id(header, read_id, sizeof(read_id)); + qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + observed[0] = '\0'; + if (barcode_start <= seq_len && barcode_len <= seq_len - barcode_start && barcode_len < sizeof(observed)) { + memcpy(observed, seq + barcode_start, barcode_len); + observed[barcode_len] = '\0'; + const char *read_ptr = observed; + size_t read_len = barcode_len; + qdaln_index_stats stats; + if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &result, &stats) != 0) { + fprintf(stderr, "FASTQ assignment failed\n"); + goto done; + } + apply_ambiguity_policy(&result, assignment_policy); + } + print_fastq_row(out, &targets, read_id, observed, result); + } + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + goto done; + } + rc = 0; + +done: + if (out != NULL) fclose(out); + fastq_reader_close(&reader); + qdaln_index_free(index); + free_table(&targets); + return rc; +} + +static void sanitize_filename(const char *in, char *out, size_t out_cap) { + size_t j = 0; + if (out_cap == 0) return; + for (size_t i = 0; in[i] != '\0' && j + 1 < out_cap; ++i) { + char c = in[i]; + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '_' || c == '-' || c == '.') { + out[j++] = c; + } else { + out[j++] = '_'; + } + } + if (j == 0 && out_cap > 1) out[j++] = '_'; + out[j] = '\0'; +} + +typedef struct sanitized_name_entry { + char *name; + const char *id; +} sanitized_name_entry; + +static int compare_sanitized_name_entry(const void *a, const void *b) { + const sanitized_name_entry *ea = (const sanitized_name_entry *)a; + const sanitized_name_entry *eb = (const sanitized_name_entry *)b; + return strcmp(ea->name, eb->name); +} + +static int validate_unique_sanitized_filenames(const seq_table *targets) { + sanitized_name_entry *entries = (sanitized_name_entry *)calloc(targets->count == 0 ? 1 : targets->count, + sizeof(sanitized_name_entry)); + if (entries == NULL) return -1; + + int rc = 0; + for (size_t i = 0; i < targets->count; ++i) { + char safe_id[512]; + sanitize_filename(targets->records[i].id, safe_id, sizeof(safe_id)); + entries[i].name = xstrndup(safe_id, strlen(safe_id)); + entries[i].id = targets->records[i].id; + if (entries[i].name == NULL) { + rc = -1; + goto done; + } + } + + qsort(entries, targets->count, sizeof(entries[0]), compare_sanitized_name_entry); + for (size_t i = 1; i < targets->count; ++i) { + if (strcmp(entries[i - 1].name, entries[i].name) == 0) { + fprintf(stderr, + "barcode IDs produce the same output filename after sanitization: \"%s\" and \"%s\" -> %s.fastq\n", + entries[i - 1].id, entries[i].id, entries[i].name); + rc = -2; + goto done; + } + } + +done: + for (size_t i = 0; i < targets->count; ++i) free(entries[i].name); + free(entries); + return rc; +} + +static int ensure_dir(const char *path) { + if (mkdir(path, 0777) == 0) return 0; + if (errno == EEXIST) { + struct stat st; + return stat(path, &st) == 0 && S_ISDIR(st.st_mode) ? 0 : -1; + } + return -1; +} + +static int path_join(char *out, size_t out_cap, const char *dir, const char *name) { + int n = snprintf(out, out_cap, "%s/%s", dir, name); + return n < 0 || (size_t)n >= out_cap ? -1 : 0; +} + +static size_t uf_find(size_t *parent, size_t x) { + while (parent[x] != x) { + parent[x] = parent[parent[x]]; + x = parent[x]; + } + return x; +} + +static void uf_union(size_t *parent, size_t a, size_t b) { + size_t ra = uf_find(parent, a); + size_t rb = uf_find(parent, b); + if (ra == rb) return; + if (ra < rb) parent[rb] = ra; + else parent[ra] = rb; +} + +static int string_list_contains(const string_list *list, const char *s) { + for (size_t i = 0; i < list->count; ++i) { + if (strcmp(list->items[i], s) == 0) return 1; + } + return 0; +} + +static int push_unique_string(string_list *list, const char *s) { + if (string_list_contains(list, s)) return 0; + return push_string(list, s); +} + +static int add_k1_variants_for_target(string_list *variants, const char *seq, size_t len) { + static const char dna[] = "ACGT"; + if (push_unique_string(variants, seq) != 0) return -1; + char buf[8192]; + if (len + 2 > sizeof(buf)) return -1; + + for (size_t pos = 0; pos < len; ++pos) { + for (size_t bi = 0; bi < 4; ++bi) { + if (seq[pos] == dna[bi]) continue; + memcpy(buf, seq, len); + buf[pos] = dna[bi]; + buf[len] = '\0'; + if (push_unique_string(variants, buf) != 0) return -1; + } + } + + if (len > 0) { + for (size_t pos = 0; pos < len; ++pos) { + memcpy(buf, seq, pos); + memcpy(buf + pos, seq + pos + 1, len - pos - 1); + buf[len - 1] = '\0'; + if (push_unique_string(variants, buf) != 0) return -1; + } + } + + for (size_t pos = 0; pos <= len; ++pos) { + for (size_t bi = 0; bi < 4; ++bi) { + memcpy(buf, seq, pos); + buf[pos] = dna[bi]; + memcpy(buf + pos + 1, seq + pos, len - pos); + buf[len + 1] = '\0'; + if (push_unique_string(variants, buf) != 0) return -1; + } + } + return 0; +} + +typedef struct variant_record { + char *key; + size_t target; +} variant_record; + +typedef struct variant_record_list { + variant_record *items; + size_t count; + size_t cap; +} variant_record_list; + +static void free_variant_record_list(variant_record_list *list) { + for (size_t i = 0; i < list->count; ++i) free(list->items[i].key); + free(list->items); + list->items = NULL; + list->count = 0; + list->cap = 0; +} + +static int push_variant_record(variant_record_list *list, const char *key, size_t target) { + if (list->count == list->cap) { + size_t next_cap = list->cap == 0 ? 1024 : list->cap * 2; + variant_record *next = (variant_record *)realloc(list->items, next_cap * sizeof(variant_record)); + if (next == NULL) return -1; + list->items = next; + list->cap = next_cap; + } + list->items[list->count].key = xstrndup(key, strlen(key)); + if (list->items[list->count].key == NULL) return -1; + list->items[list->count].target = target; + ++list->count; + return 0; +} + +static int cmp_variant_record(const void *a, const void *b) { + const variant_record *aa = (const variant_record *)a; + const variant_record *bb = (const variant_record *)b; + int c = strcmp(aa->key, bb->key); + if (c != 0) return c; + return aa->target > bb->target ? 1 : (aa->target < bb->target ? -1 : 0); +} + +typedef struct pair_record { + size_t a; + size_t b; +} pair_record; + +typedef struct pair_record_list { + pair_record *items; + size_t count; + size_t cap; +} pair_record_list; + +static void free_pair_record_list(pair_record_list *list) { + free(list->items); + list->items = NULL; + list->count = 0; + list->cap = 0; +} + +static int cmp_pair_record(const void *a, const void *b) { + const pair_record *aa = (const pair_record *)a; + const pair_record *bb = (const pair_record *)b; + if (aa->a != bb->a) return aa->a > bb->a ? 1 : -1; + return aa->b > bb->b ? 1 : (aa->b < bb->b ? -1 : 0); +} + +static int push_pair_record(pair_record_list *list, size_t a, size_t b) { + if (a > b) { + size_t tmp = a; + a = b; + b = tmp; + } + if (list->count == list->cap) { + size_t next_cap = list->cap == 0 ? 1024 : list->cap * 2; + pair_record *next = (pair_record *)realloc(list->items, next_cap * sizeof(pair_record)); + if (next == NULL) return -1; + list->items = next; + list->cap = next_cap; + } + list->items[list->count++] = (pair_record){a, b}; + return 0; +} + +typedef struct seq_ref { + const char *seq; + size_t len; +} seq_ref; + +static int cmp_seq_ref(const void *a, const void *b) { + const seq_ref *aa = (const seq_ref *)a; + const seq_ref *bb = (const seq_ref *)b; + size_t min_len = aa->len < bb->len ? aa->len : bb->len; + int c = memcmp(aa->seq, bb->seq, min_len); + if (c != 0) return c; + return aa->len > bb->len ? 1 : (aa->len < bb->len ? -1 : 0); +} + +static size_t count_unique_target_sequences(const seq_table *targets) { + if (targets->count == 0) return 0; + seq_ref *refs = (seq_ref *)malloc(targets->count * sizeof(seq_ref)); + if (refs == NULL) return 0; + for (size_t i = 0; i < targets->count; ++i) { + refs[i].seq = targets->records[i].seq; + refs[i].len = targets->records[i].len; + } + qsort(refs, targets->count, sizeof(seq_ref), cmp_seq_ref); + size_t unique = 1; + for (size_t i = 1; i < targets->count; ++i) { + if (refs[i].len != refs[i - 1].len || memcmp(refs[i].seq, refs[i - 1].seq, refs[i].len) != 0) { + ++unique; + } + } + free(refs); + return unique; +} + +static int write_audit_summary_json(const char *out_dir, const char *audit_mode, int k, + size_t n_targets, size_t unique_sequences, + const char *min_edit_distance_json, const char *min_hamming_distance_json, + int safe_at_k0, int safe_at_k1, const char *safe_at_k2_json, + const char *safe_at_hamming_k2_json, const char *safe_at_hamming_k3_json, + unsigned long long pairs_d0, unsigned long long pairs_d1, + unsigned long long pairs_d2, unsigned long long pairs_within_k, + unsigned long long risk_pairs_k1, const char *risk_pairs_k2_json, + const char *risk_pairs_hamming_k2_json, const char *risk_pairs_hamming_k3_json, + unsigned long long ambiguous_query_variants_k1, int recommended_k) { + char path[4096]; + if (path_join(path, sizeof(path), out_dir, "audit_summary.json") != 0) return -1; + FILE *out = open_output_file(path); + if (out == NULL) return -1; + fprintf(out, + "{\n" + " \"audit_mode\": \"%s\",\n" + " \"k\": %d,\n" + " \"targets\": %zu,\n" + " \"unique_sequences\": %zu,\n" + " \"duplicate_sequences\": %zu,\n" + " \"min_edit_distance\": %s,\n" + " \"min_hamming_distance\": %s,\n" + " \"safe_at_k0\": %s,\n" + " \"safe_at_k1\": %s,\n" + " \"safe_at_k2\": %s,\n" + " \"safe_at_hamming_k2\": %s,\n" + " \"safe_at_hamming_k3\": %s,\n" + " \"pairs_distance_0\": %llu,\n" + " \"pairs_distance_1\": %llu,\n" + " \"pairs_distance_2\": %llu,\n" + " \"pairs_within_requested_k\": %llu,\n" + " \"risk_pairs_for_k1\": %llu,\n" + " \"risk_pairs_for_k2\": %s,\n" + " \"risk_pairs_for_hamming_k2\": %s,\n" + " \"risk_pairs_for_hamming_k3\": %s,\n" + " \"ambiguous_query_variants_k1\": %llu,\n" + " \"recommended_k\": %d\n" + "}\n", + audit_mode, k, n_targets, unique_sequences, n_targets - unique_sequences, + min_edit_distance_json, min_hamming_distance_json, + safe_at_k0 ? "true" : "false", safe_at_k1 ? "true" : "false", safe_at_k2_json, + safe_at_hamming_k2_json, safe_at_hamming_k3_json, + pairs_d0, pairs_d1, pairs_d2, pairs_within_k, risk_pairs_k1, risk_pairs_k2_json, + risk_pairs_hamming_k2_json, risk_pairs_hamming_k3_json, + ambiguous_query_variants_k1, recommended_k); + if (fclose(out) != 0) return -1; + return 0; +} + +static int audit_fast_outputs(const seq_table *targets, const char *out_dir, int k) { + int rc = -1; + int min_dist = -1; + unsigned long long pairs_d0 = 0; + unsigned long long pairs_d1 = 0; + unsigned long long pairs_d2 = 0; + unsigned long long pairs_within_k = 0; + unsigned long long risk_pairs_k1 = 0; + unsigned long long ambiguous_query_variants_k1 = 0; + int *nearest_dist = NULL; + size_t *nearest_idx = NULL; + unsigned long long *near_k1 = NULL; + size_t *parent = NULL; + variant_record_list variants = {0}; + pair_record_list candidate_pairs = {0}; + pair_record_list unique_pairs = {0}; + FILE *pairs = NULL; + FILE *clusters = NULL; + FILE *safety = NULL; + FILE *summary = NULL; + FILE *variants_out = NULL; + char path[4096]; + + nearest_dist = (int *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(int)); + nearest_idx = (size_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(size_t)); + near_k1 = (unsigned long long *)calloc(targets->count == 0 ? 1 : targets->count, sizeof(unsigned long long)); + parent = (size_t *)malloc((targets->count == 0 ? 1 : targets->count) * sizeof(size_t)); + if (nearest_dist == NULL || nearest_idx == NULL || near_k1 == NULL || parent == NULL) goto done; + for (size_t i = 0; i < targets->count; ++i) { + nearest_dist[i] = -1; + nearest_idx[i] = (size_t)-1; + parent[i] = i; + } + + for (size_t i = 0; i < targets->count; ++i) { + string_list local = {0}; + if (add_k1_variants_for_target(&local, targets->records[i].seq, targets->records[i].len) != 0) { + free_string_list(&local); + goto done; + } + for (size_t v = 0; v < local.count; ++v) { + if (push_variant_record(&variants, local.items[v], i) != 0) { + free_string_list(&local); + goto done; + } + } + free_string_list(&local); + } + qsort(variants.items, variants.count, sizeof(variant_record), cmp_variant_record); + + if (path_join(path, sizeof(path), out_dir, "ambiguous_variants.tsv") != 0) goto done; + variants_out = open_output_file(path); + if (variants_out == NULL) goto done; + fprintf(variants_out, "query_variant\ttargets_within_k1\n"); + + for (size_t start = 0; start < variants.count;) { + size_t end = start + 1; + while (end < variants.count && strcmp(variants.items[start].key, variants.items[end].key) == 0) ++end; + size_t unique_targets = 0; + size_t last_target = (size_t)-1; + for (size_t i = start; i < end; ++i) { + if (variants.items[i].target != last_target) { + variants.items[start + unique_targets].target = variants.items[i].target; + last_target = variants.items[i].target; + ++unique_targets; + } + } + if (unique_targets >= 2) { + ++ambiguous_query_variants_k1; + fprintf(variants_out, "%s\t%zu\n", variants.items[start].key, unique_targets); + for (size_t i = 0; i < unique_targets; ++i) { + for (size_t j = i + 1; j < unique_targets; ++j) { + if (push_pair_record(&candidate_pairs, variants.items[start + i].target, + variants.items[start + j].target) != 0) { + goto done; + } + } + } + } + start = end; + } + fclose(variants_out); + variants_out = NULL; + + qsort(candidate_pairs.items, candidate_pairs.count, sizeof(pair_record), cmp_pair_record); + for (size_t i = 0; i < candidate_pairs.count; ++i) { + if (i > 0 && candidate_pairs.items[i].a == candidate_pairs.items[i - 1].a && + candidate_pairs.items[i].b == candidate_pairs.items[i - 1].b) { + continue; + } + if (push_pair_record(&unique_pairs, candidate_pairs.items[i].a, candidate_pairs.items[i].b) != 0) goto done; + } + + if (path_join(path, sizeof(path), out_dir, "collision_pairs.tsv") != 0) goto done; + pairs = open_output_file(path); + if (pairs == NULL) goto done; + fprintf(pairs, "target_a\ttarget_b\tsequence_a\tsequence_b\tdistance\trisk_at_k1\trisk_at_k2\texample_ambiguous_query\n"); + + for (size_t p = 0; p < unique_pairs.count; ++p) { + size_t i = unique_pairs.items[p].a; + size_t j = unique_pairs.items[p].b; + int d = qdaln_edit_distance(targets->records[i].seq, targets->records[i].len, + targets->records[j].seq, targets->records[j].len); + if (d < 0) goto done; + if (min_dist < 0 || d < min_dist) min_dist = d; + if (nearest_dist[i] < 0 || d < nearest_dist[i]) { + nearest_dist[i] = d; + nearest_idx[i] = j; + } + if (nearest_dist[j] < 0 || d < nearest_dist[j]) { + nearest_dist[j] = d; + nearest_idx[j] = i; + } + if (d == 0) ++pairs_d0; + if (d == 1) ++pairs_d1; + if (d == 2) ++pairs_d2; + if (d <= k) ++pairs_within_k; + if (d <= 2) { + ++risk_pairs_k1; + ++near_k1[i]; + ++near_k1[j]; + uf_union(parent, i, j); + } + fprintf(pairs, "%s\t%s\t%s\t%s\t%d\t%s\tnot_computed\t\n", + targets->records[i].id, targets->records[j].id, targets->records[i].seq, targets->records[j].seq, + d, d <= 2 ? "yes" : "no"); + } + fclose(pairs); + pairs = NULL; + + if (path_join(path, sizeof(path), out_dir, "target_safety.tsv") != 0) goto done; + safety = open_output_file(path); + if (safety == NULL) goto done; + fprintf(safety, "target_id\tsequence\tnearest_target\tnearest_distance\tsafe_at_k1\tsafe_at_k2\tnum_nearby_k1_risk_targets\n"); + for (size_t i = 0; i < targets->count; ++i) { + const char *near_id = nearest_idx[i] == (size_t)-1 ? "" : targets->records[nearest_idx[i]].id; + int nd = nearest_dist[i]; + fprintf(safety, "%s\t%s\t%s\t%d\t%s\tnot_computed\t%llu\n", + targets->records[i].id, targets->records[i].seq, near_id, nd, + (nd < 0 || nd >= 3) ? "yes" : "no", near_k1[i]); + } + fclose(safety); + safety = NULL; + + if (path_join(path, sizeof(path), out_dir, "collision_clusters.tsv") != 0) goto done; + clusters = open_output_file(path); + if (clusters == NULL) goto done; + fprintf(clusters, "cluster_id\ttarget_id\tsequence\n"); + for (size_t i = 0; i < targets->count; ++i) { + if (near_k1[i] == 0) continue; + fprintf(clusters, "%zu\t%s\t%s\n", uf_find(parent, i), targets->records[i].id, targets->records[i].seq); + } + fclose(clusters); + clusters = NULL; + + size_t unique_sequences = count_unique_target_sequences(targets); + if (path_join(path, sizeof(path), out_dir, "audit_summary.tsv") != 0) goto done; + summary = open_output_file(path); + if (summary == NULL) goto done; + fprintf(summary, "metric\tvalue\n"); + fprintf(summary, "audit_mode\tfast\n"); + fprintf(summary, "targets\t%zu\n", targets->count); + fprintf(summary, "unique_sequences\t%zu\n", unique_sequences); + fprintf(summary, "duplicate_sequences\t%zu\n", targets->count - unique_sequences); + fprintf(summary, "min_edit_distance\t%s\n", min_dist < 0 ? ">=3" : (min_dist == 0 ? "0" : (min_dist == 1 ? "1" : "2"))); + fprintf(summary, "safe_at_k0\t%s\n", pairs_d0 == 0 ? "yes" : "no"); + fprintf(summary, "safe_at_k1\t%s\n", risk_pairs_k1 == 0 ? "yes" : "no"); + fprintf(summary, "safe_at_k2\tnot_computed\n"); + fprintf(summary, "safe_at_hamming_k2\tnot_computed\n"); + fprintf(summary, "safe_at_hamming_k3\tnot_computed\n"); + fprintf(summary, "pairs_distance_0\t%llu\n", pairs_d0); + fprintf(summary, "pairs_distance_1\t%llu\n", pairs_d1); + fprintf(summary, "pairs_distance_2\t%llu\n", pairs_d2); + fprintf(summary, "pairs_within_requested_k\t%llu\n", pairs_within_k); + fprintf(summary, "risk_pairs_for_k1\t%llu\n", risk_pairs_k1); + fprintf(summary, "risk_pairs_for_k2\tnot_computed\n"); + fprintf(summary, "risk_pairs_for_hamming_k2\tnot_computed\n"); + fprintf(summary, "risk_pairs_for_hamming_k3\tnot_computed\n"); + fprintf(summary, "ambiguous_query_variants_k1\t%llu\n", ambiguous_query_variants_k1); + fprintf(summary, "recommended_k\t%d\n", pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0); + fclose(summary); + summary = NULL; + if (write_audit_summary_json(out_dir, "fast", k, targets->count, unique_sequences, + min_dist < 0 ? "\">=3\"" : (min_dist == 0 ? "0" : (min_dist == 1 ? "1" : "2")), + "null", + pairs_d0 == 0, risk_pairs_k1 == 0, "null", + "null", "null", + pairs_d0, pairs_d1, pairs_d2, pairs_within_k, risk_pairs_k1, "null", + "null", "null", + ambiguous_query_variants_k1, + pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0) != 0) { + goto done; + } + rc = 0; + +done: + if (pairs != NULL) fclose(pairs); + if (clusters != NULL) fclose(clusters); + if (safety != NULL) fclose(safety); + if (summary != NULL) fclose(summary); + if (variants_out != NULL) fclose(variants_out); + free(nearest_dist); + free(nearest_idx); + free(near_k1); + free(parent); + free_variant_record_list(&variants); + free_pair_record_list(&candidate_pairs); + free_pair_record_list(&unique_pairs); + return rc; +} + +static int run_audit(const char *argv0, int argc, char **argv) { + const char *targets_path = NULL; + const char *out_dir = NULL; + const char *audit_mode = "auto"; + int k = 1; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if ((strcmp(arg, "--targets") == 0 || strcmp(arg, "--library") == 0) && i < argc) { + targets_path = argv[i++]; + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 3) { + usage(argv0); + return 2; + } + } else if ((strcmp(arg, "--out-dir") == 0 || strcmp(arg, "--out") == 0) && i < argc) { + out_dir = argv[i++]; + } else if (strcmp(arg, "--audit-mode") == 0 && i < argc) { + audit_mode = argv[i++]; + if (strcmp(audit_mode, "auto") != 0 && strcmp(audit_mode, "exact") != 0 && strcmp(audit_mode, "fast") != 0) { + usage(argv0); + return 2; + } + } else { + usage(argv0); + return 2; + } + } + if (targets_path == NULL || out_dir == NULL) { + usage(argv0); + return 2; + } + + seq_table targets = {0}; + int rc = 1; + int min_dist = -1; + unsigned long long pairs_d0 = 0; + unsigned long long pairs_d1 = 0; + unsigned long long pairs_d2 = 0; + unsigned long long pairs_within_k = 0; + unsigned long long risk_pairs_k1 = 0; + unsigned long long risk_pairs_k2 = 0; + unsigned long long risk_pairs_hamming_k2 = 0; + unsigned long long risk_pairs_hamming_k3 = 0; + int min_hamming_dist = -1; + int *nearest_dist = NULL; + size_t *nearest_idx = NULL; + unsigned long long *near_k1 = NULL; + size_t *parent = NULL; + FILE *pairs = NULL; + FILE *clusters = NULL; + FILE *safety = NULL; + FILE *summary = NULL; + FILE *variants_out = NULL; + string_list k1_variants = {0}; + unsigned long long ambiguous_query_variants_k1 = 0; + char path[4096]; + + if (read_target_table(targets_path, &targets) != 0) { + fprintf(stderr, "failed to read targets\n"); + goto done; + } + if (ensure_dir(out_dir) != 0) { + fprintf(stderr, "failed to create audit output directory\n"); + goto done; + } + int use_fast = strcmp(audit_mode, "fast") == 0 || (strcmp(audit_mode, "auto") == 0 && targets.count > 2000); + if (use_fast) { + rc = audit_fast_outputs(&targets, out_dir, k) == 0 ? 0 : 1; + if (rc == 0) printf("%s\n", out_dir); + goto done; + } + nearest_dist = (int *)malloc((targets.count == 0 ? 1 : targets.count) * sizeof(int)); + nearest_idx = (size_t *)malloc((targets.count == 0 ? 1 : targets.count) * sizeof(size_t)); + near_k1 = (unsigned long long *)calloc(targets.count == 0 ? 1 : targets.count, sizeof(unsigned long long)); + parent = (size_t *)malloc((targets.count == 0 ? 1 : targets.count) * sizeof(size_t)); + if (nearest_dist == NULL || nearest_idx == NULL || near_k1 == NULL || parent == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t i = 0; i < targets.count; ++i) { + nearest_dist[i] = -1; + nearest_idx[i] = (size_t)-1; + parent[i] = i; + } + + if (path_join(path, sizeof(path), out_dir, "collision_pairs.tsv") != 0) goto done; + pairs = open_output_file(path); + if (pairs == NULL) goto done; + fprintf(pairs, "target_a\ttarget_b\tsequence_a\tsequence_b\tdistance\trisk_at_k1\trisk_at_k2\texample_ambiguous_query\n"); + + for (size_t i = 0; i < targets.count; ++i) { + for (size_t j = i + 1; j < targets.count; ++j) { + int d = qdaln_edit_distance(targets.records[i].seq, targets.records[i].len, + targets.records[j].seq, targets.records[j].len); + if (d < 0) goto done; + if (min_dist < 0 || d < min_dist) min_dist = d; + if (nearest_dist[i] < 0 || d < nearest_dist[i]) { + nearest_dist[i] = d; + nearest_idx[i] = j; + } + if (nearest_dist[j] < 0 || d < nearest_dist[j]) { + nearest_dist[j] = d; + nearest_idx[j] = i; + } + if (d == 0) ++pairs_d0; + if (d == 1) ++pairs_d1; + if (d == 2) ++pairs_d2; + if (d <= k) ++pairs_within_k; + if (d <= 2) { + ++risk_pairs_k1; + ++near_k1[i]; + ++near_k1[j]; + uf_union(parent, i, j); + } + if (d <= 4) ++risk_pairs_k2; + int hd = hamming_distance_cli(targets.records[i].seq, targets.records[i].len, + targets.records[j].seq, targets.records[j].len); + if (hd >= 0) { + if (min_hamming_dist < 0 || hd < min_hamming_dist) min_hamming_dist = hd; + if (hd <= 4) ++risk_pairs_hamming_k2; + if (hd <= 6) ++risk_pairs_hamming_k3; + } + if (d <= 2 || d <= 2 * k) { + const char *example = d == 0 ? targets.records[i].seq : ""; + fprintf(pairs, "%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\n", + targets.records[i].id, targets.records[j].id, targets.records[i].seq, targets.records[j].seq, + d, d <= 2 ? "yes" : "no", d <= 4 ? "yes" : "no", example); + } + } + } + fclose(pairs); + pairs = NULL; + + if (path_join(path, sizeof(path), out_dir, "target_safety.tsv") != 0) goto done; + safety = open_output_file(path); + if (safety == NULL) goto done; + fprintf(safety, "target_id\tsequence\tnearest_target\tnearest_distance\tsafe_at_k1\tsafe_at_k2\tnum_nearby_k1_risk_targets\n"); + for (size_t i = 0; i < targets.count; ++i) { + const char *near_id = nearest_idx[i] == (size_t)-1 ? "" : targets.records[nearest_idx[i]].id; + int nd = nearest_dist[i]; + fprintf(safety, "%s\t%s\t%s\t%d\t%s\t%s\t%llu\n", + targets.records[i].id, targets.records[i].seq, near_id, nd, + (nd < 0 || nd >= 3) ? "yes" : "no", + (nd < 0 || nd >= 5) ? "yes" : "no", + near_k1[i]); + } + fclose(safety); + safety = NULL; + + if (path_join(path, sizeof(path), out_dir, "collision_clusters.tsv") != 0) goto done; + clusters = open_output_file(path); + if (clusters == NULL) goto done; + fprintf(clusters, "cluster_id\ttarget_id\tsequence\n"); + for (size_t i = 0; i < targets.count; ++i) { + if (near_k1[i] == 0) continue; + fprintf(clusters, "%zu\t%s\t%s\n", uf_find(parent, i), targets.records[i].id, targets.records[i].seq); + } + fclose(clusters); + clusters = NULL; + + size_t unique_sequences = 0; + for (size_t i = 0; i < targets.count; ++i) { + int seen = 0; + for (size_t j = 0; j < i; ++j) { + if (targets.records[i].len == targets.records[j].len && + memcmp(targets.records[i].seq, targets.records[j].seq, targets.records[i].len) == 0) { + seen = 1; + break; + } + } + if (!seen) ++unique_sequences; + } + + if (path_join(path, sizeof(path), out_dir, "ambiguous_variants.tsv") != 0) goto done; + variants_out = open_output_file(path); + if (variants_out == NULL) goto done; + fprintf(variants_out, "query_variant\ttargets_within_k1\n"); + for (size_t i = 0; i < targets.count; ++i) { + if (add_k1_variants_for_target(&k1_variants, targets.records[i].seq, targets.records[i].len) != 0) { + fprintf(stderr, "failed to enumerate k=1 variants\n"); + goto done; + } + } + for (size_t vi = 0; vi < k1_variants.count; ++vi) { + unsigned long long within = 0; + size_t q_len = strlen(k1_variants.items[vi]); + for (size_t ti = 0; ti < targets.count; ++ti) { + int ok = qdaln_edit_distance_leq(k1_variants.items[vi], q_len, targets.records[ti].seq, + targets.records[ti].len, 1); + if (ok < 0) goto done; + if (ok) ++within; + } + if (within >= 2) { + ++ambiguous_query_variants_k1; + fprintf(variants_out, "%s\t%llu\n", k1_variants.items[vi], within); + } + } + fclose(variants_out); + variants_out = NULL; + + if (path_join(path, sizeof(path), out_dir, "audit_summary.tsv") != 0) goto done; + summary = open_output_file(path); + if (summary == NULL) goto done; + fprintf(summary, "metric\tvalue\n"); + fprintf(summary, "audit_mode\texact\n"); + fprintf(summary, "targets\t%zu\n", targets.count); + fprintf(summary, "unique_sequences\t%zu\n", unique_sequences); + fprintf(summary, "duplicate_sequences\t%zu\n", targets.count - unique_sequences); + fprintf(summary, "min_edit_distance\t%d\n", min_dist); + if (min_hamming_dist >= 0) { + fprintf(summary, "min_hamming_distance\t%d\n", min_hamming_dist); + } else { + fprintf(summary, "min_hamming_distance\tnot_computed\n"); + } + fprintf(summary, "safe_at_k0\t%s\n", pairs_d0 == 0 ? "yes" : "no"); + fprintf(summary, "safe_at_k1\t%s\n", risk_pairs_k1 == 0 ? "yes" : "no"); + fprintf(summary, "safe_at_k2\t%s\n", risk_pairs_k2 == 0 ? "yes" : "no"); + fprintf(summary, "safe_at_hamming_k2\t%s\n", risk_pairs_hamming_k2 == 0 ? "yes" : "no"); + fprintf(summary, "safe_at_hamming_k3\t%s\n", risk_pairs_hamming_k3 == 0 ? "yes" : "no"); + fprintf(summary, "pairs_distance_0\t%llu\n", pairs_d0); + fprintf(summary, "pairs_distance_1\t%llu\n", pairs_d1); + fprintf(summary, "pairs_distance_2\t%llu\n", pairs_d2); + fprintf(summary, "pairs_within_requested_k\t%llu\n", pairs_within_k); + fprintf(summary, "risk_pairs_for_k1\t%llu\n", risk_pairs_k1); + fprintf(summary, "risk_pairs_for_k2\t%llu\n", risk_pairs_k2); + fprintf(summary, "risk_pairs_for_hamming_k2\t%llu\n", risk_pairs_hamming_k2); + fprintf(summary, "risk_pairs_for_hamming_k3\t%llu\n", risk_pairs_hamming_k3); + fprintf(summary, "ambiguous_query_variants_k1\t%llu\n", ambiguous_query_variants_k1); + fprintf(summary, "recommended_k\t%d\n", pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0); + fclose(summary); + summary = NULL; + char min_dist_json[32]; + char min_hamming_dist_json[32]; + char risk_pairs_k2_json[32]; + char risk_pairs_hamming_k2_json[32]; + char risk_pairs_hamming_k3_json[32]; + snprintf(min_dist_json, sizeof(min_dist_json), "%d", min_dist); + snprintf(min_hamming_dist_json, sizeof(min_hamming_dist_json), "%d", min_hamming_dist); + snprintf(risk_pairs_k2_json, sizeof(risk_pairs_k2_json), "%llu", risk_pairs_k2); + snprintf(risk_pairs_hamming_k2_json, sizeof(risk_pairs_hamming_k2_json), "%llu", risk_pairs_hamming_k2); + snprintf(risk_pairs_hamming_k3_json, sizeof(risk_pairs_hamming_k3_json), "%llu", risk_pairs_hamming_k3); + if (write_audit_summary_json(out_dir, "exact", k, targets.count, unique_sequences, + min_dist_json, min_hamming_dist >= 0 ? min_hamming_dist_json : "null", + pairs_d0 == 0, risk_pairs_k1 == 0, + risk_pairs_k2 == 0 ? "true" : "false", + risk_pairs_hamming_k2 == 0 ? "true" : "false", + risk_pairs_hamming_k3 == 0 ? "true" : "false", + pairs_d0, pairs_d1, pairs_d2, pairs_within_k, risk_pairs_k1, + risk_pairs_k2_json, risk_pairs_hamming_k2_json, risk_pairs_hamming_k3_json, + ambiguous_query_variants_k1, + pairs_d0 == 0 && (k == 0 || risk_pairs_k1 == 0) ? k : 0) != 0) { + goto done; + } + + printf("%s\n", out_dir); + rc = 0; + +done: + if (pairs != NULL) fclose(pairs); + if (clusters != NULL) fclose(clusters); + if (safety != NULL) fclose(safety); + if (summary != NULL) fclose(summary); + if (variants_out != NULL) fclose(variants_out); + free_string_list(&k1_variants); + free(nearest_dist); + free(nearest_idx); + free(near_k1); + free(parent); + free_table(&targets); + return rc; +} + +static void write_fastq_record(FILE *out, const char *header, const char *seq, const char *plus, const char *qual) { + fprintf(out, "%s\n%s\n%s\n%s\n", header, seq, plus, qual); +} + +typedef struct unmatched_entry { + char *seq; + unsigned long long count; + int offset_hint; + unsigned long long low_quality_count; + char *adapter_hint; +} unmatched_entry; + +typedef struct unmatched_table { + unmatched_entry *entries; + size_t count; + size_t cap; +} unmatched_table; + +static void free_unmatched_table(unmatched_table *table) { + for (size_t i = 0; i < table->count; ++i) { + free(table->entries[i].seq); + free(table->entries[i].adapter_hint); + } + free(table->entries); + table->entries = NULL; + table->count = 0; + table->cap = 0; +} + +static int add_unmatched_observation(unmatched_table *table, const char *seq, int offset_hint, + int low_quality, const char *adapter_hint) { + for (size_t i = 0; i < table->count; ++i) { + if (strcmp(table->entries[i].seq, seq) == 0) { + ++table->entries[i].count; + if (table->entries[i].offset_hint == 0 && offset_hint != 0) table->entries[i].offset_hint = offset_hint; + if (low_quality) ++table->entries[i].low_quality_count; + if ((table->entries[i].adapter_hint == NULL || table->entries[i].adapter_hint[0] == '\0') && + adapter_hint != NULL && adapter_hint[0] != '\0') { + free(table->entries[i].adapter_hint); + table->entries[i].adapter_hint = xstrndup(adapter_hint, strlen(adapter_hint)); + if (table->entries[i].adapter_hint == NULL) return -1; + } + return 0; + } + } + if (table->count == table->cap) { + size_t next_cap = table->cap == 0 ? 16 : table->cap * 2; + unmatched_entry *next = (unmatched_entry *)realloc(table->entries, next_cap * sizeof(unmatched_entry)); + if (next == NULL) return -1; + table->entries = next; + table->cap = next_cap; + } + table->entries[table->count].seq = xstrndup(seq, strlen(seq)); + if (table->entries[table->count].seq == NULL) return -1; + table->entries[table->count].count = 1; + table->entries[table->count].offset_hint = offset_hint; + table->entries[table->count].low_quality_count = low_quality ? 1 : 0; + table->entries[table->count].adapter_hint = adapter_hint == NULL ? xstrndup("", 0) : xstrndup(adapter_hint, strlen(adapter_hint)); + if (table->entries[table->count].adapter_hint == NULL) { + free(table->entries[table->count].seq); + return -1; + } + ++table->count; + return 0; +} + +static int cmp_unmatched_entry_desc(const void *a, const void *b) { + const unmatched_entry *aa = (const unmatched_entry *)a; + const unmatched_entry *bb = (const unmatched_entry *)b; + if (aa->count != bb->count) return aa->count < bb->count ? 1 : -1; + return strcmp(aa->seq, bb->seq); +} + +static int contains_base_n(const char *seq) { + for (; *seq != '\0'; ++seq) { + if (*seq == 'N' || *seq == 'n') return 1; + } + return 0; +} + +static char complement_base(char c) { + switch (c) { + case 'A': + case 'a': + return 'T'; + case 'C': + case 'c': + return 'G'; + case 'G': + case 'g': + return 'C'; + case 'T': + case 't': + return 'A'; + default: + return 'N'; + } +} + +static int reverse_complement_seq(const char *seq, char *out, size_t out_cap) { + size_t len = strlen(seq); + if (len + 1 > out_cap) return -1; + for (size_t i = 0; i < len; ++i) out[i] = complement_base(seq[len - 1 - i]); + out[len] = '\0'; + return 0; +} + +static int nearest_target_for_query(const seq_table *targets, const char *query, int *nearest_index, int *nearest_dist) { + *nearest_index = -1; + *nearest_dist = -1; + size_t q_len = strlen(query); + for (size_t i = 0; i < targets->count; ++i) { + int d = qdaln_edit_distance(query, q_len, targets->records[i].seq, targets->records[i].len); + if (d < 0) return -1; + if (*nearest_dist < 0 || d < *nearest_dist) { + *nearest_dist = d; + *nearest_index = (int)i; + } + } + return 0; +} + +static int find_offset_hint(const qdaln_index *index, const char *seq, size_t seq_len, size_t target_start, + size_t target_len, int k, size_t offset_window) { + if (offset_window == 0) return 0; + char observed[8192]; + if (target_len >= sizeof(observed)) return 0; + for (size_t step = 1; step <= offset_window; ++step) { + for (int sign = 1; sign >= -1; sign -= 2) { + if (sign < 0 && target_start < step) continue; + size_t offset = sign > 0 ? target_start + step : target_start - step; + if (offset > seq_len || target_len > seq_len - offset) continue; + memcpy(observed, seq + offset, target_len); + observed[target_len] = '\0'; + uppercase_ascii(observed); + const char *read_ptr = observed; + size_t read_len = target_len; + qdaln_match_result r; + qdaln_index_stats stats; + if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &r, &stats) != 0) return 0; + if (r.status == QDALN_MATCH_UNIQUE) return sign > 0 ? (int)step : -(int)step; + } + } + return 0; +} + +static int window_has_low_quality(const char *qual, size_t target_start, size_t target_len, int threshold) { + if (threshold < 0) return 0; + size_t qual_len = strlen(qual); + if (target_start > qual_len || target_len > qual_len - target_start) return 0; + for (size_t i = 0; i < target_len; ++i) { + int phred = (int)((unsigned char)qual[target_start + i]) - 33; + if (phred < threshold) return 1; + } + return 0; +} + +static int run_inspect_unmatched(const char *argv0, int argc, char **argv) { + const char *targets_path = NULL; + const char *reads_path = NULL; + const char *out_path = NULL; + size_t target_start = 0; + size_t target_len = 0; + size_t top_n = 100; + size_t offset_window = 0; + char adapter[1024] = ""; + int low_quality_threshold = -1; + int k = -1; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if ((strcmp(arg, "--targets") == 0 || strcmp(arg, "--library") == 0) && i < argc) { + targets_path = argv[i++]; + } else if (strcmp(arg, "--reads") == 0 && i < argc) { + reads_path = argv[i++]; + } else if ((strcmp(arg, "--target-start") == 0 || strcmp(arg, "--guide-start") == 0) && i < argc) { + if (parse_size_value(argv[i++], &target_start) != 0) { + usage(argv0); + return 2; + } + } else if ((strcmp(arg, "--target-length") == 0 || strcmp(arg, "--guide-length") == 0) && i < argc) { + if (parse_size_value(argv[i++], &target_len) != 0 || target_len == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || (k != 0 && k != 1)) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--top") == 0 && i < argc) { + if (parse_size_value(argv[i++], &top_n) != 0 || top_n == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--offset-window") == 0 && i < argc) { + if (parse_size_value(argv[i++], &offset_window) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--adapter") == 0 && i < argc) { + const char *value = argv[i++]; + size_t n = strlen(value); + if (n == 0 || n >= sizeof(adapter)) { + usage(argv0); + return 2; + } + memcpy(adapter, value, n + 1); + uppercase_ascii(adapter); + } else if (strcmp(arg, "--low-quality-threshold") == 0 && i < argc) { + if (parse_int_value(argv[i++], &low_quality_threshold) != 0 || + low_quality_threshold < 0 || low_quality_threshold > 93) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--out") == 0 && i < argc) { + out_path = argv[i++]; + } else { + usage(argv0); + return 2; + } + } + if (targets_path == NULL || reads_path == NULL || out_path == NULL || target_len == 0 || k < 0) { + usage(argv0); + return 2; + } + + seq_table targets = {0}; + const char **target_ptrs = NULL; + size_t *target_lens = NULL; + qdaln_index *index = NULL; + fastq_reader reader = {0}; + unmatched_table unmatched = {0}; + FILE *out = NULL; + int rc = 1; + + if (read_target_table(targets_path, &targets) != 0) { + fprintf(stderr, "failed to read targets\n"); + goto done; + } + if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + index = qdaln_index_build(target_ptrs, target_lens, targets.count); + if (index == NULL) { + fprintf(stderr, "failed to build target index\n"); + goto done; + } + if (fastq_reader_open(&reader, reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + goto done; + } + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + char observed[8192]; + int got = 0; + size_t seq_len = 0; + while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { + qdaln_match_result r = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + observed[0] = '\0'; + if (target_start <= seq_len && target_len <= seq_len - target_start && target_len < sizeof(observed)) { + memcpy(observed, seq + target_start, target_len); + observed[target_len] = '\0'; + uppercase_ascii(observed); + const char *read_ptr = observed; + size_t read_len = target_len; + qdaln_index_stats stats; + if (qdaln_index_assign_stats(index, &read_ptr, &read_len, 1, k, &r, &stats) != 0) { + fprintf(stderr, "assignment failed\n"); + goto done; + } + } else { + strncpy(observed, "", sizeof(observed) - 1); + observed[sizeof(observed) - 1] = '\0'; + } + if (r.status == QDALN_MATCH_NONE || r.status == QDALN_MATCH_INVALID) { + int offset_hint = strcmp(observed, "") == 0 ? 0 : + find_offset_hint(index, seq, seq_len, target_start, target_len, k, offset_window); + char seq_upper[8192]; + seq_upper[0] = '\0'; + const char *adapter_hint = ""; + int low_quality = strcmp(observed, "") == 0 ? 0 : + window_has_low_quality(qual, target_start, target_len, low_quality_threshold); + if (adapter[0] != '\0') { + strncpy(seq_upper, seq, sizeof(seq_upper) - 1); + seq_upper[sizeof(seq_upper) - 1] = '\0'; + uppercase_ascii(seq_upper); + if (strstr(seq_upper, adapter) != NULL || strstr(observed, adapter) != NULL) adapter_hint = adapter; + } + if (add_unmatched_observation(&unmatched, observed, offset_hint, low_quality, adapter_hint) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + } + } + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + goto done; + } + + qsort(unmatched.entries, unmatched.count, sizeof(unmatched_entry), cmp_unmatched_entry_desc); + out = open_output_file(out_path); + if (out == NULL) { + fprintf(stderr, "failed to open unmatched inspection output\n"); + goto done; + } + fprintf(out, "sequence\tcount\tlength\tnearest_target\tnearest_distance\tnearest_edit_class\tpossible_reason\treverse_complement\trevcomp_nearest_target\trevcomp_nearest_distance\toffset_hint\tadapter_hint\n"); + size_t limit = unmatched.count < top_n ? unmatched.count : top_n; + for (size_t i = 0; i < limit; ++i) { + int nearest_index = -1; + int nearest_dist = -1; + int rc_nearest_index = -1; + int rc_nearest_dist = -1; + const char *nearest_id = ""; + const char *rc_nearest_id = ""; + const char *edit_class = "invalid"; + const char *reason = "wrong_length"; + char rc_seq[8192] = ""; + if (strcmp(unmatched.entries[i].seq, "") != 0 && + nearest_target_for_query(&targets, unmatched.entries[i].seq, &nearest_index, &nearest_dist) == 0) { + if (nearest_index >= 0) { + nearest_id = targets.records[nearest_index].id; + int kind = correction_kind(unmatched.entries[i].seq, strlen(unmatched.entries[i].seq), + targets.records[nearest_index].seq, targets.records[nearest_index].len, + nearest_dist); + edit_class = correction_name(kind); + } + if (contains_base_n(unmatched.entries[i].seq)) reason = "contains_N"; + else if (nearest_dist > k) reason = "near_known_target_above_k"; + else reason = "unknown"; + if (unmatched.entries[i].low_quality_count != 0) reason = "low_quality_candidate"; + if (unmatched.entries[i].adapter_hint != NULL && unmatched.entries[i].adapter_hint[0] != '\0') { + reason = "adapter_or_primer_candidate"; + } + if (unmatched.entries[i].offset_hint != 0) reason = "offset_shift_candidate"; + if (reverse_complement_seq(unmatched.entries[i].seq, rc_seq, sizeof(rc_seq)) == 0 && + nearest_target_for_query(&targets, rc_seq, &rc_nearest_index, &rc_nearest_dist) == 0 && + rc_nearest_index >= 0) { + rc_nearest_id = targets.records[rc_nearest_index].id; + if (unmatched.entries[i].offset_hint == 0 && + rc_nearest_dist <= k && (nearest_dist < 0 || rc_nearest_dist < nearest_dist)) { + reason = "reverse_complement_candidate"; + } + } + } + fprintf(out, "%s\t%llu\t%zu\t%s\t%d\t%s\t%s\t%s\t%s\t%d\t", + unmatched.entries[i].seq, unmatched.entries[i].count, strlen(unmatched.entries[i].seq), + nearest_id, nearest_dist, edit_class, reason, rc_seq, rc_nearest_id, rc_nearest_dist); + if (unmatched.entries[i].offset_hint != 0) fprintf(out, "%d", unmatched.entries[i].offset_hint); + fprintf(out, "\t%s\n", unmatched.entries[i].adapter_hint == NULL ? "" : unmatched.entries[i].adapter_hint); + } + rc = 0; + +done: + if (out != NULL) fclose(out); + fastq_reader_close(&reader); + qdaln_index_free(index); + free(target_ptrs); + free(target_lens); + free_unmatched_table(&unmatched); + free_table(&targets); + return rc; +} + +typedef struct pair_count_stats { + unsigned long long total_reads; + unsigned long long assigned_pairs; + unsigned long long pair_ambiguous; + unsigned long long left_unmatched; + unsigned long long right_unmatched; + unsigned long long invalid; + unsigned long long candidates_considered; + unsigned long long candidates_verified; +} pair_count_stats; + +static const char *pair_status_name(qdaln_match_result left, qdaln_match_result right) { + if (left.status == QDALN_MATCH_INVALID || right.status == QDALN_MATCH_INVALID) return "invalid"; + if (left.status == QDALN_MATCH_AMBIGUOUS || right.status == QDALN_MATCH_AMBIGUOUS) return "ambiguous"; + if (left.status == QDALN_MATCH_NONE || right.status == QDALN_MATCH_NONE) return "none"; + if (left.status == QDALN_MATCH_UNIQUE && right.status == QDALN_MATCH_UNIQUE) return "unique"; + return "invalid"; +} + +static void print_pair_assignment_row(FILE *out, const char *read_id, const seq_table *left_targets, + const seq_table *right_targets, const char *left_observed, + qdaln_match_result left, const char *right_observed, + qdaln_match_result right) { + const char *left_id = left.target_index >= 0 ? left_targets->records[left.target_index].id : ""; + const char *right_id = right.target_index >= 0 ? right_targets->records[right.target_index].id : ""; + fprintf(out, "%s\t%s\t%d\t%s\t%s\t%d\t%s\t%d\t%s\t%s\t%d\t%s\n", + read_id, left_observed, left.target_index, left_id, status_name(left.status), left.best_distance, + right_observed, right.target_index, right_id, status_name(right.status), right.best_distance, + pair_status_name(left, right)); +} + +static int run_pair_count(const char *argv0, int argc, char **argv) { + const char *left_path = NULL; + const char *right_path = NULL; + const char *reads_path = NULL; + const char *out_path = NULL; + const char *summary_path = NULL; + const char *assignments_path = NULL; + size_t left_start = 0; + size_t right_start = 0; + size_t left_len = 0; + size_t right_len = 0; + int k = -1; + count_metric metric = COUNT_METRIC_LEVENSHTEIN; + ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if (strcmp(arg, "--left-targets") == 0 && i < argc) { + left_path = argv[i++]; + } else if (strcmp(arg, "--right-targets") == 0 && i < argc) { + right_path = argv[i++]; + } else if (strcmp(arg, "--reads") == 0 && i < argc) { + reads_path = argv[i++]; + } else if (strcmp(arg, "--left-start") == 0 && i < argc) { + if (parse_size_value(argv[i++], &left_start) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--left-length") == 0 && i < argc) { + if (parse_size_value(argv[i++], &left_len) != 0 || left_len == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--right-start") == 0 && i < argc) { + if (parse_size_value(argv[i++], &right_start) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--right-length") == 0 && i < argc) { + if (parse_size_value(argv[i++], &right_len) != 0 || right_len == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 2) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--metric") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "hamming") == 0) { + metric = COUNT_METRIC_HAMMING; + } else if (strcmp(value, "levenshtein") == 0) { + metric = COUNT_METRIC_LEVENSHTEIN; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "radius") == 0) { + assignment_policy = AMBIGUITY_POLICY_RADIUS; + } else if (strcmp(value, "best") == 0) { + assignment_policy = AMBIGUITY_POLICY_BEST; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--out") == 0 && i < argc) { + out_path = argv[i++]; + } else if (strcmp(arg, "--summary") == 0 && i < argc) { + summary_path = argv[i++]; + } else if (strcmp(arg, "--assignments") == 0 && i < argc) { + assignments_path = argv[i++]; + } else { + usage(argv0); + return 2; + } + } + + if (left_path == NULL || right_path == NULL || reads_path == NULL || out_path == NULL || + left_len == 0 || right_len == 0 || k < 0) { + usage(argv0); + return 2; + } + if (metric == COUNT_METRIC_HAMMING && k > 1) { + fprintf(stderr, "--k 2 is only valid with --metric levenshtein\n"); + return 2; + } + + seq_table left_targets = {0}; + seq_table right_targets = {0}; + const char **left_ptrs = NULL; + const char **right_ptrs = NULL; + size_t *left_lens = NULL; + size_t *right_lens = NULL; + qdaln_index *left_index = NULL; + qdaln_index *right_index = NULL; + fastq_reader reader = {0}; + FILE *out = NULL; + FILE *summary = NULL; + FILE *assignments = NULL; + unsigned long long *pair_counts = NULL; + pair_count_stats stats = {0}; + int rc = 1; + + if (read_target_table(left_path, &left_targets) != 0 || read_target_table(right_path, &right_targets) != 0) { + fprintf(stderr, "failed to read pair target tables\n"); + goto done; + } + int left_id_check = validate_unique_seq_ids(&left_targets, "left target"); + int right_id_check = validate_unique_seq_ids(&right_targets, "right target"); + if (left_id_check != 0 || right_id_check != 0) { + if (left_id_check == -1 || right_id_check == -1) fprintf(stderr, "out of memory\n"); + goto done; + } + if (metric == COUNT_METRIC_HAMMING && + (!all_targets_have_length(&left_targets, left_len) || !all_targets_have_length(&right_targets, right_len))) { + fprintf(stderr, "--metric hamming requires targets to match their configured window lengths\n"); + goto done; + } + if (build_target_arrays(&left_targets, &left_ptrs, &left_lens) != 0 || + build_target_arrays(&right_targets, &right_ptrs, &right_lens) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + left_index = qdaln_index_build(left_ptrs, left_lens, left_targets.count); + right_index = qdaln_index_build(right_ptrs, right_lens, right_targets.count); + if (left_index == NULL || right_index == NULL) { + fprintf(stderr, "failed to build pair target indexes\n"); + goto done; + } + size_t pair_count_slots = 0; + if (checked_mul_size(left_targets.count, right_targets.count, &pair_count_slots) != 0) { + fprintf(stderr, "pair count matrix is too large\n"); + goto done; + } + pair_counts = (unsigned long long *)calloc(alloc_count_or_one(pair_count_slots), sizeof(unsigned long long)); + if (pair_counts == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + if (fastq_reader_open(&reader, reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + goto done; + } + if (assignments_path != NULL) { + assignments = open_output_file(assignments_path); + if (assignments == NULL) { + fprintf(stderr, "failed to open assignments output\n"); + goto done; + } + fprintf(assignments, "read_id\tleft_observed\tleft_index\tleft_id\tleft_status\tleft_distance\tright_observed\tright_index\tright_id\tright_status\tright_distance\tpair_status\n"); + } + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + char read_id[8192]; + char left_observed[8192]; + char right_observed[8192]; + size_t seq_len = 0; + int got = 0; + while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { + (void)plus; + (void)qual; + qdaln_match_result left = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_match_result right = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_index_stats left_stats = {0, 0}; + qdaln_index_stats right_stats = {0, 0}; + fastq_read_id(header, read_id, sizeof(read_id)); + left_observed[0] = '\0'; + right_observed[0] = '\0'; + ++stats.total_reads; + + if (assign_count_window(left_index, seq, seq_len, left_start, left_len, k, metric, 0, + &left, &left_stats, left_observed, sizeof(left_observed), 0) != 0 || + assign_count_window(right_index, seq, seq_len, right_start, right_len, k, metric, 0, + &right, &right_stats, right_observed, sizeof(right_observed), 0) != 0) { + fprintf(stderr, "FASTQ pair assignment failed\n"); + goto done; + } + apply_ambiguity_policy(&left, assignment_policy); + apply_ambiguity_policy(&right, assignment_policy); + stats.candidates_considered += left_stats.candidates_considered + right_stats.candidates_considered; + stats.candidates_verified += left_stats.candidates_verified + right_stats.candidates_verified; + + const char *pair_status = pair_status_name(left, right); + if (strcmp(pair_status, "unique") == 0) { + size_t slot = (size_t)left.target_index * right_targets.count + (size_t)right.target_index; + ++pair_counts[slot]; + ++stats.assigned_pairs; + } else if (strcmp(pair_status, "invalid") == 0) { + ++stats.invalid; + } else { + if (left.status == QDALN_MATCH_AMBIGUOUS || right.status == QDALN_MATCH_AMBIGUOUS) ++stats.pair_ambiguous; + if (left.status == QDALN_MATCH_NONE) ++stats.left_unmatched; + if (right.status == QDALN_MATCH_NONE) ++stats.right_unmatched; + } + if (assignments != NULL) { + print_pair_assignment_row(assignments, read_id, &left_targets, &right_targets, + left_observed, left, right_observed, right); + } + } + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + goto done; + } + + out = open_output_file(out_path); + if (out == NULL) { + fprintf(stderr, "failed to open pair-count output\n"); + goto done; + } + fprintf(out, "left_id\tright_id\tcount\n"); + for (size_t li = 0; li < left_targets.count; ++li) { + for (size_t ri = 0; ri < right_targets.count; ++ri) { + unsigned long long count = pair_counts[li * right_targets.count + ri]; + if (count == 0) continue; + fprintf(out, "%s\t%s\t%llu\n", left_targets.records[li].id, right_targets.records[ri].id, count); + } + } + + if (summary_path != NULL) { + summary = open_output_file(summary_path); + if (summary == NULL) { + fprintf(stderr, "failed to open pair-count summary\n"); + goto done; + } + fprintf(summary, + "{\n \"workflow\": \"pair-count\",\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"left_start\": %zu,\n \"left_length\": %zu,\n \"right_start\": %zu,\n \"right_length\": %zu,\n \"n_left_targets\": %zu,\n \"n_right_targets\": %zu,\n \"total_reads\": %llu,\n \"assigned_pairs\": %llu,\n \"pair_ambiguous\": %llu,\n \"left_unmatched\": %llu,\n \"right_unmatched\": %llu,\n \"invalid\": %llu,\n \"candidates_considered\": %llu,\n \"candidates_verified\": %llu\n}\n", + k, metric_name(metric), ambiguity_policy_name(assignment_policy), qdaln_alphabet_policy(), left_start, left_len, right_start, right_len, + left_targets.count, right_targets.count, stats.total_reads, stats.assigned_pairs, + stats.pair_ambiguous, stats.left_unmatched, stats.right_unmatched, stats.invalid, + stats.candidates_considered, stats.candidates_verified); + } + + rc = 0; + +done: + if (out != NULL) fclose(out); + if (summary != NULL) fclose(summary); + if (assignments != NULL) fclose(assignments); + fastq_reader_close(&reader); + qdaln_index_free(left_index); + qdaln_index_free(right_index); + free(left_ptrs); + free(right_ptrs); + free(left_lens); + free(right_lens); + free(pair_counts); + free_table(&left_targets); + free_table(&right_targets); + return rc; +} + +static FILE *open_demux_target_file(FILE **files, const seq_table *targets, size_t target_index, const char *out_dir) { + if (files[target_index] != NULL) return files[target_index]; + char safe_id[512]; + sanitize_filename(targets->records[target_index].id, safe_id, sizeof(safe_id)); + char name[600]; + int n = snprintf(name, sizeof(name), "%s.fastq", safe_id); + if (n < 0 || (size_t)n >= sizeof(name)) return NULL; + char path[4096]; + if (path_join(path, sizeof(path), out_dir, name) != 0) return NULL; + files[target_index] = open_output_file(path); + return files[target_index]; +} + +static int run_demux(const char *argv0, int argc, char **argv) { + const char *barcodes_path = NULL; + const char *reads_path = NULL; + const char *out_dir = NULL; + const char *summary_path = NULL; + const char *assignments_path = NULL; + const char *ambiguous_path = NULL; + const char *unmatched_path = NULL; + count_metric metric = COUNT_METRIC_LEVENSHTEIN; + size_t barcode_start = 0; + size_t barcode_len = 0; + int auto_barcode_len = 0; + size_t indel_window = 0; + int max_correction_qual = -1; + int k = -1; + ambiguity_policy assignment_policy = AMBIGUITY_POLICY_RADIUS; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if ((strcmp(arg, "--barcodes") == 0 || strcmp(arg, "--targets") == 0) && i < argc) { + barcodes_path = argv[i++]; + } else if (strcmp(arg, "--reads") == 0 && i < argc) { + reads_path = argv[i++]; + } else if ((strcmp(arg, "--barcode-start") == 0 || strcmp(arg, "--target-start") == 0) && i < argc) { + if (parse_size_value(argv[i++], &barcode_start) != 0) { + usage(argv0); + return 2; + } + } else if ((strcmp(arg, "--barcode-length") == 0 || strcmp(arg, "--target-length") == 0) && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "auto") == 0) { + auto_barcode_len = 1; + barcode_len = 0; + } else if (parse_size_value(value, &barcode_len) != 0 || barcode_len == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || k < 0 || k > 2) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--metric") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "hamming") == 0) { + metric = COUNT_METRIC_HAMMING; + } else if (strcmp(value, "levenshtein") == 0) { + metric = COUNT_METRIC_LEVENSHTEIN; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--ambiguity-policy") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "radius") == 0) { + assignment_policy = AMBIGUITY_POLICY_RADIUS; + } else if (strcmp(value, "best") == 0) { + assignment_policy = AMBIGUITY_POLICY_BEST; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--indel-window") == 0 && i < argc) { + if (parse_size_value(argv[i++], &indel_window) != 0 || indel_window > 1) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--max-correction-qual") == 0 && i < argc) { + if (parse_int_value(argv[i++], &max_correction_qual) != 0 || + max_correction_qual < 0 || max_correction_qual > 93) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--out-dir") == 0 && i < argc) { + out_dir = argv[i++]; + } else if (strcmp(arg, "--summary") == 0 && i < argc) { + summary_path = argv[i++]; + } else if (strcmp(arg, "--qc") == 0 && i < argc) { + summary_path = argv[i++]; + } else if (strcmp(arg, "--assignments") == 0 && i < argc) { + assignments_path = argv[i++]; + } else if (strcmp(arg, "--ambiguous-out") == 0 && i < argc) { + ambiguous_path = argv[i++]; + } else if (strcmp(arg, "--unmatched-out") == 0 && i < argc) { + unmatched_path = argv[i++]; + } else { + usage(argv0); + return 2; + } + } + + if (barcodes_path == NULL || reads_path == NULL || out_dir == NULL || (barcode_len == 0 && !auto_barcode_len) || k < 0) { + usage(argv0); + return 2; + } + if (metric == COUNT_METRIC_HAMMING && indel_window != 0) { + fprintf(stderr, "--indel-window is only valid with --metric levenshtein\n"); + return 2; + } + if (metric == COUNT_METRIC_HAMMING && k > 1) { + fprintf(stderr, "--k 2 is only valid with --metric levenshtein\n"); + return 2; + } + if (indel_window != 0 && k != 1) { + fprintf(stderr, "--indel-window requires --k 1\n"); + return 2; + } + + seq_table targets = {0}; + fastq_reader reader = {0}; + qdaln_index *index = NULL; + hamming_lookup hlookup = {0}; + levenshtein1_lookup levlookup = {0}; + const char **target_ptrs = NULL; + size_t *target_lens = NULL; + size_t *auto_barcode_lens = NULL; + size_t auto_barcode_lens_count = 0; + size_t fixed_barcode_lens[1] = {0}; + const size_t *barcode_lens = NULL; + size_t barcode_lens_count = 0; + FILE **target_files = NULL; + FILE *assignments = NULL; + FILE *ambiguous_out = NULL; + FILE *unmatched_out = NULL; + count_stats stats = {0}; + unsigned long long *target_counts = NULL; + const char *assignment_engine = "generic_indexed"; + int rc = 1; + + if (read_target_table(barcodes_path, &targets) != 0) { + fprintf(stderr, "failed to read barcodes\n"); + goto done; + } + int barcode_id_check = validate_unique_seq_ids(&targets, "barcode"); + if (barcode_id_check != 0) { + if (barcode_id_check == -1) fprintf(stderr, "out of memory\n"); + goto done; + } + if (!auto_barcode_len && metric == COUNT_METRIC_HAMMING && !all_targets_have_length(&targets, barcode_len)) { + fprintf(stderr, "--metric hamming requires every barcode to have --barcode-length bases\n"); + goto done; + } + if (auto_barcode_len) { + if (collect_target_lengths(&targets, &auto_barcode_lens, &auto_barcode_lens_count) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + barcode_lens = auto_barcode_lens; + barcode_lens_count = auto_barcode_lens_count; + } else { + fixed_barcode_lens[0] = barcode_len; + barcode_lens = fixed_barcode_lens; + barcode_lens_count = 1; + } + if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + index = qdaln_index_build(target_ptrs, target_lens, targets.count); + if (index == NULL) { + fprintf(stderr, "failed to build barcode index\n"); + goto done; + } + if (!auto_barcode_len && metric == COUNT_METRIC_HAMMING && indel_window == 0 && (k == 0 || k == 1) && + barcode_len <= 32) { + int lookup_rc = k == 0 ? build_hamming_exact_lookup(&targets, barcode_len, &hlookup) + : build_hamming_lookup(&targets, barcode_len, &hlookup); + if (lookup_rc < 0) { + fprintf(stderr, "failed to build barcode Hamming lookup\n"); + goto done; + } + if (hlookup.ready) assignment_engine = k == 0 ? "hamming_exact_lookup_direct" : "hamming_k1_lookup_direct"; + } + if (!auto_barcode_len && metric == COUNT_METRIC_LEVENSHTEIN && indel_window == 1 && k == 1 && + assignment_policy == AMBIGUITY_POLICY_BEST && barcode_len <= 31) { + int lookup_rc = build_levenshtein1_lookup(&targets, barcode_len, &levlookup); + if (lookup_rc < 0) { + fprintf(stderr, "failed to build barcode Levenshtein lookup\n"); + goto done; + } + if (levlookup.ready) assignment_engine = "levenshtein_k1_lookup_direct"; + } + int filename_check = validate_unique_sanitized_filenames(&targets); + if (filename_check != 0) { + if (filename_check == -1) fprintf(stderr, "out of memory\n"); + goto done; + } + if (ensure_dir(out_dir) != 0) { + fprintf(stderr, "failed to create output directory\n"); + goto done; + } + target_files = (FILE **)calloc(targets.count == 0 ? 1 : targets.count, sizeof(FILE *)); + target_counts = (unsigned long long *)calloc(targets.count == 0 ? 1 : targets.count, sizeof(unsigned long long)); + if (target_files == NULL || target_counts == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + if (fastq_reader_open(&reader, reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + goto done; + } + if (assignments_path != NULL) { + assignments = open_output_file(assignments_path); + if (assignments == NULL) { + fprintf(stderr, "failed to open assignments output\n"); + goto done; + } + fprintf(assignments, "read_id\tobserved_barcode\ttarget_index\ttarget_id\ttarget_seq\tbest_distance\tsecond_best_distance\tmatch_count\tstatus\n"); + } + if (ambiguous_path != NULL) { + ambiguous_out = open_output_file(ambiguous_path); + if (ambiguous_out == NULL) { + fprintf(stderr, "failed to open ambiguous FASTQ output\n"); + goto done; + } + } + if (unmatched_path != NULL) { + unmatched_out = open_output_file(unmatched_path); + if (unmatched_out == NULL) { + fprintf(stderr, "failed to open unmatched FASTQ output\n"); + goto done; + } + } + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + char read_id[8192]; + char observed[8192]; + int got = 0; + size_t seq_len = 0; + while ((got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { + fastq_read_id(header, read_id, sizeof(read_id)); + qdaln_match_result result = {-1, -1, -1, 0, QDALN_MATCH_INVALID}; + qdaln_index_stats istats = {0, 0}; + observed[0] = '\0'; + ++stats.total; + + int handled = 0; + if (hlookup.ready) { + int exact_merge = assignment_policy == AMBIGUITY_POLICY_RADIUS || + assignments != NULL || ambiguous_out != NULL || unmatched_out != NULL; + int lookup_rc = assign_hamming_lookup_offsets(&hlookup, seq, seq_len, NULL, barcode_start, k, + &result, &istats, observed, sizeof(observed), + exact_merge); + if (lookup_rc < 0) { + fprintf(stderr, "FASTQ assignment failed\n"); + goto done; + } + handled = lookup_rc; + } + if (!handled && levlookup.ready) { + int lookup_rc = assign_levenshtein1_lookup_offset(&levlookup, seq, seq_len, barcode_start, &result, + &istats, observed, sizeof(observed)); + if (lookup_rc < 0) { + fprintf(stderr, "FASTQ assignment failed\n"); + goto done; + } + handled = lookup_rc; + } + if (!handled && + assign_count_length_set(index, seq, seq_len, barcode_start, barcode_lens, barcode_lens_count, k, metric, + indel_window, &result, &istats, observed, sizeof(observed), 0) != 0) { + fprintf(stderr, "FASTQ assignment failed\n"); + goto done; + } + apply_ambiguity_policy(&result, assignment_policy); + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0 && result.best_distance > 0) { + seq_record *target = &targets.records[result.target_index]; + offset_list barcode_offset = {0}; + if (push_offset_unique(&barcode_offset, barcode_start) != 0) { + free_offset_list(&barcode_offset); + fprintf(stderr, "out of memory\n"); + goto done; + } + if (!quality_allows_unique_correction(seq, seq_len, qual, &barcode_offset, barcode_start, + target->len, metric, indel_window, k, observed, target, result, + max_correction_qual)) { + result = (qdaln_match_result){-1, -1, -1, 0, QDALN_MATCH_NONE}; + } + free_offset_list(&barcode_offset); + } + if (result.status != QDALN_MATCH_INVALID) { + stats.candidates_considered += (unsigned long long)istats.candidates_considered; + stats.candidates_verified += (unsigned long long)istats.candidates_verified; + } + + if (assignments != NULL) print_fastq_row(assignments, &targets, read_id, observed, result); + + if (result.status == QDALN_MATCH_UNIQUE && result.target_index >= 0) { + FILE *target_out = open_demux_target_file(target_files, &targets, (size_t)result.target_index, out_dir); + if (target_out == NULL) { + fprintf(stderr, "failed to open per-barcode FASTQ output\n"); + goto done; + } + write_fastq_record(target_out, header, seq, plus, qual); + ++target_counts[result.target_index]; + ++stats.unique; + if (result.best_distance == 0) ++stats.exact; + else ++stats.corrected; + } else if (result.status == QDALN_MATCH_AMBIGUOUS) { + ++stats.ambiguous; + if (ambiguous_out != NULL) write_fastq_record(ambiguous_out, header, seq, plus, qual); + } else if (result.status == QDALN_MATCH_NONE) { + ++stats.unmatched; + if (unmatched_out != NULL) write_fastq_record(unmatched_out, header, seq, plus, qual); + } else { + ++stats.invalid; + if (unmatched_out != NULL) write_fastq_record(unmatched_out, header, seq, plus, qual); + } + } + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + goto done; + } + + if (summary_path != NULL) { + FILE *summary = open_output_file(summary_path); + if (summary == NULL) { + fprintf(stderr, "failed to open summary output\n"); + goto done; + } + unsigned long long top_count = 0; + size_t top_target = 0; + size_t nonempty = 0; + for (size_t i = 0; i < targets.count; ++i) { + if (target_counts[i] != 0) ++nonempty; + if (target_counts[i] > top_count) { + top_count = target_counts[i]; + top_target = i; + } + } + fprintf(summary, + "{\n \"workflow\": \"demux\",\n \"k\": %d,\n \"metric\": \"%s\",\n \"ambiguity_policy\": \"%s\",\n \"assignment_engine\": \"%s\",\n \"alphabet_policy\": \"%s\",\n \"max_correction_qual\": ", + k, metric_name(metric), ambiguity_policy_name(assignment_policy), assignment_engine, + qdaln_alphabet_policy()); + if (max_correction_qual >= 0) { + fprintf(summary, "%d", max_correction_qual); + } else { + fprintf(summary, "null"); + } + fprintf(summary, + ",\n \"indel_window\": %zu,\n \"barcode_start\": %zu,\n \"barcode_length\": %zu,\n \"barcode_length_mode\": \"%s\",\n \"barcode_lengths\": [", + indel_window, barcode_start, barcode_len, auto_barcode_len ? "auto" : "fixed"); + for (size_t i = 0; i < barcode_lens_count; ++i) { + fprintf(summary, "%s%zu", i == 0 ? "" : ", ", barcode_lens[i]); + } + fprintf(summary, + "],\n \"n_barcodes\": %zu,\n \"total_reads\": %llu,\n \"assigned_unique\": %llu,\n \"assigned_exact\": %llu,\n \"assigned_corrected\": %llu,\n \"ambiguous\": %llu,\n \"unmatched\": %llu,\n \"invalid\": %llu,\n \"nonempty_outputs\": %zu,\n \"top_barcode_id\": \"%s\",\n \"top_barcode_count\": %llu,\n \"candidates_considered\": %llu,\n \"candidates_verified\": %llu\n}\n", + targets.count, stats.total, + stats.unique, stats.exact, stats.corrected, stats.ambiguous, stats.unmatched, stats.invalid, + nonempty, targets.count == 0 ? "" : targets.records[top_target].id, top_count, + stats.candidates_considered, stats.candidates_verified); + fclose(summary); + } + + rc = 0; + +done: + if (target_files != NULL) { + for (size_t i = 0; i < targets.count; ++i) { + if (target_files[i] != NULL) fclose(target_files[i]); + } + } + if (assignments != NULL) fclose(assignments); + if (ambiguous_out != NULL) fclose(ambiguous_out); + if (unmatched_out != NULL) fclose(unmatched_out); + fastq_reader_close(&reader); + qdaln_index_free(index); + free_hamming_lookup(&hlookup); + free_levenshtein1_lookup(&levlookup); + free(target_ptrs); + free(target_lens); + free(auto_barcode_lens); + free(target_files); + free(target_counts); + free_table(&targets); + return rc; +} + +typedef struct bcl_sample { + char *id; + char *name; + char *index1; + char *index2; + int lane; + size_t output_index; + int is_alias; + unsigned long long assigned; +} bcl_sample; + +typedef struct bcl_sample_table { + bcl_sample *items; + size_t count; + size_t cap; +} bcl_sample_table; + +typedef struct bcl_read_info { + int number; + size_t cycles; + int indexed; + size_t start_cycle; +} bcl_read_info; + +typedef struct bcl_run_info { + bcl_read_info reads[16]; + size_t read_count; + size_t total_cycles; +} bcl_run_info; + +typedef struct bcl_unknown_barcode { + char *index; + unsigned long long count; +} bcl_unknown_barcode; + +typedef struct bcl_unknown_table { + bcl_unknown_barcode *items; + size_t count; + size_t cap; +} bcl_unknown_table; + +typedef struct text_buffer { + char *data; + size_t len; + size_t cap; +} text_buffer; + +static void free_bcl_unknowns(bcl_unknown_table *table) { + for (size_t i = 0; i < table->count; ++i) free(table->items[i].index); + free(table->items); + table->items = NULL; + table->count = 0; + table->cap = 0; +} + +static int add_bcl_unknown_count(bcl_unknown_table *table, const char *index, unsigned long long count) { + if (count == 0) return 0; + for (size_t i = 0; i < table->count; ++i) { + if (strcmp(table->items[i].index, index) == 0) { + table->items[i].count += count; + return 0; + } + } + if (table->count == table->cap) { + size_t next_cap = table->cap == 0 ? 64 : table->cap * 2; + bcl_unknown_barcode *next = (bcl_unknown_barcode *)realloc(table->items, next_cap * sizeof(bcl_unknown_barcode)); + if (next == NULL) return -1; + table->items = next; + table->cap = next_cap; + } + table->items[table->count].index = xstrndup(index, strlen(index)); + if (table->items[table->count].index == NULL) return -1; + table->items[table->count].count = count; + ++table->count; + return 0; +} + +static int add_bcl_unknown(bcl_unknown_table *table, const char *index) { + return add_bcl_unknown_count(table, index, 1); +} + +static int cmp_bcl_unknown_desc(const void *a, const void *b) { + const bcl_unknown_barcode *aa = (const bcl_unknown_barcode *)a; + const bcl_unknown_barcode *bb = (const bcl_unknown_barcode *)b; + if (aa->count < bb->count) return 1; + if (aa->count > bb->count) return -1; + return strcmp(aa->index, bb->index); +} + +static int merge_bcl_unknowns(bcl_unknown_table *dst, const bcl_unknown_table *src) { + for (size_t i = 0; i < src->count; ++i) { + if (add_bcl_unknown_count(dst, src->items[i].index, src->items[i].count) != 0) return -1; + } + return 0; +} + +static void free_text_buffer(text_buffer *buf) { + free(buf->data); + buf->data = NULL; + buf->len = 0; + buf->cap = 0; +} + +static int text_buffer_reserve(text_buffer *buf, size_t extra) { + if (extra > SIZE_MAX - buf->len) return -1; + size_t need = buf->len + extra; + if (need <= buf->cap) return 0; + size_t next = buf->cap == 0 ? 65536 : buf->cap; + while (next < need) { + if (next > SIZE_MAX / 2) { + next = need; + break; + } + next *= 2; + } + char *p = (char *)realloc(buf->data, next); + if (p == NULL) return -1; + buf->data = p; + buf->cap = next; + return 0; +} + +static int text_buffer_append(text_buffer *buf, const char *s, size_t n) { + if (text_buffer_reserve(buf, n) != 0) return -1; + memcpy(buf->data + buf->len, s, n); + buf->len += n; + return 0; +} + +static int gzwrite_all(gzFile gz, const char *data, size_t len) { + size_t written = 0; + while (written < len) { + size_t remaining = len - written; + unsigned int chunk = remaining > (size_t)UINT_MAX ? UINT_MAX : (unsigned int)remaining; + int rc = gzwrite(gz, data + written, chunk); + if (rc <= 0 || (unsigned int)rc != chunk) return -1; + written += (size_t)rc; + } + return 0; +} + +static void free_bcl_samples(bcl_sample_table *table) { + for (size_t i = 0; i < table->count; ++i) { + free(table->items[i].id); + free(table->items[i].name); + free(table->items[i].index1); + free(table->items[i].index2); + } + free(table->items); + table->items = NULL; + table->count = 0; + table->cap = 0; +} + +static int push_bcl_sample(bcl_sample_table *table, const char *id, const char *name, + const char *index1, const char *index2, int lane) { + size_t output_index = table->count; + int is_alias = 0; + for (size_t i = 0; i < table->count; ++i) { + bcl_sample *existing = &table->items[i]; + if (strcmp(existing->id, id) == 0 && existing->lane == lane) { + output_index = existing->output_index; + is_alias = 1; + break; + } + } + if (table->count == table->cap) { + size_t next_cap = table->cap == 0 ? 16 : table->cap * 2; + bcl_sample *next = (bcl_sample *)realloc(table->items, next_cap * sizeof(bcl_sample)); + if (next == NULL) return -1; + table->items = next; + table->cap = next_cap; + } + bcl_sample *s = &table->items[table->count]; + memset(s, 0, sizeof(*s)); + s->id = xstrndup(id, strlen(id)); + s->name = xstrndup(name != NULL && name[0] != '\0' ? name : id, strlen(name != NULL && name[0] != '\0' ? name : id)); + s->index1 = xstrndup(index1, strlen(index1)); + s->index2 = xstrndup(index2 != NULL ? index2 : "", strlen(index2 != NULL ? index2 : "")); + s->lane = lane; + s->output_index = output_index; + s->is_alias = is_alias; + if (s->id == NULL || s->name == NULL || s->index1 == NULL || s->index2 == NULL) return -1; + uppercase_ascii(s->index1); + uppercase_ascii(s->index2); + ++table->count; + return 0; +} + +static char *read_text_file(const char *path) { + FILE *fp = fopen(path, "rb"); + if (fp == NULL) return NULL; + if (fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return NULL; + } + long n = ftell(fp); + if (n < 0) { + fclose(fp); + return NULL; + } + rewind(fp); + char *text = (char *)malloc((size_t)n + 1); + if (text == NULL) { + fclose(fp); + return NULL; + } + if (fread(text, 1, (size_t)n, fp) != (size_t)n) { + free(text); + fclose(fp); + return NULL; + } + text[n] = '\0'; + fclose(fp); + return text; +} + +static int xml_attr_value(const char *tag, const char *name, char *out, size_t out_cap) { + char pattern[64]; + snprintf(pattern, sizeof(pattern), "%s=\"", name); + const char *p = strstr(tag, pattern); + if (p == NULL) return -1; + p += strlen(pattern); + const char *end = strchr(p, '"'); + if (end == NULL) return -1; + size_t n = (size_t)(end - p); + if (n >= out_cap) n = out_cap - 1; + memcpy(out, p, n); + out[n] = '\0'; + return 0; +} + +static int parse_run_info(const char *run_folder, bcl_run_info *info) { + char path[4096]; + if (path_join(path, sizeof(path), run_folder, "RunInfo.xml") != 0) return -1; + char *xml = read_text_file(path); + if (xml == NULL) return -1; + memset(info, 0, sizeof(*info)); + const char *p = xml; + while ((p = strstr(p, "'); + if (end == NULL) { + free(xml); + return -1; + } + char tag[1024]; + size_t tag_len = (size_t)(end - p + 1); + if (tag_len >= sizeof(tag)) tag_len = sizeof(tag) - 1; + memcpy(tag, p, tag_len); + tag[tag_len] = '\0'; + char number[32] = ""; + char cycles[32] = ""; + char indexed[32] = ""; + if (xml_attr_value(tag, "Number", number, sizeof(number)) == 0 && + xml_attr_value(tag, "NumCycles", cycles, sizeof(cycles)) == 0 && + xml_attr_value(tag, "IsIndexedRead", indexed, sizeof(indexed)) == 0) { + if (info->read_count >= 16) { + free(xml); + return -1; + } + int parsed_number = 0; + size_t parsed_cycles = 0; + if (parse_int_value(number, &parsed_number) != 0 || parsed_number < 1 || + parse_size_value(cycles, &parsed_cycles) != 0 || parsed_cycles == 0 || + parsed_cycles > MAX_BCL_READ_CYCLES || + info->total_cycles > MAX_BCL_TOTAL_CYCLES - parsed_cycles) { + free(xml); + return -1; + } + if (!(indexed[0] == 'Y' || indexed[0] == 'y' || indexed[0] == 'N' || indexed[0] == 'n') || + indexed[1] != '\0') { + free(xml); + return -1; + } + bcl_read_info *r = &info->reads[info->read_count++]; + r->number = parsed_number; + r->cycles = parsed_cycles; + r->indexed = indexed[0] == 'Y' || indexed[0] == 'y'; + r->start_cycle = info->total_cycles + 1; + info->total_cycles += r->cycles; + } + p = end + 1; + } + free(xml); + return info->read_count == 0 ? -1 : 0; +} + +static int read_bcl_sample_sheet(const char *path, bcl_sample_table *samples) { + FILE *fp = fopen(path, "r"); + if (fp == NULL) return -1; + char buf[16384]; + int in_data = 0; + int have_header = 0; + int id_col = -1; + int name_col = -1; + int index_col = -1; + int index2_col = -1; + int lane_col = -1; + size_t row = 0; + while (fgets(buf, sizeof(buf), fp) != NULL) { + ++row; + trim_line(buf); + if (buf[0] == '\0') continue; + if (buf[0] == '[') { + in_data = field_eq(buf, "[Data]") || field_eq(buf, "[BCLConvert_Data]"); + have_header = 0; + continue; + } + if (!in_data) continue; + char *fields[64]; + size_t nf = split_fields(buf, ',', fields, 64); + if (!have_header) { + id_col = find_column(fields, nf, "Sample_ID", "SampleID", "Sample_ID"); + if (id_col < 0) id_col = find_column(fields, nf, "Sample_Project", "SampleName", "Sample_Name"); + name_col = find_column(fields, nf, "Sample_Name", "SampleName", "sample_name"); + index_col = find_column(fields, nf, "index", "Index", "Index1"); + index2_col = find_column(fields, nf, "index2", "Index2", "index_2"); + lane_col = find_column(fields, nf, "Lane", "lane", NULL); + have_header = 1; + if (id_col < 0 || index_col < 0) { + fclose(fp); + return -1; + } + continue; + } + if ((size_t)id_col >= nf || (size_t)index_col >= nf || fields[id_col][0] == '\0' || fields[index_col][0] == '\0') { + fprintf(stderr, "%s:%zu: BCL sample sheet Sample_ID and index must be non-empty\n", path, row); + fclose(fp); + return -1; + } + const char *name = (name_col >= 0 && (size_t)name_col < nf) ? fields[name_col] : fields[id_col]; + const char *index2 = (index2_col >= 0 && (size_t)index2_col < nf) ? fields[index2_col] : ""; + int lane = 0; + if (lane_col >= 0 && (size_t)lane_col < nf && fields[lane_col][0] != '\0') { + if (parse_int_value(fields[lane_col], &lane) != 0 || lane < 1) { + fprintf(stderr, "%s:%zu: BCL sample sheet Lane must be a positive integer\n", path, row); + fclose(fp); + return -1; + } + if (lane != 1) { + fprintf(stderr, "%s:%zu: classic BCL demux currently supports sample-sheet Lane 1 only\n", path, row); + fclose(fp); + return -1; + } + } + if (push_bcl_sample(samples, fields[id_col], name, fields[index_col], index2, lane) != 0) { + fclose(fp); + return -1; + } + } + fclose(fp); + return samples->count == 0 ? -1 : 0; +} + +static int hamming_distance_limit(const char *a, const char *b, int limit) { + size_t na = strlen(a); + size_t nb = strlen(b); + if (na != nb) return limit + 1; + int d = 0; + for (size_t i = 0; i < na; ++i) { + if (a[i] != b[i] && ++d > limit) return d; + } + return d; +} + +static int assign_bcl_sample(const bcl_sample_table *samples, int lane, const char *index1, const char *index2, + int k1, int k2, int *match_count_out) { + int best = -1; + int best_d = 1000000; + size_t best_output = (size_t)-1; + int matches = 0; + for (size_t i = 0; i < samples->count; ++i) { + const bcl_sample *s = &samples->items[i]; + if (s->lane != 0 && s->lane != lane) continue; + int d1 = hamming_distance_limit(index1, s->index1, k1); + if (d1 > k1) continue; + int d2 = 0; + if (s->index2[0] != '\0' || index2[0] != '\0') { + d2 = hamming_distance_limit(index2, s->index2, k2); + if (d2 > k2) continue; + } + int d = d1 + d2; + if (d < best_d) { + best_d = d; + best = (int)i; + best_output = s->output_index; + matches = 1; + } else if (d == best_d && s->output_index != best_output) { + ++matches; + } + } + *match_count_out = matches; + return matches == 1 ? best : -1; +} + +static int read_u32_le(const unsigned char *p) { + return (int)((unsigned int)p[0] | ((unsigned int)p[1] << 8) | ((unsigned int)p[2] << 16) | ((unsigned int)p[3] << 24)); +} + +static int path_exists(const char *path) { + struct stat st; + return stat(path, &st) == 0; +} + +static int build_bcl_path(char *out, size_t out_cap, const char *basecalls, int lane, size_t cycle, const char *tile) { + int n = snprintf(out, out_cap, "%s/L%03d/C%zu.1/s_%d_%s.bcl.gz", basecalls, lane, cycle, lane, tile); + if (n < 0 || (size_t)n >= out_cap) return -1; + if (path_exists(out)) return 0; + n = snprintf(out, out_cap, "%s/L%03d/C%zu.1/s_%d_%s.bcl", basecalls, lane, cycle, lane, tile); + if (n < 0 || (size_t)n >= out_cap) return -1; + return path_exists(out) ? 0 : -1; +} + +static int read_bcl_cycle(const char *path, unsigned char **bytes_out, size_t *count_out) { + gzFile gz = gzopen(path, "rb"); + if (gz == NULL) return -1; + unsigned char header[4]; + if (gzread(gz, header, 4) != 4) { + gzclose(gz); + return -1; + } + int n = read_u32_le(header); + if (n < 0 || n > MAX_BCL_CYCLE_CLUSTERS) { + gzclose(gz); + return -1; + } + unsigned char *bytes = (unsigned char *)malloc((size_t)n == 0 ? 1 : (size_t)n); + if (bytes == NULL) { + gzclose(gz); + return -1; + } + if (n != 0 && gzread(gz, bytes, (unsigned int)n) != n) { + free(bytes); + gzclose(gz); + return -1; + } + gzclose(gz); + *bytes_out = bytes; + *count_out = (size_t)n; + return 0; +} + +static int read_filter_file(const char *basecalls, int lane, const char *tile, unsigned char **pf_out, size_t *count_out) { + char path[4096]; + int n = snprintf(path, sizeof(path), "%s/L%03d/s_%d_%s.filter", basecalls, lane, lane, tile); + if (n < 0 || (size_t)n >= sizeof(path)) return -1; + if (!path_exists(path)) { + *pf_out = NULL; + *count_out = 0; + return 0; + } + FILE *fp = fopen(path, "rb"); + if (fp == NULL) return -1; + unsigned char header[8]; + if (fread(header, 1, 8, fp) != 8) { + fclose(fp); + return -1; + } + int count = read_u32_le(header + 4); + if (count < 0 || count > MAX_BCL_CYCLE_CLUSTERS) { + fclose(fp); + return -1; + } + unsigned char *pf = (unsigned char *)malloc((size_t)count == 0 ? 1 : (size_t)count); + if (pf == NULL) { + fclose(fp); + return -1; + } + if (count != 0 && fread(pf, 1, (size_t)count, fp) != (size_t)count) { + free(pf); + fclose(fp); + return -1; + } + fclose(fp); + *pf_out = pf; + *count_out = (size_t)count; + return 0; +} + +static char bcl_base(unsigned char b) { + if (b == 0) return 'N'; + switch (b & 3u) { + case 0: return 'A'; + case 1: return 'C'; + case 2: return 'G'; + default: return 'T'; + } +} + +static char bcl_qual(unsigned char b) { + if (b == 0) return '#'; + unsigned int q = b >> 2; + if (q > 93) q = 93; + return (char)(q + 33); +} + +static int collect_tiles(const char *basecalls, int lane, char ***tiles_out, size_t *tile_count_out) { + char path[4096]; + int n = snprintf(path, sizeof(path), "%s/L%03d/C1.1", basecalls, lane); + if (n < 0 || (size_t)n >= sizeof(path)) return -1; + DIR *dir = opendir(path); + if (dir == NULL) return -1; + string_list tiles = {0}; + struct dirent *ent; + char prefix[32]; + n = snprintf(prefix, sizeof(prefix), "s_%d_", lane); + if (n < 0 || (size_t)n >= sizeof(prefix)) { + closedir(dir); + return -1; + } + while ((ent = readdir(dir)) != NULL) { + if (strncmp(ent->d_name, prefix, strlen(prefix)) != 0) continue; + char *start = ent->d_name + strlen(prefix); + char *bcl = strstr(start, ".bcl"); + if (bcl == NULL) continue; + size_t len = (size_t)(bcl - start); + char tile[128]; + if (len == 0 || len >= sizeof(tile)) continue; + memcpy(tile, start, len); + tile[len] = '\0'; + if (push_string(&tiles, tile) != 0) { + closedir(dir); + free_string_list(&tiles); + return -1; + } + } + closedir(dir); + *tiles_out = tiles.items; + *tile_count_out = tiles.count; + return tiles.count == 0 ? -1 : 0; +} + +static gzFile open_bcl_fastq(const char *out_dir, const char *sample_id, size_t sample_number, int lane, + char read_kind, int read_number, int gzip_level) { + char safe_id[512]; + sanitize_filename(sample_id, safe_id, sizeof(safe_id)); + char name[700]; + int n = snprintf(name, sizeof(name), "%s_S%zu_L%03d_%c%d_001.fastq.gz", safe_id, sample_number, + lane, read_kind, read_number); + if (n < 0 || (size_t)n >= sizeof(name)) return NULL; + char path[4096]; + if (path_join(path, sizeof(path), out_dir, name) != 0) return NULL; + char mode[8]; + snprintf(mode, sizeof(mode), "wb%d", gzip_level); + gzFile gz = gzopen(path, mode); + if (gz != NULL) gzbuffer(gz, 1024 * 1024); + return gz; +} + +static int bcl_output_enabled(const bcl_read_info *read, int emit_index_fastqs) { + return !read->indexed || emit_index_fastqs; +} + +static char bcl_output_kind(const bcl_read_info *read) { + return read->indexed ? 'I' : 'R'; +} + +static int bcl_output_number(const bcl_run_info *run, size_t read_i) { + int n = 0; + for (size_t i = 0; i <= read_i && i < run->read_count; ++i) { + if (run->reads[i].indexed == run->reads[read_i].indexed) ++n; + } + return n == 0 ? 1 : n; +} + +static gzFile *bcl_output_slot(gzFile *files, size_t sample_i, size_t read_i, size_t read_count) { + return &files[sample_i * read_count + read_i]; +} + +static int append_fastq_record(text_buffer *out, const char *header, const char *seq, const char *qual) { + char buf[20000]; + int n = snprintf(buf, sizeof(buf), "%s\n%s\n+\n%s\n", header, seq, qual); + if (n < 0 || (size_t)n >= sizeof(buf)) return -1; + return text_buffer_append(out, buf, (size_t)n); +} + +typedef struct bcl_block_result { + text_buffer *sample_buffers; + text_buffer *undetermined_buffers; + unsigned long long *sample_assigned; + unsigned long long passed_clusters; + unsigned long long filtered_clusters; + unsigned long long undetermined_reads; + bcl_unknown_table unknowns; + int error; +} bcl_block_result; + +typedef struct bcl_block_job { + const bcl_run_info *run; + const bcl_sample_table *samples; + unsigned char **cycles; + const unsigned char *pf; + const char *tile; + size_t start; + size_t end; + int k1; + int k2; + int emit_index_fastqs; + bcl_block_result *result; +} bcl_block_job; + +static void free_bcl_block_result(bcl_block_result *result, size_t sample_count, size_t read_count) { + if (result->sample_buffers != NULL) { + size_t n = 0; + if (checked_mul_size(sample_count, read_count, &n) != 0) n = 0; + for (size_t i = 0; i < n; ++i) free_text_buffer(&result->sample_buffers[i]); + } + if (result->undetermined_buffers != NULL) { + for (size_t i = 0; i < read_count; ++i) free_text_buffer(&result->undetermined_buffers[i]); + } + free(result->sample_buffers); + free(result->undetermined_buffers); + free(result->sample_assigned); + free_bcl_unknowns(&result->unknowns); + memset(result, 0, sizeof(*result)); +} + +static int init_bcl_block_result(bcl_block_result *result, size_t sample_count, size_t read_count) { + memset(result, 0, sizeof(*result)); + size_t sample_read_slots = 0; + if (checked_mul_size(sample_count, read_count, &sample_read_slots) != 0) return -1; + result->sample_buffers = (text_buffer *)calloc(alloc_count_or_one(sample_read_slots), sizeof(text_buffer)); + result->undetermined_buffers = (text_buffer *)calloc(alloc_count_or_one(read_count), sizeof(text_buffer)); + result->sample_assigned = (unsigned long long *)calloc(alloc_count_or_one(sample_count), sizeof(unsigned long long)); + if (result->sample_buffers == NULL || result->undetermined_buffers == NULL || result->sample_assigned == NULL) { + free_bcl_block_result(result, sample_count, read_count); + return -1; + } + return 0; +} + +static int process_bcl_block(const bcl_block_job *job) { + const bcl_run_info *run = job->run; + const bcl_sample_table *samples = job->samples; + bcl_block_result *result = job->result; + for (size_t cluster = job->start; cluster < job->end; ++cluster) { + if (job->pf != NULL && job->pf[cluster] == 0) { + ++result->filtered_clusters; + continue; + } + ++result->passed_clusters; + char index1[MAX_BCL_READ_CYCLES + 1] = ""; + char index2[MAX_BCL_READ_CYCLES + 1] = ""; + char seqs[16][MAX_BCL_READ_CYCLES + 1]; + char quals[16][MAX_BCL_READ_CYCLES + 1]; + memset(seqs, 0, sizeof(seqs)); + memset(quals, 0, sizeof(quals)); + int indexed_seen = 0; + for (size_t r = 0; r < run->read_count; ++r) { + const bcl_read_info *ri = &run->reads[r]; + char *seq_out = seqs[r]; + char *qual_out = quals[r]; + size_t cap = sizeof(seqs[r]); + for (size_t j = 0; j < ri->cycles && j + 1 < cap; ++j) { + unsigned char b = job->cycles[ri->start_cycle - 1 + j][cluster]; + seq_out[j] = bcl_base(b); + qual_out[j] = bcl_qual(b); + } + seq_out[ri->cycles] = '\0'; + qual_out[ri->cycles] = '\0'; + if (ri->indexed) { + if (indexed_seen == 0) { + int n = snprintf(index1, sizeof(index1), "%s", seq_out); + if (n < 0 || (size_t)n >= sizeof(index1)) return -1; + } else if (indexed_seen == 1) { + int n = snprintf(index2, sizeof(index2), "%s", seq_out); + if (n < 0 || (size_t)n >= sizeof(index2)) return -1; + } + ++indexed_seen; + } + } + int match_count = 0; + int sample_index = assign_bcl_sample(samples, 1, index1, index2, job->k1, job->k2, &match_count); + if (sample_index >= 0) { + const bcl_sample *s = &samples->items[sample_index]; + size_t out_i = s->output_index; + for (size_t r = 0; r < run->read_count; ++r) { + const bcl_read_info *ri = &run->reads[r]; + if (!bcl_output_enabled(ri, job->emit_index_fastqs)) continue; + char header[4096]; + int n = snprintf(header, sizeof(header), "@DOTMATCH:1:%s:%zu %d:N:0:%s%s%s", + job->tile, cluster + 1, bcl_output_number(run, r), + index1, index2[0] ? "+" : "", index2); + if (n < 0 || (size_t)n >= sizeof(header)) return -1; + text_buffer *buf = &result->sample_buffers[out_i * run->read_count + r]; + if (append_fastq_record(buf, header, seqs[r], quals[r]) != 0) return -1; + } + ++result->sample_assigned[out_i]; + } else { + char unknown_index[(MAX_BCL_READ_CYCLES * 2) + 2]; + int n = snprintf(unknown_index, sizeof(unknown_index), "%s%s%s", index1, index2[0] ? "+" : "", index2); + if (n < 0 || (size_t)n >= sizeof(unknown_index)) return -1; + if (add_bcl_unknown(&result->unknowns, unknown_index) != 0) return -1; + for (size_t r = 0; r < run->read_count; ++r) { + const bcl_read_info *ri = &run->reads[r]; + if (!bcl_output_enabled(ri, job->emit_index_fastqs)) continue; + char header[4096]; + n = snprintf(header, sizeof(header), "@DOTMATCH:1:%s:%zu %d:N:0:%s%s%s", + job->tile, cluster + 1, bcl_output_number(run, r), + index1, index2[0] ? "+" : "", index2); + if (n < 0 || (size_t)n >= sizeof(header)) return -1; + if (append_fastq_record(&result->undetermined_buffers[r], header, seqs[r], quals[r]) != 0) return -1; + } + ++result->undetermined_reads; + } + } + return 0; +} + +static void *bcl_block_worker(void *arg) { + bcl_block_job *job = (bcl_block_job *)arg; + job->result->error = process_bcl_block(job); + return NULL; +} + +static int write_bcl_block_result(const bcl_block_result *result, gzFile *sample_fastqs, gzFile *undetermined_fastqs, + size_t sample_count, size_t read_count) { + for (size_t i = 0; i < sample_count; ++i) { + for (size_t r = 0; r < read_count; ++r) { + const text_buffer *buf = &result->sample_buffers[i * read_count + r]; + if (buf->len == 0) continue; + gzFile gz = *bcl_output_slot(sample_fastqs, i, r, read_count); + if (gz != NULL && gzwrite_all(gz, buf->data, buf->len) != 0) return -1; + } + } + for (size_t r = 0; r < read_count; ++r) { + const text_buffer *buf = &result->undetermined_buffers[r]; + if (buf->len == 0) continue; + if (undetermined_fastqs[r] != NULL && gzwrite_all(undetermined_fastqs[r], buf->data, buf->len) != 0) return -1; + } + return 0; +} + +static int parse_mismatches(const char *s, int *k1, int *k2) { + char *comma = strchr(s, ','); + if (comma == NULL) { + int k = 0; + if (parse_int_value(s, &k) != 0) return -1; + if (k < 0 || k > 1) return -1; + *k1 = k; + *k2 = k; + return 0; + } + char left[16]; + size_t n = (size_t)(comma - s); + if (n >= sizeof(left)) return -1; + memcpy(left, s, n); + left[n] = '\0'; + int a = 0; + int b = 0; + if (parse_int_value(left, &a) != 0 || parse_int_value(comma + 1, &b) != 0) return -1; + if (a < 0 || a > 1 || b < 0 || b > 1) return -1; + *k1 = a; + *k2 = b; + return 0; +} + +static int run_bcl_demux(const char *argv0, int argc, char **argv) { + const char *run_folder = NULL; + const char *sample_sheet = NULL; + const char *out_dir = NULL; + const char *summary_path = NULL; + const char *mismatches = "1"; + const char *lanes = "1"; + int k1 = 1; + int k2 = 1; + int emit_index_fastqs = 0; + size_t requested_threads = 0; + int gzip_level = 1; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if (strcmp(arg, "--run-folder") == 0 && i < argc) { + run_folder = argv[i++]; + } else if (strcmp(arg, "--sample-sheet") == 0 && i < argc) { + sample_sheet = argv[i++]; + } else if (strcmp(arg, "--out-dir") == 0 && i < argc) { + out_dir = argv[i++]; + } else if (strcmp(arg, "--summary") == 0 && i < argc) { + summary_path = argv[i++]; + } else if (strcmp(arg, "--barcode-mismatches") == 0 && i < argc) { + mismatches = argv[i++]; + } else if (strcmp(arg, "--emit-index-fastqs") == 0) { + emit_index_fastqs = 1; + } else if (strcmp(arg, "--threads") == 0 && i < argc) { + if (parse_size_value(argv[i++], &requested_threads) != 0) { + fprintf(stderr, "invalid --threads value\n"); + return 2; + } + } else if (strcmp(arg, "--gzip-level") == 0 && i < argc) { + if (parse_int_value(argv[i++], &gzip_level) != 0 || gzip_level < 0 || gzip_level > 9) { + fprintf(stderr, "invalid --gzip-level value\n"); + return 2; + } + } else if (strcmp(arg, "--lanes") == 0 && i < argc) { + lanes = argv[i++]; + } else if (strcmp(arg, "--interop-dir") == 0 && i < argc) { + i++; + } else { + usage(argv0); + return 2; + } + } + if (run_folder == NULL || sample_sheet == NULL || out_dir == NULL || parse_mismatches(mismatches, &k1, &k2) != 0) { + usage(argv0); + return 2; + } + if (strcmp(lanes, "1") != 0 && strcmp(lanes, "001") != 0) { + fprintf(stderr, "classic BCL demux currently supports lane 1 only; rerun with --lanes 1 or split the run externally\n"); + return 2; + } + + char basecalls[4096]; + int basecalls_n = snprintf(basecalls, sizeof(basecalls), "%s/Data/Intensities/BaseCalls", run_folder); + if (basecalls_n < 0 || (size_t)basecalls_n >= sizeof(basecalls)) { + fprintf(stderr, "run folder path is too long\n"); + return 2; + } + bcl_run_info run = {0}; + bcl_sample_table samples = {0}; + gzFile *sample_fastqs = NULL; + gzFile *undetermined_fastqs = NULL; + char **tiles = NULL; + size_t tile_count = 0; + unsigned long long total_clusters = 0; + unsigned long long passed_clusters = 0; + unsigned long long filtered_clusters = 0; + unsigned long long undetermined_reads = 0; + size_t effective_threads = 1; + bcl_unknown_table unknowns = {0}; + int rc = 1; + + if (parse_run_info(run_folder, &run) != 0) { + fprintf(stderr, "failed to parse RunInfo.xml\n"); + goto done; + } + if (read_bcl_sample_sheet(sample_sheet, &samples) != 0) { + fprintf(stderr, "failed to parse sample sheet\n"); + goto done; + } + if (samples.count > MAX_BCL_SAMPLE_ROWS) { + fprintf(stderr, "sample sheet exceeds supported BCL sample count\n"); + goto done; + } + if (requested_threads == 0) { + requested_threads = get_cpu_count(); + } + if (ensure_dir(out_dir) != 0) { + fprintf(stderr, "failed to create BCL output directory\n"); + goto done; + } + if (collect_tiles(basecalls, 1, &tiles, &tile_count) != 0) { + fprintf(stderr, "failed to find classic BCL tiles; CBCL is not supported in this milestone\n"); + goto done; + } + + size_t output_file_count = 0; + if (checked_mul_size(samples.count == 0 ? 1 : samples.count, + run.read_count == 0 ? 1 : run.read_count, + &output_file_count) != 0) { + fprintf(stderr, "BCL output file count overflow\n"); + goto done; + } + sample_fastqs = (gzFile *)calloc(output_file_count, sizeof(gzFile)); + undetermined_fastqs = (gzFile *)calloc(run.read_count == 0 ? 1 : run.read_count, sizeof(gzFile)); + if (sample_fastqs == NULL || undetermined_fastqs == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t i = 0; i < samples.count; ++i) { + if (samples.items[i].is_alias) continue; + for (size_t r = 0; r < run.read_count; ++r) { + bcl_read_info *ri = &run.reads[r]; + if (!bcl_output_enabled(ri, emit_index_fastqs)) continue; + gzFile *slot = bcl_output_slot(sample_fastqs, i, r, run.read_count); + *slot = open_bcl_fastq(out_dir, samples.items[i].id, i + 1, 1, bcl_output_kind(ri), + bcl_output_number(&run, r), gzip_level); + if (*slot == NULL) { + fprintf(stderr, "failed to open sample FASTQ\n"); + goto done; + } + } + } + for (size_t r = 0; r < run.read_count; ++r) { + bcl_read_info *ri = &run.reads[r]; + if (!bcl_output_enabled(ri, emit_index_fastqs)) continue; + undetermined_fastqs[r] = open_bcl_fastq(out_dir, "Undetermined", 0, 1, bcl_output_kind(ri), + bcl_output_number(&run, r), gzip_level); + if (undetermined_fastqs[r] == NULL) { + fprintf(stderr, "failed to open undetermined FASTQ\n"); + goto done; + } + } + + for (size_t tile_i = 0; tile_i < tile_count; ++tile_i) { + unsigned char **cycles = (unsigned char **)calloc(run.total_cycles == 0 ? 1 : run.total_cycles, sizeof(unsigned char *)); + size_t cluster_count = 0; + if (cycles == NULL) { + fprintf(stderr, "out of memory\n"); + goto done; + } + for (size_t c = 1; c <= run.total_cycles; ++c) { + char bcl_path[4096]; + size_t n = 0; + if (build_bcl_path(bcl_path, sizeof(bcl_path), basecalls, 1, c, tiles[tile_i]) != 0 || + read_bcl_cycle(bcl_path, &cycles[c - 1], &n) != 0) { + fprintf(stderr, "failed to read BCL cycle\n"); + for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); + free(cycles); + goto done; + } + if (c == 1) cluster_count = n; + else if (n != cluster_count) { + fprintf(stderr, "BCL cycle cluster counts do not match\n"); + for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); + free(cycles); + goto done; + } + } + unsigned char *pf = NULL; + size_t pf_count = 0; + if (read_filter_file(basecalls, 1, tiles[tile_i], &pf, &pf_count) != 0) { + fprintf(stderr, "failed to read filter file\n"); + for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); + free(cycles); + goto done; + } + if (pf != NULL && pf_count != cluster_count) { + fprintf(stderr, "filter cluster count does not match BCL\n"); + free(pf); + for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); + free(cycles); + goto done; + } + + total_clusters += cluster_count; + size_t threads = requested_threads; + if (threads > cluster_count) threads = cluster_count == 0 ? 1 : cluster_count; + if (threads > effective_threads) effective_threads = threads; + const size_t block_size = 8192; + for (size_t block_start = 0; block_start < cluster_count;) { + size_t batch = 0; + pthread_t *thread_ids = NULL; + bcl_block_job *jobs = (bcl_block_job *)calloc(threads, sizeof(bcl_block_job)); + bcl_block_result *results = (bcl_block_result *)calloc(threads, sizeof(bcl_block_result)); + if (jobs == NULL || results == NULL) { + free(jobs); + free(results); + fprintf(stderr, "out of memory\n"); + goto done; + } + if (threads > 1) { + thread_ids = (pthread_t *)calloc(threads, sizeof(pthread_t)); + if (thread_ids == NULL) { + free(jobs); + free(results); + fprintf(stderr, "out of memory\n"); + goto done; + } + } + while (batch < threads && block_start < cluster_count) { + size_t block_end = block_start + block_size; + if (block_end > cluster_count) block_end = cluster_count; + if (init_bcl_block_result(&results[batch], samples.count, run.read_count) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + jobs[batch].run = &run; + jobs[batch].samples = &samples; + jobs[batch].cycles = cycles; + jobs[batch].pf = pf; + jobs[batch].tile = tiles[tile_i]; + jobs[batch].start = block_start; + jobs[batch].end = block_end; + jobs[batch].k1 = k1; + jobs[batch].k2 = k2; + jobs[batch].emit_index_fastqs = emit_index_fastqs; + jobs[batch].result = &results[batch]; + if (threads > 1) { + if (pthread_create(&thread_ids[batch], NULL, bcl_block_worker, &jobs[batch]) != 0) { + fprintf(stderr, "failed to create BCL worker\n"); + goto done; + } + } else { + results[batch].error = process_bcl_block(&jobs[batch]); + } + ++batch; + block_start = block_end; + } + if (threads > 1) { + for (size_t i = 0; i < batch; ++i) pthread_join(thread_ids[i], NULL); + } + for (size_t i = 0; i < batch; ++i) { + if (results[i].error != 0) { + fprintf(stderr, "failed to format BCL block\n"); + goto done; + } + passed_clusters += results[i].passed_clusters; + filtered_clusters += results[i].filtered_clusters; + undetermined_reads += results[i].undetermined_reads; + for (size_t s = 0; s < samples.count; ++s) samples.items[s].assigned += results[i].sample_assigned[s]; + if (merge_bcl_unknowns(&unknowns, &results[i].unknowns) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + if (write_bcl_block_result(&results[i], sample_fastqs, undetermined_fastqs, samples.count, run.read_count) != 0) { + fprintf(stderr, "failed to write BCL block\n"); + goto done; + } + free_bcl_block_result(&results[i], samples.count, run.read_count); + } + free(thread_ids); + free(jobs); + free(results); + } + free(pf); + for (size_t j = 0; j < run.total_cycles; ++j) free(cycles[j]); + free(cycles); + } + + char stats_path[4096]; + if (path_join(stats_path, sizeof(stats_path), out_dir, "Demultiplex_Stats.csv") != 0) goto done; + FILE *stats = open_output_file(stats_path); + if (stats == NULL) goto done; + fprintf(stats, "sample_id,assigned_reads"); + int non_index_read_count = 0; + for (size_t r = 0; r < run.read_count; ++r) { + if (!run.reads[r].indexed) fprintf(stats, ",read%d_records", ++non_index_read_count); + } + fprintf(stats, "\n"); + unsigned long long assigned_reads = 0; + for (size_t i = 0; i < samples.count; ++i) { + if (samples.items[i].is_alias) continue; + fprintf(stats, "%s,%llu", samples.items[i].id, samples.items[i].assigned); + for (int r = 0; r < non_index_read_count; ++r) fprintf(stats, ",%llu", samples.items[i].assigned); + fprintf(stats, "\n"); + assigned_reads += samples.items[i].assigned; + } + fprintf(stats, "Undetermined,%llu", undetermined_reads); + for (int r = 0; r < non_index_read_count; ++r) fprintf(stats, ",%llu", undetermined_reads); + fprintf(stats, "\n"); + fclose(stats); + + if (unknowns.count > 0) { + qsort(unknowns.items, unknowns.count, sizeof(unknowns.items[0]), cmp_bcl_unknown_desc); + char unknown_path[4096]; + if (path_join(unknown_path, sizeof(unknown_path), out_dir, "Top_Unknown_Barcodes.csv") != 0) goto done; + FILE *unknown = open_output_file(unknown_path); + if (unknown == NULL) goto done; + fprintf(unknown, "index,count\n"); + size_t n = unknowns.count < 100 ? unknowns.count : 100; + for (size_t i = 0; i < n; ++i) fprintf(unknown, "%s,%llu\n", unknowns.items[i].index, unknowns.items[i].count); + fclose(unknown); + } + + char normalized_path[4096]; + if (path_join(normalized_path, sizeof(normalized_path), out_dir, "SampleSheet.normalized.csv") != 0) goto done; + FILE *normalized = open_output_file(normalized_path); + if (normalized != NULL) { + fprintf(normalized, "sample_id,sample_name,lane,index,index2\n"); + for (size_t i = 0; i < samples.count; ++i) { + fprintf(normalized, "%s,%s,%d,%s,%s\n", samples.items[i].id, samples.items[i].name, + samples.items[i].lane, samples.items[i].index1, samples.items[i].index2); + } + fclose(normalized); + } + + if (summary_path != NULL) { + FILE *summary = open_output_file(summary_path); + if (summary == NULL) goto done; + fprintf(summary, + "{\n \"workflow\": \"bcl-demux\",\n \"format\": \"classic_bcl\",\n \"lanes\": 1,\n \"tiles\": %zu,\n \"total_clusters\": %llu,\n \"passed_filter_clusters\": %llu,\n \"filtered_clusters\": %llu,\n \"assigned_reads\": %llu,\n \"undetermined_reads\": %llu,\n \"barcode_mismatches_index1\": %d,\n \"barcode_mismatches_index2\": %d,\n \"requested_threads\": %zu,\n \"effective_threads\": %zu,\n \"gzip_level\": %d,\n \"emit_index_fastqs\": %s\n}\n", + tile_count, total_clusters, passed_clusters, filtered_clusters, assigned_reads, undetermined_reads, + k1, k2, requested_threads, effective_threads, gzip_level, emit_index_fastqs ? "true" : "false"); + fclose(summary); + } + + rc = 0; + +done: + if (sample_fastqs != NULL) { + for (size_t i = 0; i < samples.count; ++i) { + if (samples.items[i].is_alias) continue; + for (size_t r = 0; r < run.read_count; ++r) { + gzFile *slot = bcl_output_slot(sample_fastqs, i, r, run.read_count); + if (*slot != NULL) gzclose(*slot); + } + } + } + if (undetermined_fastqs != NULL) { + for (size_t r = 0; r < run.read_count; ++r) { + if (undetermined_fastqs[r] != NULL) gzclose(undetermined_fastqs[r]); + } + } + if (tiles != NULL) { + for (size_t i = 0; i < tile_count; ++i) free(tiles[i]); + free(tiles); + } + free(sample_fastqs); + free(undetermined_fastqs); + free_bcl_unknowns(&unknowns); + free_bcl_samples(&samples); + return rc; +} + +static int compare_gzip_fastq_files(const char *a_path, const char *b_path, unsigned long long *records_out) { + gzFile a = gzopen(a_path, "rb"); + gzFile b = gzopen(b_path, "rb"); + if (a == NULL || b == NULL) { + if (a != NULL) gzclose(a); + if (b != NULL) gzclose(b); + return -1; + } + char abuf[8192]; + char bbuf[8192]; + unsigned long long lines = 0; + int mismatch = 0; + for (;;) { + char *ag = gzgets(a, abuf, sizeof(abuf)); + char *bg = gzgets(b, bbuf, sizeof(bbuf)); + if (ag == NULL || bg == NULL) { + if (ag != bg) mismatch = 1; + break; + } + if (strcmp(abuf, bbuf) != 0) mismatch = 1; + ++lines; + } + gzclose(a); + gzclose(b); + *records_out = lines / 4; + return mismatch ? 1 : 0; +} + +static int run_bcl_validate(const char *argv0, int argc, char **argv) { + const char *dotmatch_out = NULL; + const char *truth_out = NULL; + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if (strcmp(arg, "--dotmatch-out") == 0 && i < argc) { + dotmatch_out = argv[i++]; + } else if (strcmp(arg, "--truth-out") == 0 && i < argc) { + truth_out = argv[i++]; + } else { + usage(argv0); + return 2; + } + } + if (dotmatch_out == NULL || truth_out == NULL) { + usage(argv0); + return 2; + } + DIR *dir = opendir(truth_out); + if (dir == NULL) { + fprintf(stderr, "failed to open truth output directory\n"); + return 1; + } + unsigned long long compared_files = 0; + unsigned long long compared_records = 0; + unsigned long long missing_files = 0; + unsigned long long mismatched_files = 0; + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + if (!ends_with(ent->d_name, ".fastq.gz")) continue; + char truth_path[4096]; + char dotmatch_path[4096]; + if (path_join(truth_path, sizeof(truth_path), truth_out, ent->d_name) != 0 || + path_join(dotmatch_path, sizeof(dotmatch_path), dotmatch_out, ent->d_name) != 0) { + closedir(dir); + fprintf(stderr, "BCL validation path is too long\n"); + return 1; + } + if (!path_exists(dotmatch_path)) { + ++missing_files; + continue; + } + unsigned long long records = 0; + int cmp = compare_gzip_fastq_files(dotmatch_path, truth_path, &records); + if (cmp != 0) ++mismatched_files; + compared_records += records; + ++compared_files; + } + closedir(dir); + printf("{\n \"compared_fastq_files\": %llu,\n \"compared_records\": %llu,\n \"missing_fastq_files\": %llu,\n \"mismatched_fastq_files\": %llu\n}\n", + compared_files, compared_records, missing_files, mismatched_files); + return missing_files == 0 && mismatched_files == 0 ? 0 : 1; +} + +static int run_edlib_validate_helper(const char *targets_path, const char *reads_path, + size_t target_start, size_t target_len, int k, size_t indel_window, size_t sample_limit, + size_t auto_offset, size_t auto_offset_sample, offset_mode offsets_mode, double offset_min_fraction, + size_t threads) { + const char *helper_path = "./build/dotmatch_edlib_validate"; + if (access(helper_path, X_OK) != 0) { + fprintf(stderr, "edlib oracle validation requires build/dotmatch_edlib_validate; run `make edlib-tools`\n"); + return 2; + } + + char target_start_buf[32]; + char target_len_buf[32]; + char k_buf[32]; + char indel_window_buf[32]; + char sample_buf[32]; + char auto_offset_buf[32]; + char auto_offset_sample_buf[32]; + char threads_buf[32]; + char offset_min_fraction_buf[64]; + snprintf(target_start_buf, sizeof(target_start_buf), "%zu", target_start); + snprintf(target_len_buf, sizeof(target_len_buf), "%zu", target_len); + snprintf(k_buf, sizeof(k_buf), "%d", k); + snprintf(indel_window_buf, sizeof(indel_window_buf), "%zu", indel_window); + snprintf(sample_buf, sizeof(sample_buf), "%zu", sample_limit); + snprintf(auto_offset_buf, sizeof(auto_offset_buf), "%zu", auto_offset); + snprintf(auto_offset_sample_buf, sizeof(auto_offset_sample_buf), "%zu", auto_offset_sample); + snprintf(threads_buf, sizeof(threads_buf), "%zu", threads); + snprintf(offset_min_fraction_buf, sizeof(offset_min_fraction_buf), "%.8f", offset_min_fraction); + + pid_t pid = fork(); + if (pid < 0) { + perror("fork"); + return 1; + } + if (pid == 0) { + execl(helper_path, helper_path, + "--targets", targets_path, + "--reads", reads_path, + "--target-start", target_start_buf, + "--target-length", target_len_buf, + "--k", k_buf, + "--indel-window", indel_window_buf, + "--auto-offset", auto_offset_buf, + "--auto-offset-sample", auto_offset_sample_buf, + "--offset-mode", offset_mode_name(offsets_mode), + "--offset-min-fraction", offset_min_fraction_buf, + "--sample", sample_buf, + "--threads", threads_buf, + (char *)NULL); + perror("execl"); + _exit(127); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + perror("waitpid"); + return 1; + } + if (WIFEXITED(status)) { + return WEXITSTATUS(status); + } + return 1; +} + +static int run_validate(const char *argv0, int argc, char **argv) { + const char *targets_path = NULL; + const char *reads_path = NULL; + const char *oracle = "scan"; + size_t target_start = 0; + size_t target_len = 0; + size_t indel_window = 0; + size_t sample_limit = 100000; + size_t auto_offset = 0; + size_t auto_offset_sample = 1000; + size_t threads = 0; + offset_mode offsets_mode = OFFSET_MODE_BEST; + double offset_min_fraction = 0.005; + count_metric metric = COUNT_METRIC_LEVENSHTEIN; + int k = -1; + + int i = 2; + while (i < argc) { + const char *arg = argv[i++]; + if (strcmp(arg, "--targets") == 0 && i < argc) { + targets_path = argv[i++]; + } else if (strcmp(arg, "--reads") == 0 && i < argc) { + reads_path = argv[i++]; + } else if (strcmp(arg, "--target-start") == 0 && i < argc) { + if (parse_size_value(argv[i++], &target_start) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--target-length") == 0 && i < argc) { + if (parse_size_value(argv[i++], &target_len) != 0 || target_len == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--k") == 0 && i < argc) { + if (parse_int_value(argv[i++], &k) != 0 || (k != 0 && k != 1)) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--indel-window") == 0 && i < argc) { + if (parse_size_value(argv[i++], &indel_window) != 0 || indel_window > 1) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--metric") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "hamming") == 0) { + metric = COUNT_METRIC_HAMMING; + } else if (strcmp(value, "levenshtein") == 0) { + metric = COUNT_METRIC_LEVENSHTEIN; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--auto-offset") == 0 && i < argc) { + if (parse_size_value(argv[i++], &auto_offset) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--auto-offset-sample") == 0 && i < argc) { + if (parse_size_value(argv[i++], &auto_offset_sample) != 0 || auto_offset_sample == 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--offset-mode") == 0 && i < argc) { + const char *value = argv[i++]; + if (strcmp(value, "best") == 0) { + offsets_mode = OFFSET_MODE_BEST; + } else if (strcmp(value, "multi") == 0) { + offsets_mode = OFFSET_MODE_MULTI; + } else { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--offset-min-fraction") == 0 && i < argc) { + if (parse_double_value(argv[i++], &offset_min_fraction) != 0 || + offset_min_fraction < 0.0 || offset_min_fraction > 1.0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--oracle") == 0 && i < argc) { + oracle = argv[i++]; + } else if (strcmp(arg, "--sample") == 0 && i < argc) { + if (parse_size_value(argv[i++], &sample_limit) != 0) { + usage(argv0); + return 2; + } + } else if (strcmp(arg, "--threads") == 0 && i < argc) { + if (parse_size_value(argv[i++], &threads) != 0) { + usage(argv0); + return 2; + } + } else { + usage(argv0); + return 2; + } + } + + if (targets_path == NULL || reads_path == NULL || target_len == 0 || k < 0) { + usage(argv0); + return 2; + } + if (auto_offset > MAX_AUTO_OFFSET) { + fprintf(stderr, "--auto-offset must be <= %d\n", MAX_AUTO_OFFSET); + return 2; + } + if (metric == COUNT_METRIC_HAMMING && indel_window != 0) { + fprintf(stderr, "--indel-window is only valid with --metric levenshtein\n"); + return 2; + } + if (threads == 0) { + threads = get_cpu_count(); + } + if (strcmp(oracle, "edlib") == 0) { + if (metric != COUNT_METRIC_LEVENSHTEIN) { + fprintf(stderr, "--oracle edlib is only valid with --metric levenshtein\n"); + return 2; + } + int status = run_edlib_validate_helper(targets_path, reads_path, target_start, target_len, k, indel_window, + sample_limit, auto_offset, auto_offset_sample, offsets_mode, + offset_min_fraction, threads); + return status == 0 ? 0 : status; + } + if (strcmp(oracle, "scan") != 0) { + usage(argv0); + return 2; + } + + seq_table targets = {0}; + fastq_reader reader = {0}; + qdaln_index *index = NULL; + const char **target_ptrs = NULL; + size_t *target_lens = NULL; + int rc = 1; + size_t checked = 0; + size_t mismatches = 0; + offset_list offsets = {0}; + + if (read_target_table(targets_path, &targets) != 0) { + fprintf(stderr, "failed to read targets\n"); + goto done; + } + if (build_target_arrays(&targets, &target_ptrs, &target_lens) != 0) { + fprintf(stderr, "out of memory\n"); + goto done; + } + index = qdaln_index_build(target_ptrs, target_lens, targets.count); + if (index == NULL) { + fprintf(stderr, "failed to build target index\n"); + goto done; + } + if (fastq_reader_open(&reader, reads_path) != 0) { + fprintf(stderr, "failed to open FASTQ input\n"); + goto done; + } + if (detect_offsets(index, NULL, reads_path, target_start, target_len, auto_offset, auto_offset_sample, + offsets_mode, offset_min_fraction, &offsets) != 0) { + fprintf(stderr, "automatic offset detection failed\n"); + goto done; + } + + char header[8192]; + char seq[8192]; + char plus[8192]; + char qual[8192]; + char observed[8192]; + char scan_observed[8192]; + int got = 0; + size_t seq_len = 0; + while ((sample_limit == 0 || checked < sample_limit) && + (got = fastq_read_record_len(&reader, header, seq, plus, qual, sizeof(header), &seq_len)) == 1) { + qdaln_match_result indexed; + qdaln_match_result scan; + qdaln_index_stats stats; + if (assign_count_offsets(index, seq, seq_len, &offsets, target_start, target_len, k, metric, + indel_window, &indexed, &stats, observed, sizeof(observed), 0) != 0 || + scan_count_offsets(target_ptrs, target_lens, targets.count, seq, seq_len, &offsets, target_start, + target_len, k, metric, indel_window, &scan, scan_observed, + sizeof(scan_observed)) != 0) { + fprintf(stderr, "validation assignment failed\n"); + goto done; + } + if (indexed.target_index != scan.target_index || + indexed.best_distance != scan.best_distance || + indexed.second_best_distance != scan.second_best_distance || + indexed.match_count != scan.match_count || + indexed.status != scan.status) { + ++mismatches; + } + ++checked; + } + if (got < 0) { + fprintf(stderr, "malformed FASTQ input\n"); + goto done; + } + printf("{\n \"oracle\": \"native_scan\",\n \"checked_reads\": %zu,\n \"mismatches\": %zu,\n \"k\": %d,\n \"metric\": \"%s\",\n \"target_start\": %zu,\n \"target_length\": %zu,\n \"offset_mode\": \"%s\",\n \"selected_target_starts\": [", + checked, mismatches, k, metric_name(metric), target_start, target_len, offset_mode_name(offsets_mode)); + for (size_t i = 0; i < offsets.count; ++i) { + if (i != 0) printf(", "); + printf("%zu", offsets.items[i]); + } + printf("]\n}\n"); + rc = mismatches == 0 ? 0 : 1; + +done: + fastq_reader_close(&reader); + qdaln_index_free(index); + free_offset_list(&offsets); + free(target_ptrs); + free(target_lens); + free_table(&targets); + return rc; +} + +int main(int argc, char **argv) { + if (argc < 2) { + usage(argv[0]); + return 2; + } + + if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "help") == 0) { + if (argc != 2) { + usage(argv[0]); + return 2; + } + help_manual(stdout, argv[0]); + return 0; + } + + if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "version") == 0) { + if (argc != 2) { + usage(argv[0]); + return 2; + } + printf("dotmatch %s\n", DOTMATCH_VERSION); + return 0; + } + + if (strcmp(argv[1], "citation") == 0 || strcmp(argv[1], "cite") == 0) { + if (argc != 2) { + usage(argv[0]); + return 2; + } + print_citation(stdout); + return 0; + } + + if (strcmp(argv[1], "dist") == 0) { + if (argc != 4) { + usage(argv[0]); + return 2; + } + int d = qdaln_edit_distance(argv[2], strlen(argv[2]), argv[3], strlen(argv[3])); + if (d < 0) return 1; + printf("%d\n", d); + return 0; + } + + if (strcmp(argv[1], "leq") == 0) { + if (argc != 5) { + usage(argv[0]); + return 2; + } + int k = 0; + if (parse_int_value(argv[2], &k) != 0 || k < 0) { + usage(argv[0]); + return 2; + } + int ok = qdaln_edit_distance_leq(argv[3], strlen(argv[3]), argv[4], strlen(argv[4]), k); + if (ok < 0) return 1; + printf("%s\n", ok ? "true" : "false"); + return 0; + } + + if (strcmp(argv[1], "assign") == 0 || strcmp(argv[1], "match") == 0) { + return run_batch(argv[0], argc, argv, argv[1]); + } + + if (strcmp(argv[1], "fastq-assign") == 0) { + return run_fastq_assign(argv[0], argc, argv); + } + + if (strcmp(argv[1], "pair-count") == 0) { + return run_pair_count(argv[0], argc, argv); + } + + if (strcmp(argv[1], "demux") == 0) { + return run_demux(argv[0], argc, argv); + } + + if (strcmp(argv[1], "bcl-demux") == 0) { + return run_bcl_demux(argv[0], argc, argv); + } + + if (strcmp(argv[1], "bcl-validate") == 0) { + return run_bcl_validate(argv[0], argc, argv); + } + + if (strcmp(argv[1], "count") == 0 || strcmp(argv[1], "crispr-count") == 0) { + if (help_requested(argc, argv)) { + count_help_manual(stdout, argv[0], strcmp(argv[1], "crispr-count") == 0); + return 0; + } + return run_count(argv[0], argc, argv); + } + + if (strcmp(argv[1], "guide-counter") == 0 || strcmp(argv[1], "guide-counter-count") == 0 || + strcmp(argv[1], "guide-count") == 0) { + return run_guide_counter_compatible(argv[0], argc, argv); + } + + if (strcmp(argv[1], "inspect-unmatched") == 0) { + return run_inspect_unmatched(argv[0], argc, argv); + } + + if (strcmp(argv[1], "audit") == 0 || strcmp(argv[1], "audit-targets") == 0) { + if (help_requested(argc, argv)) { + audit_help_manual(stdout, argv[0]); + return 0; + } + return run_audit(argv[0], argc, argv); + } + + if (strcmp(argv[1], "validate") == 0) { + return run_validate(argv[0], argc, argv); + } + + usage(argv[0]); + return 2; +} From 37dfba7c0d3776f13e74256b596c48b9d70b0d12 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Mon, 13 Jul 2026 15:09:28 +0100 Subject: [PATCH 04/93] Include new review assets in docs navigation --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.md b/docs/index.md index 256cabac..aae1ae7a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -96,6 +96,8 @@ pilot-program workflow-integration-kit workflow-integration-roadmap adoption-metrics +adopters/record-template +benchmarks/performance-improvements-2026-07-05 ``` ```{toctree} From 007709644a06090bdfa437089b6b0f8741ccb52d Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:14:15 +0100 Subject: [PATCH 05/93] Add AssayCode platform identity --- python/assaycode/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 python/assaycode/__init__.py diff --git a/python/assaycode/__init__.py b/python/assaycode/__init__.py new file mode 100644 index 00000000..c1004d17 --- /dev/null +++ b/python/assaycode/__init__.py @@ -0,0 +1,23 @@ +"""AssayCode platform identity backed by the DotMatch assignment engine. + +AssayCode is the assay-level product surface. The published dotmatch package, +native library, scientific citation, and compatibility contracts remain the +engine of record. +""" + +from __future__ import annotations + +import dotmatch as engine + +__version__ = engine.__version__ +PLATFORM_NAME = "AssayCode" +SPEC_NAME = "AssayScript" +ENGINE_NAME = "DotMatch" + +__all__ = [ + "ENGINE_NAME", + "PLATFORM_NAME", + "SPEC_NAME", + "__version__", + "engine", +] From fbff021c78dd090dcbc6410f28789254095e390d Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:14:24 +0100 Subject: [PATCH 06/93] Add AssayCode platform identity --- python/assaycode/cli.py | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 python/assaycode/cli.py diff --git a/python/assaycode/cli.py b/python/assaycode/cli.py new file mode 100644 index 00000000..a58ba057 --- /dev/null +++ b/python/assaycode/cli.py @@ -0,0 +1,74 @@ +"""Compatibility-safe AssayCode command surface.""" + +from __future__ import annotations + +import sys +from typing import Sequence + +from dotmatch import __version__ +from dotmatch import cli as _engine_cli + + +_SHORTCUTS = {"new", "infer", "check", "plan", "run", "start"} + + +def print_help() -> None: + print( + f"""AssayCode {__version__} — powered by the DotMatch engine + +Compile, validate, decode, and diagnose sequencing assays built from known +guides, barcodes, primers, feature tags, and panel targets. + +Usage: + assaycode --help + assaycode --version + assaycode check assay.toml + assaycode plan assay.toml + assaycode run assay.toml + assaycode start assay.toml + assaycode assay [options] + assaycode engine [options] + +Assay workflow shortcuts: + new scaffold an AssayScript/AssaySpec project + infer infer a reviewable assay window from reads + check validate a specification and its referenced inputs + plan print the deterministic execution plan + run execute an already-reviewed specification + start check, plan, run, and write reliability artifacts + +Specialized DotMatch workflows remain available unchanged: + assaycode crispr ... + assaycode barcode ... + assaycode panel ... + assaycode count ... + assaycode demux ... + +Compatibility: + The dotmatch executable, Python package, native ABI, output schemas, DOI, and + citation remain authoritative and supported. AssayCode is an additive + assay-level identity, not a fork or destructive package rename. +""" + ) + + +def main(argv: Sequence[str] | None = None) -> int: + raw_args = list(sys.argv[1:] if argv is None else argv) + if not raw_args or raw_args[0] in {"-h", "--help", "help"}: + print_help() + return 0 + if raw_args == ["--version"]: + print(f"assaycode {__version__} (DotMatch engine {__version__})") + return 0 + if raw_args[0] == "engine": + if len(raw_args) == 1: + print("usage: assaycode engine [options]", file=sys.stderr) + return 2 + return _engine_cli.main(raw_args[1:]) + if raw_args[0] in _SHORTCUTS: + return _engine_cli.main(["assay", *raw_args]) + return _engine_cli.main(raw_args) + + +if __name__ == "__main__": + raise SystemExit(main()) From 80abda9a23702aeccef19c14a55b9de98e699108 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:14:28 +0100 Subject: [PATCH 07/93] Add AssayCode compatibility tests --- python/tests/test_assaycode_brand.py | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 python/tests/test_assaycode_brand.py diff --git a/python/tests/test_assaycode_brand.py b/python/tests/test_assaycode_brand.py new file mode 100644 index 00000000..22c74d16 --- /dev/null +++ b/python/tests/test_assaycode_brand.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import assaycode +from assaycode import cli + + +def test_platform_identity_uses_dotmatch_engine() -> None: + assert assaycode.PLATFORM_NAME == "AssayCode" + assert assaycode.SPEC_NAME == "AssayScript" + assert assaycode.ENGINE_NAME == "DotMatch" + assert assaycode.__version__ == assaycode.engine.__version__ + + +def test_assaycode_help_states_compatibility_boundary(capsys) -> None: + assert cli.main(["--help"]) == 0 + output = capsys.readouterr().out + assert "powered by the DotMatch engine" in output + assert "additive assay-level identity" in output + assert "assaycode check assay.toml" in output + + +def test_assaycode_version_names_platform_and_engine(capsys) -> None: + assert cli.main(["--version"]) == 0 + output = capsys.readouterr().out + assert output.startswith(f"assaycode {assaycode.__version__}") + assert f"DotMatch engine {assaycode.__version__}" in output + + +def test_assaycode_shortcut_delegates_to_assay_namespace(monkeypatch) -> None: + calls: list[list[str]] = [] + monkeypatch.setattr(cli._engine_cli, "main", lambda argv: calls.append(list(argv)) or 0) + assert cli.main(["check", "assay.toml"]) == 0 + assert calls == [["assay", "check", "assay.toml"]] + + +def test_assaycode_engine_escape_hatch_delegates_unchanged(monkeypatch) -> None: + calls: list[list[str]] = [] + monkeypatch.setattr(cli._engine_cli, "main", lambda argv: calls.append(list(argv)) or 0) + assert cli.main(["engine", "dist", "ACGT", "AGGT"]) == 0 + assert calls == [["dist", "ACGT", "AGGT"]] + + +def test_assaycode_specialized_namespace_delegates_unchanged(monkeypatch) -> None: + calls: list[list[str]] = [] + monkeypatch.setattr(cli._engine_cli, "main", lambda argv: calls.append(list(argv)) or 0) + assert cli.main(["barcode", "infer", "--help"]) == 0 + assert calls == [["barcode", "infer", "--help"]] From 28973ffef4ce97d2cb6c0695728537fb10d7405e Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:14:33 +0100 Subject: [PATCH 08/93] Add AssayCode platform guide --- docs/assaycode.md | 111 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 docs/assaycode.md diff --git a/docs/assaycode.md b/docs/assaycode.md new file mode 100644 index 00000000..957a2753 --- /dev/null +++ b/docs/assaycode.md @@ -0,0 +1,111 @@ +# AssayCode Platform + +AssayCode is the assay-level product identity built on the DotMatch deterministic +known-target assignment engine. + +> **AssayCode expresses and operates the assay. DotMatch performs and validates +> the sequence assignment.** + +This is an additive brand architecture. It does not rename or replace the +published `dotmatch` package, CLI, native library, output schemas, DOI, or +citation. + +## Product Architecture + +| Surface | Responsibility | +| --- | --- | +| AssayCode | Platform for specifying, validating, running, and diagnosing known-target sequencing assays | +| AssayScript | Human-reviewable declarative assay specification | +| DotMatch Engine | Native indexed assignment, ambiguity accounting, counting, demultiplexing, and validation | +| AssayCode Audit | Pre-run collision, input, and safety checks | +| AssayCode Autopsy | Evidence-backed diagnosis of offsets, unsafe correction, unmatched reads, and other failures | +| AssayCode Design | Barcode-panel design, optimization, simulation, layout, and export | +| AssayCode Watch | Reserved identity for future streaming run intelligence; not a current capability | +| AssayCode Pro | Commercial workbench, registry, signed-report, private-assay, audit-trail, and support boundary | + +Names in this table are architecture and positioning. A surface is not a public +capability until its command, schema, tests, evidence, and release gate exist. + +## Commands + +Installing the Python distribution provides both command identities: + +```bash +dotmatch --version +assaycode --version +``` + +The existing command remains authoritative and fully supported. AssayCode adds +shortcuts for assay-level work: + +```bash +assaycode check assay.toml +assaycode plan assay.toml +assaycode run assay.toml +assaycode start assay.toml +``` + +These are exact convenience routes to the corresponding DotMatch AssaySpec +commands. They do not alter matching semantics or output contracts: + +```text +assaycode check assay.toml + == dotmatch assay check assay.toml +``` + +Specialized namespaces pass through unchanged: + +```bash +assaycode crispr quickstart --library guides.csv --fastq '*.fastq.gz' --out run/ +assaycode barcode autopsy --barcodes barcodes.tsv --reads reads.fastq.gz --out-dir autopsy/ +assaycode panel check barcodes.tsv --k 1 --metric hamming --out-dir panel_check/ +``` + +The explicit engine escape hatch is useful in scripts and documentation: + +```bash +assaycode engine dist ACGT AGGT +assaycode engine validate --targets targets.tsv --reads reads.fastq.gz \ + --target-length 20 +``` + +## Python Identity + +```python +import assaycode + +assert assaycode.PLATFORM_NAME == "AssayCode" +assert assaycode.ENGINE_NAME == "DotMatch" +assert assaycode.__version__ == assaycode.engine.__version__ + +distance = assaycode.engine.distance("ACGT", "AGGT") +``` + +Scientific APIs continue to live under `dotmatch`. The AssayCode namespace +intentionally exposes the engine rather than silently copying its API and +creating two competing contracts. + +## Compatibility Contract + +The transition must preserve all of the following: + +- `pip install dotmatch` and Bioconda package identity; +- `dotmatch` CLI commands and behavior; +- `import dotmatch` Python APIs; +- native C ABI and header/library artifacts; +- output schemas and provenance; +- DOI, CITATION.cff, and release citations; +- legacy `quickdna` compatibility where currently supported. + +AssayCode may become the broader website or commercial identity, but scientific +methods should continue to cite the DotMatch release that performed assignment. + +## Claim Boundary + +AssayCode currently organizes existing assay-level capabilities. It does not yet +claim a general assay compiler, calibrated probabilistic decoder, production +streaming monitor, or universal sequencing platform. + +Those capabilities require implementation plus independent correctness, +calibration, performance, and public-data evidence before their names become +claims. From d6caa9a5312f34f625d9fe616874c4ed9ba9da03 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:15:07 +0100 Subject: [PATCH 09/93] Install AssayCode command with DotMatch --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aca31133..92f54c1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,13 +60,14 @@ dotmatch = "dotmatch.multiqc:DotMatchModule" [tool.setuptools.packages.find] where = ["python"] -include = ["dotmatch*", "quickdna*"] +include = ["assaycode*", "dotmatch*", "quickdna*"] [project.scripts] +assaycode = "assaycode.cli:main" dotmatch = "dotmatch.cli:main" quickdna = "dotmatch.cli:main" [tool.cibuildwheel] build = "cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp39-musllinux_x86_64 cp310-musllinux_x86_64 cp311-musllinux_x86_64 cp312-musllinux_x86_64" skip = "pp*" -test-command = "python -c \"import dotmatch, quickdna; assert dotmatch.distance('ACGT', 'AGGT') == 1; assert quickdna.distance_leq('ACGT', 'AGGT', 1)\" && dotmatch --version && dotmatch dist ACGT AGGT" +test-command = "python -c \"import dotmatch, quickdna; assert dotmatch.distance('ACGT', 'AGGT') == 1; assert quickdna.distance_leq('ACGT', 'AGGT', 1)\" && assaycode --version && dotmatch --version && dotmatch dist ACGT AGGT" From c967444dcf2410009d7dff69051727a23beb2815 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:15:18 +0100 Subject: [PATCH 10/93] Add AssayCode to documentation --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.md b/docs/index.md index aae1ae7a..a1fa123a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,6 +27,7 @@ need their own gates before they are public claims. ## Start Here +- Read [AssayCode Platform](assaycode.md) for the additive platform identity, AssayScript terminology, command shortcuts, and DotMatch compatibility contract. - New users should begin with [Getting Started](getting-started.md). - Use [Command Reference](command-reference.md) when choosing the right namespace or compatibility entrypoint. @@ -64,6 +65,7 @@ people and consumed by workflow systems. :caption: User Guide getting-started +assaycode command-reference tutorials/crispr-count-first-run tutorials/scverse-perturb-seq From f38cd0682edd4933c65088bf7662d414fdb65b46 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:15:38 +0100 Subject: [PATCH 11/93] Position AssayCode as the DotMatch platform --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 20e6cb81..b96ece17 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,29 @@ ![Cinematic DotMatch workflow: sequencing reads flow through a precise known-target matching gate into count matrices, demultiplexed barcode lanes, QC panels, and visible ambiguity diagnostics.](public/dotmatch-header-cinematic.png) +## AssayCode Platform + +**AssayCode is the assay-level platform powered by the DotMatch engine.** It gives +scientists one identity for specifying, validating, running, and diagnosing +sequencing assays built from known guides, barcodes, primers, feature tags, and +panel targets. **AssayScript** is the human-reviewable assay specification; +**DotMatch** remains the native assignment engine, published package, CLI, +scientific citation, and compatibility contract. + +The transition is additive: existing `dotmatch` commands, Python APIs, native +artifacts, output schemas, DOI, and citations remain supported. The same Python +distribution also installs the `assaycode` command: + +```bash +assaycode check assay.toml +assaycode plan assay.toml +assaycode start assay.toml +assaycode engine dist ACGT AGGT +``` + +See the [AssayCode Platform guide](docs/assaycode.md) for the product +architecture, compatibility boundary, and honest current capability limits. + DotMatch counts CRISPR guides, splits inline barcodes, designs barcode panels, and writes QC reports from FASTQ. Use it when you already know the short DNA sequences you expect and need to see which reads matched, which did not, and @@ -275,6 +298,7 @@ native library. ```bash python3 -m pip install dotmatch==0.1.9 +assaycode --version dotmatch --version dotmatch dist ACGT AGGT ``` From 9edbdab9f538e5581cef3390ed5b81d1e043097a Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:17:26 +0100 Subject: [PATCH 12/93] Add AssayScript v2 compiler foundation --- python/dotmatch/assayscript.py | 400 +++++++++++++++++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 python/dotmatch/assayscript.py diff --git a/python/dotmatch/assayscript.py b/python/dotmatch/assayscript.py new file mode 100644 index 00000000..8d1f4f8c --- /dev/null +++ b/python/dotmatch/assayscript.py @@ -0,0 +1,400 @@ +"""AssayScript v2 validation and deterministic compilation. + +The existing AssaySpec schema_version=1 remains supported by dotmatch.assayspec. +This module introduces a multi-read, multi-segment representation without +changing current execution semantics. Compilation produces a portable JSON plan +that records strategy selection, input fingerprints, and safety findings. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover + import tomli as tomllib # type: ignore[no-redef] + + +READS = {"R1", "R2", "I1", "I2"} +ORIENTATIONS = {"forward", "reverse_complement", "auto"} +METRICS = {"hamming", "levenshtein"} +POLICIES = {"radius", "best"} +DNA = frozenset("ACGTN") + + +class AssayScriptError(ValueError): + pass + + +@dataclass(frozen=True) +class Segment: + name: str + read: str + library: Path + length: int + start: int | None + anchor: str | None + after_anchor: int + jitter: int + orientation: str + metric: str + k: int + ambiguity_policy: str + required: bool + + +@dataclass(frozen=True) +class CompiledSegment: + name: str + read: str + library: str + library_sha256: str + target_count: int + target_lengths: list[int] + strategy: str + start: int | None + anchor: str | None + after_anchor: int + jitter: int + orientation: str + metric: str + k: int + ambiguity_policy: str + required: bool + safety_status: str + findings: list[str] + + +@dataclass(frozen=True) +class CompiledAssay: + compiler_schema_version: int + source_schema_version: int + name: str + assay_type: str + source: str + source_sha256: str + segments: list[CompiledSegment] + allowed_combinations: str | None + allowed_combinations_sha256: str | None + execution_order: list[str] + findings: list[str] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def load_and_compile(path: str | Path) -> CompiledAssay: + source = Path(path).expanduser().resolve() + try: + data = tomllib.loads(source.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise AssayScriptError(f"AssayScript does not exist: {source}") from exc + except tomllib.TOMLDecodeError as exc: + raise AssayScriptError(f"{source}: invalid TOML: {exc}") from exc + if not isinstance(data, dict): + raise AssayScriptError("AssayScript top level must be a table") + return compile_assayscript(data, source=source) + + +def compile_assayscript(data: Mapping[str, Any], *, source: Path) -> CompiledAssay: + if data.get("schema_version") != 2: + raise AssayScriptError("schema_version must be 2 for AssayScript") + name = _required_text(data.get("name"), "name") + assay_type = _required_text(data.get("assay_type"), "assay_type") + raw_segments = data.get("segments") + if not isinstance(raw_segments, list) or not raw_segments: + raise AssayScriptError("at least one [[segments]] entry is required") + + segments: list[Segment] = [] + names: set[str] = set() + for index, raw in enumerate(raw_segments): + if not isinstance(raw, dict): + raise AssayScriptError(f"segments[{index}] must be a table") + segment = _parse_segment(raw, source=source, index=index) + if segment.name in names: + raise AssayScriptError(f"duplicate segment name: {segment.name}") + names.add(segment.name) + segments.append(segment) + + constraints = data.get("constraints", {}) + if not isinstance(constraints, dict): + raise AssayScriptError("constraints must be a table") + allowed_path: Path | None = None + if constraints.get("allowed_combinations") is not None: + allowed_path = _resolve_input(source, constraints["allowed_combinations"], "constraints.allowed_combinations") + + compiled_segments = [_compile_segment(segment) for segment in segments] + findings = [finding for segment in compiled_segments for finding in segment.findings] + if allowed_path is not None: + findings.extend(_validate_combinations(allowed_path, names)) + + execution_order = [ + segment.name + for segment in sorted( + compiled_segments, + key=lambda item: ( + item.anchor is not None, + item.jitter, + item.k, + item.target_count, + item.name, + ), + ) + ] + return CompiledAssay( + compiler_schema_version=1, + source_schema_version=2, + name=name, + assay_type=assay_type, + source=str(source), + source_sha256=_sha256(source), + segments=compiled_segments, + allowed_combinations=str(allowed_path) if allowed_path else None, + allowed_combinations_sha256=_sha256(allowed_path) if allowed_path else None, + execution_order=execution_order, + findings=sorted(set(findings)), + ) + + +def write_compiled_plan(plan: CompiledAssay, path: str | Path) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(plan.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + return output + + +def _parse_segment(raw: Mapping[str, Any], *, source: Path, index: int) -> Segment: + prefix = f"segments[{index}]" + name = _required_text(raw.get("name"), f"{prefix}.name") + read = _enum(raw.get("read"), READS, f"{prefix}.read") + library = _resolve_input(source, raw.get("library"), f"{prefix}.library") + length = _integer(raw.get("length"), 1, 512, f"{prefix}.length") + start_value = raw.get("start") + anchor_value = raw.get("anchor") + if (start_value is None) == (anchor_value is None): + raise AssayScriptError(f"{prefix} requires exactly one of start or anchor") + start = None if start_value is None else _integer(start_value, 0, 100000, f"{prefix}.start") + anchor = None + if anchor_value is not None: + anchor = _required_text(anchor_value, f"{prefix}.anchor").upper() + if any(base not in DNA for base in anchor): + raise AssayScriptError(f"{prefix}.anchor contains unsupported symbols") + after_anchor = _integer(raw.get("after_anchor", 0), -512, 512, f"{prefix}.after_anchor") + jitter = _integer(raw.get("jitter", 0), 0, 32, f"{prefix}.jitter") + orientation = _enum(raw.get("orientation", "forward"), ORIENTATIONS, f"{prefix}.orientation") + metric = _enum(raw.get("metric", "hamming"), METRICS, f"{prefix}.metric") + k = _integer(raw.get("k", 0), 0, 3, f"{prefix}.k") + if metric == "levenshtein" and k > 2: + raise AssayScriptError(f"{prefix}: levenshtein currently supports k <= 2") + policy = _enum(raw.get("ambiguity_policy", "radius"), POLICIES, f"{prefix}.ambiguity_policy") + required = raw.get("required", True) + if not isinstance(required, bool): + raise AssayScriptError(f"{prefix}.required must be true or false") + return Segment( + name=name, + read=read, + library=library, + length=length, + start=start, + anchor=anchor, + after_anchor=after_anchor, + jitter=jitter, + orientation=orientation, + metric=metric, + k=k, + ambiguity_policy=policy, + required=required, + ) + + +def _compile_segment(segment: Segment) -> CompiledSegment: + targets = _read_library(segment.library) + lengths = sorted({len(sequence) for sequence in targets}) + findings: list[str] = [] + if lengths != [segment.length]: + findings.append( + f"{segment.name}: declared length {segment.length} differs from target lengths {lengths}" + ) + duplicates = len(targets) - len(set(targets)) + if duplicates: + findings.append(f"{segment.name}: {duplicates} duplicate target sequences") + safety_status = "safe" + if duplicates: + safety_status = "unsafe" + elif segment.k > 0: + close_pairs = _bounded_close_pairs(targets, segment.k, segment.metric) + if close_pairs is None: + safety_status = "not_computed" + findings.append( + f"{segment.name}: pairwise safety not computed above 5000 targets; run native audit" + ) + elif close_pairs: + safety_status = "unsafe" + findings.append( + f"{segment.name}: {close_pairs} target pairs overlap the configured correction radius" + ) + + strategy = _select_strategy(segment, targets) + if segment.orientation == "auto": + findings.append(f"{segment.name}: orientation auto requires runtime evidence and review") + if segment.jitter: + findings.append(f"{segment.name}: jitter search spans ±{segment.jitter} bases") + return CompiledSegment( + name=segment.name, + read=segment.read, + library=str(segment.library), + library_sha256=_sha256(segment.library), + target_count=len(targets), + target_lengths=lengths, + strategy=strategy, + start=segment.start, + anchor=segment.anchor, + after_anchor=segment.after_anchor, + jitter=segment.jitter, + orientation=segment.orientation, + metric=segment.metric, + k=segment.k, + ambiguity_policy=segment.ambiguity_policy, + required=segment.required, + safety_status=safety_status, + findings=findings, + ) + + +def _select_strategy(segment: Segment, targets: Sequence[str]) -> str: + uniform = len({len(target) for target in targets}) == 1 + if segment.k == 0 and uniform: + base = "exact_hash" + elif segment.metric == "hamming" and uniform and segment.k == 1 and segment.length <= 32: + base = "packed_hamming_neighborhood" + elif segment.metric == "hamming" and uniform: + base = "seeded_hamming_verify" + elif segment.metric == "levenshtein" and segment.length <= 32 and segment.k <= 2: + base = "packed_levenshtein_neighborhood" + else: + base = "seeded_levenshtein_verify" + if segment.anchor is not None: + base = "anchor_scan+" + base + if segment.jitter: + base = "jitter_scan+" + base + if segment.orientation != "forward": + base = "orientation_dispatch+" + base + return base + + +def _read_library(path: Path) -> list[str]: + rows: list[str] = [] + delimiter = "," if path.suffix.lower() == ".csv" else "\t" + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + columns = [column.strip() for column in line.split(delimiter)] + sequence = columns[1] if len(columns) > 1 else columns[0] + normalized = sequence.upper() + if normalized.lower() in {"sequence", "seq", "target_seq", "guide_seq", "barcode_seq"}: + continue + if not normalized: + raise AssayScriptError(f"empty target sequence in {path}") + rows.append(normalized) + if not rows: + raise AssayScriptError(f"no targets found in {path}") + return rows + + +def _bounded_close_pairs(targets: Sequence[str], k: int, metric: str) -> int | None: + if len(targets) > 5000: + return None + threshold = 2 * k + count = 0 + for left_index, left in enumerate(targets): + for right in targets[left_index + 1 :]: + if metric == "hamming": + if len(left) == len(right) and sum(a != b for a, b in zip(left, right)) <= threshold: + count += 1 + elif _levenshtein_bounded(left, right, threshold) <= threshold: + count += 1 + return count + + +def _levenshtein_bounded(left: str, right: str, limit: int) -> int: + if abs(len(left) - len(right)) > limit: + return limit + 1 + previous = list(range(len(right) + 1)) + for i, left_base in enumerate(left, start=1): + current = [i] + row_min = i + for j, right_base in enumerate(right, start=1): + value = min( + current[j - 1] + 1, + previous[j] + 1, + previous[j - 1] + (left_base != right_base), + ) + current.append(value) + row_min = min(row_min, value) + if row_min > limit: + return limit + 1 + previous = current + return previous[-1] + + +def _validate_combinations(path: Path, names: set[str]) -> list[str]: + lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + if not lines: + raise AssayScriptError("allowed combinations file is empty") + delimiter = "," if path.suffix.lower() == ".csv" else "\t" + header = {column.strip() for column in lines[0].split(delimiter)} + unknown = sorted(header - names) + if unknown: + raise AssayScriptError(f"allowed combinations reference unknown segments: {unknown}") + missing = sorted(names - header) + findings: list[str] = [] + if missing: + findings.append(f"allowed combinations do not constrain optional segments: {missing}") + findings.append(f"allowed combinations contain {max(0, len(lines) - 1)} tuples") + return findings + + +def _resolve_input(source: Path, value: Any, name: str) -> Path: + text = _required_text(value, name) + path = Path(text).expanduser() + if not path.is_absolute(): + path = source.parent / path + path = path.resolve() + if not path.is_file(): + raise AssayScriptError(f"{name} does not exist: {path}") + return path + + +def _required_text(value: Any, name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise AssayScriptError(f"{name} must be a non-empty string") + return value.strip() + + +def _enum(value: Any, choices: set[str], name: str) -> str: + text = _required_text(value, name) + if text not in choices: + raise AssayScriptError(f"{name} must be one of {sorted(choices)}") + return text + + +def _integer(value: Any, minimum: int, maximum: int, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise AssayScriptError(f"{name} must be an integer") + if value < minimum or value > maximum: + raise AssayScriptError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() From 5834b425dd07d97c944db136e7f6f5f06d8e50a4 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:18:00 +0100 Subject: [PATCH 13/93] Expose AssayScript compilation in AssayCode --- python/assaycode/cli.py | 75 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/python/assaycode/cli.py b/python/assaycode/cli.py index a58ba057..a60816f0 100644 --- a/python/assaycode/cli.py +++ b/python/assaycode/cli.py @@ -2,16 +2,79 @@ from __future__ import annotations +import argparse +import json import sys +from pathlib import Path from typing import Sequence from dotmatch import __version__ from dotmatch import cli as _engine_cli +from dotmatch.assayscript import AssayScriptError, load_and_compile, write_compiled_plan _SHORTCUTS = {"new", "infer", "check", "plan", "run", "start"} +def command_compile(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="assaycode compile", + description="Validate AssayScript v2 and write a deterministic portable execution plan.", + ) + parser.add_argument("spec", help="AssayScript v2 TOML") + parser.add_argument("--out", required=True, help="compiled plan JSON") + args = parser.parse_args(list(argv)) + try: + plan = load_and_compile(args.spec) + output = write_compiled_plan(plan, args.out) + except AssayScriptError as exc: + print(f"assaycode compile: {exc}", file=sys.stderr) + return 2 + print(json.dumps({ + "status": "review" if plan.findings else "ready", + "plan": str(output), + "segments": len(plan.segments), + "execution_order": plan.execution_order, + "findings": plan.findings, + }, indent=2, sort_keys=True)) + return 0 + + +def command_inspect(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="assaycode inspect", + description="Summarize a compiled AssayScript execution plan.", + ) + parser.add_argument("plan", help="compiled plan JSON") + args = parser.parse_args(list(argv)) + try: + data = json.loads(Path(args.plan).read_text(encoding="utf-8")) + segments = data["segments"] + if data.get("compiler_schema_version") != 1 or not isinstance(segments, list): + raise ValueError("unsupported compiled plan schema") + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"assaycode inspect: invalid compiled plan: {exc}", file=sys.stderr) + return 2 + summary = { + "name": data.get("name"), + "assay_type": data.get("assay_type"), + "segments": [ + { + "name": segment.get("name"), + "read": segment.get("read"), + "strategy": segment.get("strategy"), + "target_count": segment.get("target_count"), + "safety_status": segment.get("safety_status"), + } + for segment in segments + ], + "execution_order": data.get("execution_order", []), + "findings": data.get("findings", []), + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + def print_help() -> None: print( f"""AssayCode {__version__} — powered by the DotMatch engine @@ -22,6 +85,8 @@ def print_help() -> None: Usage: assaycode --help assaycode --version + assaycode compile assay-v2.toml --out assay.plan.json + assaycode inspect assay.plan.json assaycode check assay.toml assaycode plan assay.toml assaycode run assay.toml @@ -29,7 +94,11 @@ def print_help() -> None: assaycode assay [options] assaycode engine [options] -Assay workflow shortcuts: +AssayScript v2: + compile validate a multi-read specification and select deterministic strategies + inspect summarize a compiled plan, safety status, fingerprints, and findings + +AssaySpec v1 workflow shortcuts: new scaffold an AssayScript/AssaySpec project infer infer a reviewable assay window from reads check validate a specification and its referenced inputs @@ -60,6 +129,10 @@ def main(argv: Sequence[str] | None = None) -> int: if raw_args == ["--version"]: print(f"assaycode {__version__} (DotMatch engine {__version__})") return 0 + if raw_args[0] == "compile": + return command_compile(raw_args[1:]) + if raw_args[0] == "inspect": + return command_inspect(raw_args[1:]) if raw_args[0] == "engine": if len(raw_args) == 1: print("usage: assaycode engine [options]", file=sys.stderr) From b17363ba975bcf33a426a836dc6cb3a40e9fc34c Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:18:37 +0100 Subject: [PATCH 14/93] Test AssayScript compiler contracts --- python/tests/test_assayscript.py | 154 +++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 python/tests/test_assayscript.py diff --git a/python/tests/test_assayscript.py b/python/tests/test_assayscript.py new file mode 100644 index 00000000..175e4b27 --- /dev/null +++ b/python/tests/test_assayscript.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from assaycode import cli +from dotmatch.assayscript import AssayScriptError, load_and_compile, write_compiled_plan + + +def _write_library(path: Path, rows: list[tuple[str, str]]) -> Path: + path.write_text( + "target_id\ttarget_seq\n" + + "".join(f"{target_id}\t{sequence}\n" for target_id, sequence in rows), + encoding="utf-8", + ) + return path + + +def _write_spec(tmp_path: Path, *, duplicate: bool = False) -> Path: + guides = _write_library( + tmp_path / "guides.tsv", + [("g1", "ACGT"), ("g2", "ACGT" if duplicate else "TTTT")], + ) + samples = _write_library( + tmp_path / "samples.tsv", + [("s1", "AACCGGTT"), ("s2", "TTGGCCAA")], + ) + combinations = tmp_path / "pairs.tsv" + combinations.write_text("sample\tguide\ns1\tg1\ns2\tg2\n", encoding="utf-8") + spec = tmp_path / "assay.toml" + spec.write_text( + f""" +schema_version = 2 +name = "dual-guide-assay" +assay_type = "crispr" + +[[segments]] +name = "sample" +read = "I1" +library = "{samples.name}" +start = 0 +length = 8 +metric = "hamming" +k = 1 + +[[segments]] +name = "guide" +read = "R2" +library = "{guides.name}" +anchor = "GTTT" +after_anchor = 1 +jitter = 2 +length = 4 +metric = "levenshtein" +k = 1 +orientation = "auto" + +[constraints] +allowed_combinations = "{combinations.name}" +""".lstrip(), + encoding="utf-8", + ) + return spec + + +def test_compiler_selects_segment_specific_strategies(tmp_path: Path) -> None: + plan = load_and_compile(_write_spec(tmp_path)) + + by_name = {segment.name: segment for segment in plan.segments} + assert by_name["sample"].strategy == "packed_hamming_neighborhood" + assert by_name["guide"].strategy.startswith( + "orientation_dispatch+jitter_scan+anchor_scan+" + ) + assert plan.execution_order == ["sample", "guide"] + assert plan.allowed_combinations_sha256 + assert all(len(segment.library_sha256) == 64 for segment in plan.segments) + + +def test_compiler_records_reviewable_runtime_findings(tmp_path: Path) -> None: + plan = load_and_compile(_write_spec(tmp_path)) + + assert any("orientation auto" in finding for finding in plan.findings) + assert any("jitter search" in finding for finding in plan.findings) + assert any("allowed combinations contain 2 tuples" in finding for finding in plan.findings) + + +def test_compiler_marks_duplicate_library_unsafe(tmp_path: Path) -> None: + plan = load_and_compile(_write_spec(tmp_path, duplicate=True)) + guide = next(segment for segment in plan.segments if segment.name == "guide") + + assert guide.safety_status == "unsafe" + assert any("duplicate target" in finding for finding in guide.findings) + + +def test_compiler_rejects_ambiguous_extraction_definition(tmp_path: Path) -> None: + spec = _write_spec(tmp_path) + spec.write_text( + spec.read_text(encoding="utf-8").replace( + 'anchor = "GTTT"', + 'start = 4\nanchor = "GTTT"', + ), + encoding="utf-8", + ) + + with pytest.raises(AssayScriptError, match="exactly one of start or anchor"): + load_and_compile(spec) + + +def test_compiler_rejects_unknown_combination_segment(tmp_path: Path) -> None: + spec = _write_spec(tmp_path) + combinations = tmp_path / "pairs.tsv" + combinations.write_text("sample\tunknown\ns1\tx\n", encoding="utf-8") + + with pytest.raises(AssayScriptError, match="unknown segments"): + load_and_compile(spec) + + +def test_compiled_plan_is_portable_json_with_fingerprints(tmp_path: Path) -> None: + plan = load_and_compile(_write_spec(tmp_path)) + output = write_compiled_plan(plan, tmp_path / "build" / "assay.plan.json") + data = json.loads(output.read_text(encoding="utf-8")) + + assert data["compiler_schema_version"] == 1 + assert data["source_schema_version"] == 2 + assert len(data["source_sha256"]) == 64 + assert data["name"] == "dual-guide-assay" + + +def test_assaycode_compile_and_inspect_commands(tmp_path: Path, capsys) -> None: + spec = _write_spec(tmp_path) + output = tmp_path / "assay.plan.json" + + assert cli.main(["compile", str(spec), "--out", str(output)]) == 0 + compile_summary = json.loads(capsys.readouterr().out) + assert compile_summary["segments"] == 2 + assert output.exists() + + assert cli.main(["inspect", str(output)]) == 0 + inspect_summary = json.loads(capsys.readouterr().out) + assert inspect_summary["name"] == "dual-guide-assay" + assert {segment["name"] for segment in inspect_summary["segments"]} == { + "sample", + "guide", + } + + +def test_assaycode_compile_rejects_v1_without_breaking_dotmatch(tmp_path: Path, capsys) -> None: + spec = tmp_path / "old.toml" + spec.write_text('schema_version = 1\n', encoding="utf-8") + + assert cli.main(["compile", str(spec), "--out", str(tmp_path / "plan.json")]) == 2 + assert "schema_version must be 2" in capsys.readouterr().err From 3e9ee3df538556328864f06d66749cad59188657 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:19:57 +0100 Subject: [PATCH 15/93] Add calibrated and joint decoding foundation --- python/dotmatch/calibration.py | 304 +++++++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 python/dotmatch/calibration.py diff --git a/python/dotmatch/calibration.py b/python/dotmatch/calibration.py new file mode 100644 index 00000000..7418b2ec --- /dev/null +++ b/python/dotmatch/calibration.py @@ -0,0 +1,304 @@ +"""Calibrated quality-aware and joint known-target decoding. + +This module is experimental. It is deliberately separate from the deterministic +DotMatch default until public calibration and throughput gates pass. +""" + +from __future__ import annotations + +import math +from dataclasses import asdict, dataclass +from typing import Mapping, Sequence + + +DNA = "ACGT" + + +@dataclass(frozen=True) +class ErrorModel: + cycle_totals: tuple[int, ...] + cycle_errors: tuple[int, ...] + substitution_counts: Mapping[str, Mapping[str, int]] + prior_strength: float = 100.0 + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True) +class ProbabilisticCall: + status: str + target: str | None + posterior: float + second_posterior: float + likelihood_ratio: float + candidates: Mapping[str, float] + + +@dataclass(frozen=True) +class JointCall: + status: str + combination: Mapping[str, str] | None + posterior: float + second_posterior: float + candidates: tuple[tuple[Mapping[str, str], float], ...] + + +@dataclass(frozen=True) +class CalibrationMetrics: + count: int + accuracy: float + mean_confidence: float + expected_calibration_error: float + brier_score: float + + +def fit_error_model( + observations: Sequence[tuple[str, str, str]], + *, + prior_strength: float = 100.0, +) -> ErrorModel: + """Fit cycle and substitution errors from (observed, expected, qualities). + + Training pairs must already be independently trusted, for example exact or + uniquely assigned spike-ins. The function does not bootstrap labels from its + own probabilistic calls. + """ + if prior_strength <= 0: + raise ValueError("prior_strength must be positive") + max_length = max((len(expected) for _observed, expected, _quality in observations), default=0) + totals = [0] * max_length + errors = [0] * max_length + substitutions = {base: {other: 0 for other in DNA if other != base} for base in DNA} + for observed, expected, quality in observations: + if len(observed) != len(expected) or len(quality) != len(observed): + raise ValueError("training observed, expected, and quality strings must have equal length") + for cycle, (called, truth) in enumerate(zip(observed.upper(), expected.upper())): + if called not in DNA or truth not in DNA: + continue + totals[cycle] += 1 + if called != truth: + errors[cycle] += 1 + substitutions[truth][called] += 1 + return ErrorModel( + cycle_totals=tuple(totals), + cycle_errors=tuple(errors), + substitution_counts=substitutions, + prior_strength=float(prior_strength), + ) + + +def decode( + observed: str, + quality: str, + targets: Sequence[str], + model: ErrorModel, + *, + priors: Mapping[str, float] | None = None, + posterior_min: float = 0.99, + likelihood_ratio_min: float = 10.0, +) -> ProbabilisticCall: + if len(observed) != len(quality): + raise ValueError("observed and quality strings must have equal length") + if not 0.0 < posterior_min <= 1.0: + raise ValueError("posterior_min must be in (0, 1]") + if likelihood_ratio_min < 1.0: + raise ValueError("likelihood_ratio_min must be at least 1") + unique_targets = list(dict.fromkeys(target.upper() for target in targets)) + if not unique_targets: + raise ValueError("at least one target is required") + + log_scores: dict[str, float] = {} + default_prior = 1.0 / len(unique_targets) + for target in unique_targets: + if len(target) != len(observed): + continue + prior = default_prior if priors is None else float(priors.get(target, 0.0)) + if prior <= 0: + continue + score = math.log(prior) + for cycle, (called, truth, qchar) in enumerate(zip(observed.upper(), target, quality)): + score += math.log(_emission_probability(called, truth, ord(qchar) - 33, cycle, model)) + log_scores[target] = score + if not log_scores: + return ProbabilisticCall("none", None, 0.0, 0.0, 0.0, {}) + + probabilities = _normalize_logs(log_scores) + ranked = sorted(probabilities.items(), key=lambda item: (-item[1], item[0])) + best_target, best = ranked[0] + second = ranked[1][1] if len(ranked) > 1 else 0.0 + ratio = math.inf if second == 0.0 else best / second + status = "unique" if best >= posterior_min and ratio >= likelihood_ratio_min else "ambiguous" + return ProbabilisticCall( + status=status, + target=best_target if status == "unique" else None, + posterior=best, + second_posterior=second, + likelihood_ratio=ratio, + candidates=dict(ranked), + ) + + +def decode_joint( + components: Mapping[str, Mapping[str, float]], + allowed_combinations: Sequence[Mapping[str, str]], + *, + priors: Sequence[float] | None = None, + posterior_min: float = 0.99, + likelihood_ratio_min: float = 10.0, +) -> JointCall: + """Decode an allowed tuple from independently calibrated component masses.""" + if not components: + raise ValueError("at least one component posterior table is required") + if not allowed_combinations: + raise ValueError("at least one allowed combination is required") + if priors is not None and len(priors) != len(allowed_combinations): + raise ValueError("joint priors must match allowed combinations") + + scores: list[tuple[Mapping[str, str], float]] = [] + default_prior = 1.0 / len(allowed_combinations) + for index, combination in enumerate(allowed_combinations): + if set(combination) != set(components): + raise ValueError("each combination must define exactly the component names") + prior = default_prior if priors is None else float(priors[index]) + if prior <= 0: + continue + log_score = math.log(prior) + possible = True + for component, target in combination.items(): + mass = float(components[component].get(target, 0.0)) + if mass <= 0: + possible = False + break + log_score += math.log(mass) + if possible: + scores.append((dict(combination), log_score)) + if not scores: + return JointCall("none", None, 0.0, 0.0, ()) + + maximum = max(score for _combination, score in scores) + weights = [(combination, math.exp(score - maximum)) for combination, score in scores] + total = sum(weight for _combination, weight in weights) + ranked = sorted( + ((combination, weight / total) for combination, weight in weights), + key=lambda item: (-item[1], tuple(sorted(item[0].items()))), + ) + best_combination, best = ranked[0] + second = ranked[1][1] if len(ranked) > 1 else 0.0 + ratio = math.inf if second == 0.0 else best / second + status = "unique" if best >= posterior_min and ratio >= likelihood_ratio_min else "ambiguous" + return JointCall( + status=status, + combination=best_combination if status == "unique" else None, + posterior=best, + second_posterior=second, + candidates=tuple(ranked), + ) + + +def calibration_metrics( + calls: Sequence[tuple[float, bool]], + *, + bins: int = 10, +) -> CalibrationMetrics: + if bins <= 0: + raise ValueError("bins must be positive") + if not calls: + return CalibrationMetrics(0, 0.0, 0.0, 0.0, 0.0) + for confidence, _correct in calls: + if not 0.0 <= confidence <= 1.0: + raise ValueError("confidence values must be between 0 and 1") + count = len(calls) + accuracy = sum(correct for _confidence, correct in calls) / count + mean_confidence = sum(confidence for confidence, _correct in calls) / count + brier = sum((confidence - float(correct)) ** 2 for confidence, correct in calls) / count + ece = 0.0 + for bin_index in range(bins): + lower = bin_index / bins + upper = (bin_index + 1) / bins + bucket = [ + (confidence, correct) + for confidence, correct in calls + if lower <= confidence < upper or (bin_index == bins - 1 and confidence == 1.0) + ] + if not bucket: + continue + bucket_confidence = sum(confidence for confidence, _correct in bucket) / len(bucket) + bucket_accuracy = sum(correct for _confidence, correct in bucket) / len(bucket) + ece += len(bucket) / count * abs(bucket_confidence - bucket_accuracy) + return CalibrationMetrics(count, accuracy, mean_confidence, ece, brier) + + +def threshold_for_fdr( + calls: Sequence[tuple[float, bool]], + *, + max_fdr: float, +) -> float | None: + """Return the lowest confidence cutoff whose accepted prefix meets max FDR. + + The selected prefix maximizes accepted calls. Ties are deterministic. + """ + if not 0.0 <= max_fdr < 1.0: + raise ValueError("max_fdr must be in [0, 1)") + ranked = sorted(calls, key=lambda item: (-item[0], not item[1])) + errors = 0 + accepted_cutoff: float | None = None + for index, (confidence, correct) in enumerate(ranked, start=1): + if not 0.0 <= confidence <= 1.0: + raise ValueError("confidence values must be between 0 and 1") + errors += int(not correct) + if errors / index <= max_fdr: + accepted_cutoff = confidence + return accepted_cutoff + + +def smoothed_abundance_priors( + counts: Mapping[str, int], + *, + alpha: float = 1.0, +) -> dict[str, float]: + if alpha <= 0: + raise ValueError("alpha must be positive") + if not counts: + raise ValueError("counts must not be empty") + if any(isinstance(value, bool) or value < 0 for value in counts.values()): + raise ValueError("counts must be non-negative integers") + total = sum(counts.values()) + alpha * len(counts) + return {target: (count + alpha) / total for target, count in counts.items()} + + +def _emission_probability( + called: str, + truth: str, + quality: int, + cycle: int, + model: ErrorModel, +) -> float: + if called not in DNA or truth not in DNA: + return 0.25 + quality = min(60, max(0, quality)) + phred_error = 10.0 ** (-quality / 10.0) + if cycle < len(model.cycle_totals): + total = model.cycle_totals[cycle] + errors = model.cycle_errors[cycle] + else: + total = errors = 0 + empirical_error = (errors + 0.5) / (total + 1.0) + weight = total / (total + model.prior_strength) + error_rate = min(0.75, max(1e-9, weight * empirical_error + (1.0 - weight) * phred_error)) + if called == truth: + return max(1e-12, 1.0 - error_rate) + + substitution_row = model.substitution_counts.get(truth, {}) + substitution_total = sum(substitution_row.values()) + substitution_probability = (substitution_row.get(called, 0) + 1.0) / ( + substitution_total + 3.0 + ) + return max(1e-12, error_rate * substitution_probability) + + +def _normalize_logs(scores: Mapping[str, float]) -> dict[str, float]: + maximum = max(scores.values()) + weights = {target: math.exp(score - maximum) for target, score in scores.items()} + total = sum(weights.values()) + return {target: weight / total for target, weight in weights.items()} From d215c39fdf135d4bdcdb8b0e187225c7880e659a Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:20:33 +0100 Subject: [PATCH 16/93] Test calibrated and joint decoding --- python/tests/test_calibration.py | 156 +++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 python/tests/test_calibration.py diff --git a/python/tests/test_calibration.py b/python/tests/test_calibration.py new file mode 100644 index 00000000..61e071ff --- /dev/null +++ b/python/tests/test_calibration.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import math + +import pytest + +from dotmatch.calibration import ( + calibration_metrics, + decode, + decode_joint, + fit_error_model, + smoothed_abundance_priors, + threshold_for_fdr, +) + + +def test_fit_error_model_records_cycle_and_substitution_errors() -> None: + model = fit_error_model( + [ + ("ACGT", "ACGT", "IIII"), + ("AGGT", "ACGT", "IIII"), + ], + prior_strength=10.0, + ) + + assert model.cycle_totals == (2, 2, 2, 2) + assert model.cycle_errors == (0, 1, 0, 0) + assert model.substitution_counts["C"]["G"] == 1 + + +def test_decode_returns_selective_unique_call() -> None: + model = fit_error_model([("ACGT", "ACGT", "IIII")] * 20) + + call = decode( + "ACGT", + "IIII", + ["ACGT", "AGGT", "TTTT"], + model, + posterior_min=0.99, + likelihood_ratio_min=10.0, + ) + + assert call.status == "unique" + assert call.target == "ACGT" + assert call.posterior > 0.999 + assert call.likelihood_ratio > 10.0 + + +def test_decode_abstains_when_targets_are_not_separated() -> None: + model = fit_error_model([]) + call = decode( + "ANGT", + "!!!!", + ["ACGT", "AGGT"], + model, + posterior_min=0.9, + likelihood_ratio_min=10.0, + ) + + assert call.status == "ambiguous" + assert call.target is None + assert call.posterior == pytest.approx(0.5) + + +def test_decode_uses_smoothed_abundance_priors_without_zeroing_targets() -> None: + priors = smoothed_abundance_priors({"ACGT": 99, "AGGT": 0}, alpha=1.0) + + assert sum(priors.values()) == pytest.approx(1.0) + assert priors["AGGT"] > 0.0 + + +def test_joint_decode_uses_allowed_combinations_to_resolve_evidence() -> None: + call = decode_joint( + { + "sample": {"s1": 0.9, "s2": 0.1}, + "guide": {"g1": 0.45, "g2": 0.55}, + }, + [ + {"sample": "s1", "guide": "g2"}, + {"sample": "s2", "guide": "g1"}, + ], + posterior_min=0.8, + likelihood_ratio_min=5.0, + ) + + assert call.status == "unique" + assert call.combination == {"sample": "s1", "guide": "g2"} + assert call.posterior > 0.8 + + +def test_joint_decode_abstains_on_symmetric_evidence() -> None: + call = decode_joint( + { + "left": {"a": 0.5, "b": 0.5}, + "right": {"x": 0.5, "y": 0.5}, + }, + [ + {"left": "a", "right": "x"}, + {"left": "b", "right": "y"}, + ], + posterior_min=0.8, + ) + + assert call.status == "ambiguous" + assert call.combination is None + assert call.posterior == pytest.approx(0.5) + + +def test_calibration_metrics_report_ece_and_brier() -> None: + metrics = calibration_metrics( + [(0.9, True), (0.8, True), (0.2, False), (0.1, False)], + bins=2, + ) + + assert metrics.count == 4 + assert metrics.accuracy == 0.5 + assert metrics.mean_confidence == pytest.approx(0.5) + assert metrics.expected_calibration_error == pytest.approx(0.15) + assert metrics.brier_score == pytest.approx(0.025) + + +def test_threshold_for_fdr_maximizes_valid_prefix() -> None: + cutoff = threshold_for_fdr( + [(0.99, True), (0.95, True), (0.90, False), (0.80, True)], + max_fdr=0.25, + ) + + assert cutoff == 0.8 + + +def test_threshold_for_fdr_returns_none_when_no_call_is_safe() -> None: + assert threshold_for_fdr([(0.9, False)], max_fdr=0.0) is None + + +@pytest.mark.parametrize( + "observed, expected, quality", + [ + ("AC", "A", "II"), + ("AC", "AC", "I"), + ], +) +def test_fit_error_model_rejects_misaligned_training_rows( + observed: str, + expected: str, + quality: str, +) -> None: + with pytest.raises(ValueError, match="equal length"): + fit_error_model([(observed, expected, quality)]) + + +def test_joint_decode_rejects_incomplete_constraints() -> None: + with pytest.raises(ValueError, match="exactly the component names"): + decode_joint( + {"sample": {"s1": 1.0}, "guide": {"g1": 1.0}}, + [{"sample": "s1"}], + ) From 6a836fbd590de716e720fd9c01e5c1054eeaff6c Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:22:04 +0100 Subject: [PATCH 17/93] Add streaming assay quality monitor --- python/dotmatch/assaywatch.py | 175 ++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 python/dotmatch/assaywatch.py diff --git a/python/dotmatch/assaywatch.py b/python/dotmatch/assaywatch.py new file mode 100644 index 00000000..cb3d8b81 --- /dev/null +++ b/python/dotmatch/assaywatch.py @@ -0,0 +1,175 @@ +"""Streaming, bounded-memory assignment quality monitoring.""" + +from __future__ import annotations + +import json +import math +import sys +from collections import Counter +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterable, Iterator, Mapping, TextIO + + +STATUSES = {"unique", "ambiguous", "none", "invalid"} + + +@dataclass(frozen=True) +class WatchThresholds: + min_assignment_rate: float = 0.80 + max_ambiguous_rate: float = 0.05 + max_unmatched_rate: float = 0.15 + max_invalid_rate: float = 0.02 + min_reads: int = 1000 + + +@dataclass(frozen=True) +class RunSnapshot: + total_reads: int + unique: int + ambiguous: int + unmatched: int + invalid: int + assignment_rate: float + ambiguous_rate: float + unmatched_rate: float + invalid_rate: float + assignment_rate_interval95: tuple[float, float] + distinct_targets: int + decision: str + findings: tuple[str, ...] + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +class SequentialMonitor: + def __init__(self, thresholds: WatchThresholds | None = None) -> None: + self.thresholds = thresholds or WatchThresholds() + if self.thresholds.min_reads <= 0: + raise ValueError("min_reads must be positive") + self.counts: Counter[str] = Counter() + self.targets: Counter[str] = Counter() + + def update(self, status: str, target: str | None = None) -> None: + if status not in STATUSES: + raise ValueError(f"unsupported assignment status: {status}") + self.counts[status] += 1 + if status == "unique" and target: + self.targets[target] += 1 + + def snapshot(self) -> RunSnapshot: + total = sum(self.counts.values()) + unique = self.counts["unique"] + ambiguous = self.counts["ambiguous"] + unmatched = self.counts["none"] + invalid = self.counts["invalid"] + assignment_rate = unique / total if total else 0.0 + ambiguous_rate = ambiguous / total if total else 0.0 + unmatched_rate = unmatched / total if total else 0.0 + invalid_rate = invalid / total if total else 0.0 + findings: list[str] = [] + if total >= self.thresholds.min_reads: + if assignment_rate < self.thresholds.min_assignment_rate: + findings.append("assignment_rate_below_min") + if ambiguous_rate > self.thresholds.max_ambiguous_rate: + findings.append("ambiguous_rate_above_max") + if unmatched_rate > self.thresholds.max_unmatched_rate: + findings.append("unmatched_rate_above_max") + if invalid_rate > self.thresholds.max_invalid_rate: + findings.append("invalid_rate_above_max") + decision = "review" if findings else "on_track" + else: + decision = "insufficient_data" + return RunSnapshot( + total_reads=total, + unique=unique, + ambiguous=ambiguous, + unmatched=unmatched, + invalid=invalid, + assignment_rate=assignment_rate, + ambiguous_rate=ambiguous_rate, + unmatched_rate=unmatched_rate, + invalid_rate=invalid_rate, + assignment_rate_interval95=_wilson_interval(unique, total), + distinct_targets=len(self.targets), + decision=decision, + findings=tuple(findings), + ) + + +def monitor_events( + events: Iterable[Mapping[str, object]], + *, + every: int, + thresholds: WatchThresholds | None = None, +) -> Iterator[RunSnapshot]: + if every <= 0: + raise ValueError("every must be positive") + monitor = SequentialMonitor(thresholds) + emitted_at = 0 + for event in events: + status = event.get("status") + if not isinstance(status, str): + raise ValueError("assignment event requires a string status") + target_value = event.get("target") + target = target_value if isinstance(target_value, str) and target_value else None + monitor.update(status, target) + total = sum(monitor.counts.values()) + if total % every == 0: + emitted_at = total + yield monitor.snapshot() + total = sum(monitor.counts.values()) + if total != emitted_at: + yield monitor.snapshot() + + +def iter_jsonl(handle: TextIO) -> Iterator[Mapping[str, object]]: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON on line {line_number}: {exc}") from exc + if not isinstance(event, dict): + raise ValueError(f"line {line_number} must contain a JSON object") + yield event + + +def watch_jsonl( + source: str | Path, + output: str | Path, + *, + every: int = 100000, + thresholds: WatchThresholds | None = None, +) -> RunSnapshot | None: + input_handle = sys.stdin if str(source) == "-" else Path(source).open("rt", encoding="utf-8") + output_handle = sys.stdout if str(output) == "-" else Path(output).open("wt", encoding="utf-8") + latest: RunSnapshot | None = None + try: + for snapshot in monitor_events(iter_jsonl(input_handle), every=every, thresholds=thresholds): + latest = snapshot + output_handle.write(json.dumps(snapshot.to_dict(), sort_keys=True) + "\n") + output_handle.flush() + finally: + if input_handle is not sys.stdin: + input_handle.close() + if output_handle is not sys.stdout: + output_handle.close() + return latest + + +def _wilson_interval(successes: int, total: int, z: float = 1.959963984540054) -> tuple[float, float]: + if total == 0: + return (0.0, 1.0) + proportion = successes / total + denominator = 1.0 + z * z / total + centre = proportion + z * z / (2.0 * total) + margin = z * math.sqrt( + proportion * (1.0 - proportion) / total + z * z / (4.0 * total * total) + ) + return ( + max(0.0, (centre - margin) / denominator), + min(1.0, (centre + margin) / denominator), + ) From ee39e5747ccdbf1b24f952b036f1cd53ac7d7fe5 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:22:28 +0100 Subject: [PATCH 18/93] Expose streaming AssayCode watch command --- python/assaycode/cli.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/python/assaycode/cli.py b/python/assaycode/cli.py index a60816f0..9badad23 100644 --- a/python/assaycode/cli.py +++ b/python/assaycode/cli.py @@ -11,6 +11,7 @@ from dotmatch import __version__ from dotmatch import cli as _engine_cli from dotmatch.assayscript import AssayScriptError, load_and_compile, write_compiled_plan +from dotmatch.assaywatch import WatchThresholds, watch_jsonl _SHORTCUTS = {"new", "infer", "check", "plan", "run", "start"} @@ -75,6 +76,42 @@ def command_inspect(argv: Sequence[str]) -> int: return 0 + +def command_watch(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="assaycode watch", + description="Stream assignment JSONL into bounded-memory sequential QC snapshots.", + ) + parser.add_argument("events", help="assignment JSONL, or - for stdin") + parser.add_argument("--out", default="-", help="snapshot JSONL, or - for stdout") + parser.add_argument("--every", type=int, default=100000) + parser.add_argument("--min-reads", type=int, default=1000) + parser.add_argument("--min-assignment-rate", type=float, default=0.80) + parser.add_argument("--max-ambiguous-rate", type=float, default=0.05) + parser.add_argument("--max-unmatched-rate", type=float, default=0.15) + parser.add_argument("--max-invalid-rate", type=float, default=0.02) + parser.add_argument("--fail-on-review", action="store_true") + args = parser.parse_args(list(argv)) + try: + latest = watch_jsonl( + args.events, + args.out, + every=args.every, + thresholds=WatchThresholds( + min_assignment_rate=args.min_assignment_rate, + max_ambiguous_rate=args.max_ambiguous_rate, + max_unmatched_rate=args.max_unmatched_rate, + max_invalid_rate=args.max_invalid_rate, + min_reads=args.min_reads, + ), + ) + except (OSError, ValueError) as exc: + print(f"assaycode watch: {exc}", file=sys.stderr) + return 2 + if args.fail_on_review and latest is not None and latest.decision == "review": + return 1 + return 0 + def print_help() -> None: print( f"""AssayCode {__version__} — powered by the DotMatch engine @@ -87,6 +124,7 @@ def print_help() -> None: assaycode --version assaycode compile assay-v2.toml --out assay.plan.json assaycode inspect assay.plan.json + assaycode watch assignments.jsonl --out snapshots.jsonl assaycode check assay.toml assaycode plan assay.toml assaycode run assay.toml @@ -97,6 +135,7 @@ def print_help() -> None: AssayScript v2: compile validate a multi-read specification and select deterministic strategies inspect summarize a compiled plan, safety status, fingerprints, and findings + watch stream assignment events into sequential QC decisions AssaySpec v1 workflow shortcuts: new scaffold an AssayScript/AssaySpec project @@ -133,6 +172,8 @@ def main(argv: Sequence[str] | None = None) -> int: return command_compile(raw_args[1:]) if raw_args[0] == "inspect": return command_inspect(raw_args[1:]) + if raw_args[0] == "watch": + return command_watch(raw_args[1:]) if raw_args[0] == "engine": if len(raw_args) == 1: print("usage: assaycode engine [options]", file=sys.stderr) From 6d3ef6288ed3f60e55fe587a7d06ca691fa71c7c Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:22:54 +0100 Subject: [PATCH 19/93] Test streaming assay watch decisions --- python/tests/test_assaywatch.py | 137 ++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 python/tests/test_assaywatch.py diff --git a/python/tests/test_assaywatch.py b/python/tests/test_assaywatch.py new file mode 100644 index 00000000..25f0270a --- /dev/null +++ b/python/tests/test_assaywatch.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from assaycode import cli +from dotmatch.assaywatch import ( + SequentialMonitor, + WatchThresholds, + monitor_events, + watch_jsonl, +) + + +def test_monitor_waits_for_minimum_evidence() -> None: + monitor = SequentialMonitor(WatchThresholds(min_reads=3)) + monitor.update("unique", "g1") + monitor.update("none") + + snapshot = monitor.snapshot() + assert snapshot.decision == "insufficient_data" + assert snapshot.total_reads == 2 + assert snapshot.assignment_rate_interval95[0] < snapshot.assignment_rate + assert snapshot.assignment_rate_interval95[1] > snapshot.assignment_rate + + +def test_monitor_flags_actionable_rate_failures() -> None: + monitor = SequentialMonitor( + WatchThresholds( + min_reads=4, + min_assignment_rate=0.8, + max_ambiguous_rate=0.1, + max_unmatched_rate=0.1, + max_invalid_rate=0.1, + ) + ) + for status in ["unique", "ambiguous", "none", "invalid"]: + monitor.update(status, "g1" if status == "unique" else None) + + snapshot = monitor.snapshot() + assert snapshot.decision == "review" + assert set(snapshot.findings) == { + "assignment_rate_below_min", + "ambiguous_rate_above_max", + "unmatched_rate_above_max", + "invalid_rate_above_max", + } + + +def test_monitor_reports_on_track_and_target_coverage() -> None: + monitor = SequentialMonitor(WatchThresholds(min_reads=4)) + for target in ["g1", "g1", "g2", "g2"]: + monitor.update("unique", target) + + snapshot = monitor.snapshot() + assert snapshot.decision == "on_track" + assert snapshot.distinct_targets == 2 + assert snapshot.assignment_rate == 1.0 + + +def test_monitor_events_emits_intervals_and_final_partial_batch() -> None: + snapshots = list( + monitor_events( + [ + {"status": "unique", "target": "g1"}, + {"status": "unique", "target": "g1"}, + {"status": "none"}, + ], + every=2, + thresholds=WatchThresholds(min_reads=2), + ) + ) + + assert [snapshot.total_reads for snapshot in snapshots] == [2, 3] + + +def test_watch_jsonl_writes_machine_readable_snapshots(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text( + "\n".join( + [ + json.dumps({"status": "unique", "target": "g1"}), + json.dumps({"status": "none"}), + ] + ) + + "\n", + encoding="utf-8", + ) + output = tmp_path / "snapshots.jsonl" + + latest = watch_jsonl( + events, + output, + every=1, + thresholds=WatchThresholds(min_reads=2), + ) + + assert latest is not None + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert [row["total_reads"] for row in rows] == [1, 2] + assert rows[-1]["decision"] == "review" + + +def test_watch_rejects_unknown_status(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text('{"status":"fabricated"}\n', encoding="utf-8") + + with pytest.raises(ValueError, match="unsupported assignment status"): + watch_jsonl(events, tmp_path / "out.jsonl", every=1) + + +def test_assaycode_watch_exit_gate(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text( + '{"status":"none"}\n{"status":"none"}\n', + encoding="utf-8", + ) + output = tmp_path / "snapshots.jsonl" + + rc = cli.main( + [ + "watch", + str(events), + "--out", + str(output), + "--every", + "1", + "--min-reads", + "2", + "--fail-on-review", + ] + ) + + assert rc == 1 + assert json.loads(output.read_text(encoding="utf-8").splitlines()[-1])["decision"] == "review" From 7cd3e5cd5869269c144ec7dde90a9e45dca30de6 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:23:15 +0100 Subject: [PATCH 20/93] Test AssayCode in Bioconda recipe --- packaging/bioconda/meta.yaml | 129 ++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/packaging/bioconda/meta.yaml b/packaging/bioconda/meta.yaml index 3b5e0b2f..64cc0343 100644 --- a/packaging/bioconda/meta.yaml +++ b/packaging/bioconda/meta.yaml @@ -33,9 +33,134 @@ requirements: test: commands: - - python -c "import dotmatch; assert dotmatch.distance('ACGT', 'AGGT') == 1" + - python -c "import assaycode, dotmatch; assert assaycode.engine is dotmatch; assert dotmatch.distance('ACGT', 'AGGT') == 1" - python -c "from dotmatch.native import find_native_cli; p=find_native_cli(); assert p.name == 'dotmatch-native' and p.exists()" - - dotmatch --version | grep '^dotmatch {{ version }}$' + - assaycode --version | grep '^assaycode {{ version }} (DotMatch engine {{ version }}) + - dotmatch dist ACGT AGGT | grep '^1$' + - dotmatch leq 1 ACGT AGGT | grep '^true$' + - "dotmatch citation | grep 'Citation metadata: CITATION.cff'" + - dotmatch --help | grep 'Workflow namespaces' + - dotmatch count --help | grep 'Hamming supports k=0..3' + - dotmatch crispr-count --help | grep 'MAGeCK-ready' + - dotmatch audit --help | grep 'safe_at_hamming_k3' + - dotmatch assay --help | grep 'dotmatch assay' + - dotmatch barcode --help | grep 'dotmatch barcode' + - dotmatch panel --help | grep 'dotmatch panel' + - test -f "${PREFIX}/include/qdalign.h" + - test -f "${PREFIX}/lib/libdotmatch.a" + - test -f "${PREFIX}/lib/libdotmatch.so" || test -f "${PREFIX}/lib/libdotmatch.dylib" + - dotmatch assay init --template crispr --out assay.toml + - grep 'assay_type = "crispr"' assay.toml + - printf 'target_id\ttarget_seq\nbc0\tACGT\n' > targets.tsv + - printf '@r0\nACGT\n+\nIIII\n' > reads.fastq + - dotmatch count --targets targets.tsv --reads reads.fastq --sample-label sample --target-start 0 --target-length 4 --k 0 --metric hamming --out counts.tsv + - awk -F '\t' 'NR==2 { exit !($1=="bc0" && $2=="ACGT" && $3=="" && $4=="0" && $5=="1" && $10=="1") }' counts.tsv + - printf 'target_id\ttarget_seq\na\tAAAAAAAA\nb\tCCCCCCCC\n' > audit_targets.tsv + - dotmatch audit --targets audit_targets.tsv --k 3 --audit-mode exact --out-dir audit_out + - awk -F '\t' '$1=="safe_at_hamming_k2" { exit !($2=="yes") }' audit_out/audit_summary.tsv + - awk -F '\t' '$1=="safe_at_hamming_k3" { exit !($2=="yes") }' audit_out/audit_summary.tsv + - chmod -R a+rwX audit_out + - printf 'fastq_path,sample\nreads.fastq,treated\n' > samples.csv + - printf 'sgRNA,sgRNA_sequence,gene_symbol\ng0,ACGT,GENE0\n' > crispr_guides.csv + - dotmatch crispr-count --library crispr_guides.csv --samples samples.csv --guide-start 0 --guide-length 4 --k 2 --metric hamming --ambiguity-policy best --out crispr_counts.tsv --summary crispr_summary.json --sample-qc crispr_sample_qc.tsv + - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' crispr_counts.tsv + - awk -F '\t' 'NR==2 { exit !($1=="treated" && $2=="reads.fastq" && $3=="1" && $4=="1" && $5=="1" && $6=="1") }' crispr_sample_qc.tsv + - printf 'guide\tbases\tgene\ng0\tACGT\tGENE0\n' > guides.tsv + - dotmatch guide-counter count --input reads.fastq --samples sample --library guides.tsv --offset-sample-size 1 --offset-min-fraction 0.1 --output gc_out + - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' gc_out.counts.txt + - awk -F '\t' 'NR==1 { exit !($1=="guide" && $2=="gene" && $3=="guide_type" && $4=="sample") }' gc_out.extended-counts.txt + - awk -F '\t' 'NR==2 { exit !($1=="reads.fastq" && $2=="sample" && $3=="1" && $4=="1" && $5=="1" && $6=="1.0000") }' gc_out.stats.txt + - printf 'barcode_id\tbarcode_seq\ns1\tACGT\ns2\tTTTT\n' > barcodes.tsv + - printf '@r1\nNACGTAAAA\n+\nIIIIIIIII\n@r2\nNTTTTAAAA\n+\nIIIIIIIII\n' > barcode_reads.fastq + - dotmatch barcode infer --barcodes barcodes.tsv --reads barcode_reads.fastq --scan-starts 0:2 --barcode-length 4 --sample-reads 10 --out offset_scan.tsv --summary barcode_summary.json + - "grep '\"recommended_start\": 1' barcode_summary.json" + - dotmatch panel design --n 2 --length 4 --candidate-pool-size 100 --restarts 1 --min-hamming-distance 2 --min-levenshtein-distance 2 --out-dir panel_out + - test -f panel_out/barcodes.tsv + - test -f panel_out/design_report.json + - chmod -R a+rwX panel_out + +about: + home: https://github.com/dnncha/dotmatch + license: Apache-2.0 + license_file: LICENSE + summary: Deterministic short-DNA known-target assignment + description: | + The distribution installs AssayCode, the assay-level platform identity, and + DotMatch, its deterministic known-target short-DNA assignment engine for + CRISPR guides, barcodes, primers, panels, and whitelist-style target sets. + It is not a genome aligner and does not emit SAM/BAM or CIGAR output. + dev_url: https://github.com/dnncha/dotmatch + doc_url: https://dotmatch.readthedocs.io/ + +extra: + additional-platforms: + - osx-arm64 + recipe-maintainers: + - dnncha + + - assaycode --help | grep 'additive assay-level identity' + - dotmatch --version | grep '^dotmatch {{ version }} + - dotmatch dist ACGT AGGT | grep '^1$' + - dotmatch leq 1 ACGT AGGT | grep '^true$' + - "dotmatch citation | grep 'Citation metadata: CITATION.cff'" + - dotmatch --help | grep 'Workflow namespaces' + - dotmatch count --help | grep 'Hamming supports k=0..3' + - dotmatch crispr-count --help | grep 'MAGeCK-ready' + - dotmatch audit --help | grep 'safe_at_hamming_k3' + - dotmatch assay --help | grep 'dotmatch assay' + - dotmatch barcode --help | grep 'dotmatch barcode' + - dotmatch panel --help | grep 'dotmatch panel' + - test -f "${PREFIX}/include/qdalign.h" + - test -f "${PREFIX}/lib/libdotmatch.a" + - test -f "${PREFIX}/lib/libdotmatch.so" || test -f "${PREFIX}/lib/libdotmatch.dylib" + - dotmatch assay init --template crispr --out assay.toml + - grep 'assay_type = "crispr"' assay.toml + - printf 'target_id\ttarget_seq\nbc0\tACGT\n' > targets.tsv + - printf '@r0\nACGT\n+\nIIII\n' > reads.fastq + - dotmatch count --targets targets.tsv --reads reads.fastq --sample-label sample --target-start 0 --target-length 4 --k 0 --metric hamming --out counts.tsv + - awk -F '\t' 'NR==2 { exit !($1=="bc0" && $2=="ACGT" && $3=="" && $4=="0" && $5=="1" && $10=="1") }' counts.tsv + - printf 'target_id\ttarget_seq\na\tAAAAAAAA\nb\tCCCCCCCC\n' > audit_targets.tsv + - dotmatch audit --targets audit_targets.tsv --k 3 --audit-mode exact --out-dir audit_out + - awk -F '\t' '$1=="safe_at_hamming_k2" { exit !($2=="yes") }' audit_out/audit_summary.tsv + - awk -F '\t' '$1=="safe_at_hamming_k3" { exit !($2=="yes") }' audit_out/audit_summary.tsv + - chmod -R a+rwX audit_out + - printf 'fastq_path,sample\nreads.fastq,treated\n' > samples.csv + - printf 'sgRNA,sgRNA_sequence,gene_symbol\ng0,ACGT,GENE0\n' > crispr_guides.csv + - dotmatch crispr-count --library crispr_guides.csv --samples samples.csv --guide-start 0 --guide-length 4 --k 2 --metric hamming --ambiguity-policy best --out crispr_counts.tsv --summary crispr_summary.json --sample-qc crispr_sample_qc.tsv + - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' crispr_counts.tsv + - awk -F '\t' 'NR==2 { exit !($1=="treated" && $2=="reads.fastq" && $3=="1" && $4=="1" && $5=="1" && $6=="1") }' crispr_sample_qc.tsv + - printf 'guide\tbases\tgene\ng0\tACGT\tGENE0\n' > guides.tsv + - dotmatch guide-counter count --input reads.fastq --samples sample --library guides.tsv --offset-sample-size 1 --offset-min-fraction 0.1 --output gc_out + - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' gc_out.counts.txt + - awk -F '\t' 'NR==1 { exit !($1=="guide" && $2=="gene" && $3=="guide_type" && $4=="sample") }' gc_out.extended-counts.txt + - awk -F '\t' 'NR==2 { exit !($1=="reads.fastq" && $2=="sample" && $3=="1" && $4=="1" && $5=="1" && $6=="1.0000") }' gc_out.stats.txt + - printf 'barcode_id\tbarcode_seq\ns1\tACGT\ns2\tTTTT\n' > barcodes.tsv + - printf '@r1\nNACGTAAAA\n+\nIIIIIIIII\n@r2\nNTTTTAAAA\n+\nIIIIIIIII\n' > barcode_reads.fastq + - dotmatch barcode infer --barcodes barcodes.tsv --reads barcode_reads.fastq --scan-starts 0:2 --barcode-length 4 --sample-reads 10 --out offset_scan.tsv --summary barcode_summary.json + - "grep '\"recommended_start\": 1' barcode_summary.json" + - dotmatch panel design --n 2 --length 4 --candidate-pool-size 100 --restarts 1 --min-hamming-distance 2 --min-levenshtein-distance 2 --out-dir panel_out + - test -f panel_out/barcodes.tsv + - test -f panel_out/design_report.json + - chmod -R a+rwX panel_out + +about: + home: https://github.com/dnncha/dotmatch + license: Apache-2.0 + license_file: LICENSE + summary: Deterministic short-DNA known-target assignment + description: | + DotMatch is a deterministic known-target short-DNA assignment engine for + CRISPR guides, barcodes, primers, panels, and whitelist-style target sets. + It is not a genome aligner and does not emit SAM/BAM or CIGAR output. + dev_url: https://github.com/dnncha/dotmatch + doc_url: https://dotmatch.readthedocs.io/ + +extra: + additional-platforms: + - osx-arm64 + recipe-maintainers: + - dnncha + - dotmatch dist ACGT AGGT | grep '^1$' - dotmatch leq 1 ACGT AGGT | grep '^true$' - "dotmatch citation | grep 'Citation metadata: CITATION.cff'" From 7947c042462ecc9eb667ce84ddc96bbfe1850d7d Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:24:28 +0100 Subject: [PATCH 21/93] Rewrite paper around auditable assay compilation --- paper/paper.md | 295 +++++++++++++++++++++++++++++++------------------ 1 file changed, 185 insertions(+), 110 deletions(-) diff --git a/paper/paper.md b/paper/paper.md index 8a9cae7e..444f2914 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -1,12 +1,12 @@ --- -title: 'DotMatch: deterministic known-target short-DNA assignment for sequencing workflows' +title: 'DotMatch: ambiguity-aware known-target DNA assignment and auditable assay workflows' tags: - bioinformatics - CRISPR - FASTQ - barcode demultiplexing - edit distance - - sequence analysis + - assay reproducibility authors: - name: Donncha O'Toole orcid: 0009-0003-5012-7229 @@ -15,7 +15,7 @@ authors: affiliations: - name: Independent researcher, Ireland index: 1 -date: 4 June 2026 +date: 14 July 2026 bibliography: paper.bib repository: https://github.com/dnncha/dotmatch archive_doi: 10.5281/zenodo.20541628 @@ -23,121 +23,196 @@ archive_doi: 10.5281/zenodo.20541628 # Summary -DotMatch is an open-source command-line and Python package for assigning short -DNA read windows to a known target table. In many sequencing assays the -expected sequences are already defined: for example a CRISPR guide library, -sample barcodes, feature barcodes, primer-prefix tags, or a designed panel of -assay identifiers. In these settings the immediate task is not genome -alignment, but deciding whether a fixed window in each read matches one of the -expected targets. DotMatch extracts that window, compares it with the supplied -target table under exact, Hamming, or Levenshtein edit-distance rules, and -reports each read as uniquely assigned, ambiguous, unmatched, or invalid. - -The software provides native C routines, a command-line interface, Python -wrappers, workflow examples, and checks that connect documented claims to -reproducible examples. It is distributed from GitHub with release citation -metadata and a Zenodo archive [@dotmatch_zenodo_017]. The current examples cover -CRISPR guide counting, fixed-position barcode demultiplexing, -feature-barcode assignment, primer-start assignment, and barcode panel design. -DotMatch is deliberately narrower than a read aligner or a complete assay -analysis pipeline; it is intended to provide the assignment step that such -workflows often need before downstream analysis. +DotMatch is open-source software for assigning short sequencing-read regions to +a known set of DNA targets. Such targets include CRISPR guides, inline sample +barcodes, feature barcodes, primer or adapter prefixes, and identifiers in +designed amplicon panels. These problems differ from general genome alignment: +the target set is known before analysis, the informative sequence is usually +short, and an incorrect rescue can directly contaminate a count matrix or +sample split. + +DotMatch defines four assignment outcomes: unique, ambiguous, unmatched, and +invalid. Under its default radius policy a read is counted only when exactly one +target lies within the selected edit-distance radius. Multiple compatible +targets are reported as ambiguous rather than resolved by target order. Invalid +extraction windows and unmatched sequences remain visible in quality-control +artifacts. + +The implementation combines native C distance and indexed-assignment kernels +with command-line and Python interfaces. It provides FASTQ counting, +demultiplexing, target-library audit, CRISPR-compatible count matrices, +barcode-panel design, unmatched-read inspection, assay inference, and +self-contained reliability reports. Reproducible public-data comparisons and +independent exhaustive or Edlib validation constrain performance and +correctness claims. + +The same distribution now introduces AssayCode as an additive assay-level +identity. DotMatch remains the package, engine, scientific citation, and +compatibility contract. AssayCode supplies concise assay workflow commands and +an experimental AssayScript v2 compiler for multi-read segment descriptions. +The compiler validates target libraries and allowed combinations, fingerprints +inputs, records safety findings, and selects a deterministic execution strategy +for each segment. Experimental calibration and sequential-monitoring modules +are separated from default deterministic assignment until dedicated evidence +gates are satisfied. # Statement of need -Short-DNA assignment is a common step in sequencing analysis, but it is often -handled by a local script, a spreadsheet-derived lookup, or an option buried in -a larger pipeline. The practical question is simple: which known guide, -barcode, primer, or whitelist sequence is present at this fixed position in the -read? The details matter. A one-base correction may be acceptable for one target -table and unsafe for another; an apparently close read may be compatible with -two targets; an extraction window may fall outside a short read. These cases -should be visible in the output because they affect count matrices, sample -splits, and quality-control interpretation. - -Existing tools address adjacent needs. MAGeCK provides statistical analysis for -CRISPR screens and includes guide-counting utilities [@li2014mageck]. Cutadapt -identifies and trims adapters and primers from reads [@martin2011cutadapt]. -Edlib provides a fast exact edit-distance alignment library [@sosic2017edlib]. -These tools are valuable, but they are aimed at different layers of the -workflow. DotMatch is focused on the smaller assignment problem: a known target -set, a read window, an edit-distance policy, and explicit handling of ambiguous -or invalid reads. This makes it useful for laboratories and workflow authors -who want a deterministic assignment layer before statistical or biological -interpretation. - -# State of the field - -The bioinformatics ecosystem already contains mature tools for read alignment, -adapter trimming, CRISPR screen analysis, and workflow distribution. DotMatch is -intended to sit beside these tools rather than replace them. It uses -edit-distance rules for short known targets, but it does not produce SAM/BAM -records, CIGAR strings, or reference-index mapping output. It can produce guide -counts, but downstream phenotype modeling remains the responsibility of -MAGeCK, BAGEL, drugZ, CERES, or related screen-analysis software. It can -classify a fixed adapter-prefix window, but it does not trim reads or replace -Cutadapt. - -This limited scope is also a validation choice. The repository includes public -example data, generated reports, benchmark rows, and checks that constrain what -is claimed in documentation and release notes. Packaging work follows common -life-science software conventions, including Bioconda and container-oriented -release checks [@gruning2018bioconda]. - -# Software design - -DotMatch treats assignment as a contract over four inputs: a target table, a -read window, an edit metric, and an ambiguity policy. Under the default radius -policy, a read is counted only when exactly one target lies within the -configured edit-distance radius. If more than one target is compatible, the read -is reported as ambiguous rather than forced to an arbitrary best match. -Unmatched reads and invalid extraction windows are reported separately so they -can be inspected during quality control. - -The implementation combines native C kernels with Python and CLI interfaces. -For supported A/C/G/T windows, indexed candidate generation avoids scanning the -entire target table for every read while preserving the same assignment -semantics as exhaustive validation. Cases outside that indexed path fall back to -semantics-preserving code paths. Outputs include count tables, split FASTQ -files, target audits, top-unmatched tables, summary JSON, and HTML reports. A -barcode-panel design mode enumerates edit-distance neighborhoods, reports -nearest-neighbor risks, flags reverse-complement hazards, and writes collision -summaries that can be checked automatically. - -Tests cover native edit-distance routines, CLI workflows, Python wrappers, -packaging metadata, workflow fixtures, and the public examples used in the -documentation. The project documentation separates supported statements from -experimental or unsupported claims, so performance and use-case language is -tied to reproducible artifacts rather than broad assertions. - -# Research impact statement - -DotMatch is intended for researchers building or auditing sequencing workflows -where known short sequences determine sample, guide, feature, primer, or panel -identity. Its immediate contribution is practical: reproducible assignment -decisions, explicit ambiguity accounting, and a documented assignment step that -can be cited or reused in methods descriptions. Public examples demonstrate -CRISPR guide-counting and barcode-style workloads using checked input data and -comparator semantics. - -The project is early-stage, but it is already packaged with tests, release -metadata, and an archived DOI for citation. Future impact will depend less on -additional assignment features than on adoption in standard workflow contexts: -Bioconda/BioContainers propagation, nf-core-style modules, Galaxy wrappers, and -documented downstream uses. +Known-target sequence assignment is frequently implemented as an undocumented +local script or embedded inside a larger analysis workflow. A simple nearest +match is not generally sufficient. Correction safety depends on the target +codebook, metric, allowed error radius, sequencing quality, extraction position, +and ambiguity policy. A barcode can be one substitution from several samples; +a guide window can be shifted by library construction; a read can be too short +to contain the requested region; independently plausible barcode components can +form a biologically impossible tuple. + +These states affect downstream scientific conclusions. CRISPR screen analysis +software such as MAGeCK [@li2014mageck] expects reliable guide counts. +Single-cell CRISPR guide-assignment strategies can change the number of assigned +cells and downstream discoveries [@braunger2024crispat]. Demultiplexing systems +such as Pheniqs demonstrate the value of quality-aware probabilistic confidence +for complex barcode designs [@galanti2021pheniqs]. Cutadapt +[@martin2011cutadapt] and Flexiplex [@cheng2024flexiplex] address adjacent +adapter, barcode, or flexible sequence-search tasks. Edlib +[@sosic2017edlib] provides an exact edit-distance alignment library. + +DotMatch does not seek to replace these tools. It provides a narrow, +inspectable layer between raw sequencing reads and downstream biological +analysis: extract declared regions, compare them with declared known targets, +preserve ambiguity, audit correction safety, and record enough provenance for +the decision to be reproduced. + +# Assignment contract and algorithms + +The deterministic assignment contract consists of a target table, read window, +metric, maximum distance, and ambiguity policy. Exact, Hamming, and Levenshtein +modes are supported within documented command-specific bounds. The default +radius policy returns a unique assignment only when one target is compatible +within the complete radius. A separate best-distance policy is available for +explicit compatibility with workflows that choose the closest target. + +For supported fixed A/C/G/T windows, indexed candidate generation reduces the +number of targets requiring distance verification. Exact hash lookup, +packed-neighbourhood indexes, seeded candidate generation, specialized Hamming +paths, and bit-parallel Myers distance kernels are selected according to metric, +length, and radius. Unsupported inputs fall back to semantics-preserving paths +rather than silently changing the assignment rule. + +The implementation reports candidate-verification counts and can compare +indexed results against exhaustive native assignment or Edlib. Public evidence +gates require zero oracle disagreement for the validated lanes. Performance +claims are tied to raw benchmark artifacts, comparator semantics, software +versions, and generated reports instead of being generalized to all alignment +or demultiplexing workloads. + +# Auditable assay workflows + +AssaySpec v1 wraps assignment in a reviewable project. It validates referenced +inputs, compiles native commands, audits target libraries, executes counting or +demultiplexing, and writes normalized specifications, manifests, methods text, +software versions, reliability findings, suggested fixes, and HTML reports. +Production profiles can block unsafe correction or an unreviewed inferred +specification before assignment begins. + +The autopsy workflow examines low assignment, ambiguity, invalid windows, and +frequent unmatched sequences. It can identify evidence for a wrong extraction +offset and propose a specification change. Proposed changes are written as +review artifacts; they are not silently applied. + +Barcode-panel design uses the same assignment outcomes. It checks duplicates +and near neighbours, enumerates configured error spheres within documented +bounds, reports collision pairs, simulates reads, and exports lab-facing panel +and plate artifacts. This connects pre-sequencing codebook design with the +post-sequencing rules used to decode it. + +# AssayCode and AssayScript + +AssayCode is an additive product identity installed by the DotMatch Python +distribution. Existing commands, imports, native artifacts, schemas, DOI, and +citations remain supported. AssayCode provides assay-oriented shortcuts while +retaining an explicit escape hatch to the DotMatch engine. + +AssayScript v2 is an experimental multi-segment description. A segment declares +its source read (R1, R2, I1, or I2), target library, fixed position or anchor, +length, positional jitter, orientation, metric, radius, ambiguity policy, and +whether it is required. A constraint table can enumerate allowed combinations +across segments. + +The compiler currently produces a portable JSON plan rather than claiming a +complete universal assay runtime. The plan contains source and input SHA-256 +fingerprints, target counts and lengths, safety status, selected matching +strategy, execution order, and review findings. This creates a testable +foundation for joint combinatorial execution without overstating current +capability. + +# Experimental uncertainty and run monitoring + +DotMatch's deterministic behavior remains the default. An experimental +quality-aware module fits per-cycle error rates and substitution patterns from +independently trusted observed/expected training pairs. It combines empirical +evidence with Phred probabilities, supports abundance priors with smoothing, +reports posterior mass and likelihood ratios, and abstains when configured +selective-decoding thresholds are not met. + +A joint decoder combines calibrated component probability tables only over +declared allowed tuples. This can resolve evidence using assay constraints while +still returning ambiguous when posterior separation is insufficient. +Calibration metrics include Brier score and expected calibration error, and a +held-out threshold selector can maximize accepted calls subject to an empirical +false-discovery ceiling. These APIs remain experimental until public datasets +show calibration and yield improvements at a fixed measured error rate. + +AssayCode also includes a bounded-memory sequential monitor for assignment +events. It reports assignment, ambiguity, unmatched, and invalid rates with a +Wilson confidence interval and emits machine-readable on-track, review, or +insufficient-data decisions. It is a workflow monitoring primitive, not yet a +sequencer-control or production adaptive-sampling claim. + +# Validation and evidence boundaries + +Tests cover native kernels, deterministic fuzzing, Python APIs, CLI workflows, +AssaySpec projects, AssayScript compilation, calibration mathematics, +sequential monitoring, packaging, and public workflow fixtures. The repository +separates supported, experimental, and unsupported statements in a +machine-checked evidence inventory. + +Public examples include CRISPR guide counting, inline barcode demultiplexing, +feature-barcode and guide-capture assignment, primer-prefix workloads, and +barcode-panel design. Comparative reports keep exact, Hamming, and Levenshtein +semantics separate. Accelerated or probabilistic paths are not promoted by +association with deterministic evidence; each requires its own correctness, +calibration, performance, and public-data gate. + +# Availability and research impact + +DotMatch is distributed as source, Python wheels, and a Bioconda package +following common life-science distribution practices [@gruning2018bioconda]. +The package includes the DotMatch command, Python APIs, native library +artifacts, and the additive AssayCode command. Release metadata and a Zenodo +archive support precise software citation [@dotmatch_zenodo_017]. + +The intended impact is a reusable reliability boundary for laboratories, core +facilities, workflow authors, and assay developers. A documented assignment +contract reduces the risk that plausible but ambiguous reads become +untraceable counts. The longer-term AssayCode direction connects assay +description, pre-run safety, deterministic execution, calibrated uncertainty, +diagnosis, and monitoring while preserving the narrower evidence boundary of +each implemented component. # AI usage disclosure -OpenAI Codex was used to help prepare repository documentation, release -metadata, and this JOSS paper draft. The author remains responsible for the -software and manuscript. AI-generated edits were checked against repository -tests, evidence gates, release metadata, and public DOI records before being -retained. +OpenAI Codex was used to assist with implementation, tests, documentation, +benchmark infrastructure, and preparation of this manuscript. The author +remains responsible for the software, scientific claims, release decisions, and +manuscript. AI-assisted changes are retained only after repository tests and +the relevant evidence or release gates pass; experimental features are labeled +as such where those gates are incomplete. # Acknowledgements -DotMatch uses public sequencing datasets, public CRISPR library resources, and -open-source bioinformatics tools as comparators and validation references. No -external financial support is claimed in this draft. +DotMatch uses public sequencing datasets, public CRISPR libraries, and +open-source tools as comparators and validation references. No external +financial support is claimed. # References From 5b6ee50c44e08d91c9a18e0056fa3910409ca78d Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:24:47 +0100 Subject: [PATCH 22/93] Add assay decoding references --- paper/paper.bib | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/paper/paper.bib b/paper/paper.bib index 4c585529..43306a9d 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -50,3 +50,35 @@ @article{gruning2018bioconda year = {2018}, doi = {10.1038/s41592-018-0046-7} } + +@article{galanti2021pheniqs, + author = {Galanti, Lior and Shasha, Dennis and Gunsalus, Kristin C.}, + title = {Pheniqs 2.0: accurate, high-performance Bayesian decoding and confidence estimation for combinatorial barcode indexing}, + journal = {BMC Bioinformatics}, + volume = {22}, + pages = {359}, + year = {2021}, + doi = {10.1186/s12859-021-04267-5} +} + +@article{braunger2024crispat, + author = {Braunger, Julia M. and others}, + title = {Guide assignment in single-cell CRISPR screens using crispat}, + journal = {Bioinformatics}, + volume = {40}, + number = {9}, + pages = {btae535}, + year = {2024}, + doi = {10.1093/bioinformatics/btae535} +} + +@article{cheng2024flexiplex, + author = {Cheng, Oliver and others}, + title = {Flexiplex: a versatile demultiplexer and search tool for omics data}, + journal = {Bioinformatics}, + volume = {40}, + number = {3}, + pages = {btae102}, + year = {2024}, + doi = {10.1093/bioinformatics/btae102} +} From 1b4e2c59119179320135837bc32db1edbbbc21b8 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:25:17 +0100 Subject: [PATCH 23/93] Document AssayCode compiler and monitoring --- docs/assaycode.md | 48 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/assaycode.md b/docs/assaycode.md index 957a2753..eb7fda5d 100644 --- a/docs/assaycode.md +++ b/docs/assaycode.md @@ -20,12 +20,29 @@ citation. | AssayCode Audit | Pre-run collision, input, and safety checks | | AssayCode Autopsy | Evidence-backed diagnosis of offsets, unsafe correction, unmatched reads, and other failures | | AssayCode Design | Barcode-panel design, optimization, simulation, layout, and export | -| AssayCode Watch | Reserved identity for future streaming run intelligence; not a current capability | +| AssayCode Watch | Experimental bounded-memory sequential QC over assignment event streams | | AssayCode Pro | Commercial workbench, registry, signed-report, private-assay, audit-trail, and support boundary | Names in this table are architecture and positioning. A surface is not a public capability until its command, schema, tests, evidence, and release gate exist. +## AssayScript v2 Compilation + +AssayScript v2 describes multiple target-bearing segments across R1, R2, I1, +and I2. Each segment declares a fixed position or anchor, target library, +orientation, jitter, metric, radius, ambiguity policy, and requirement status. +An optional constraint table declares allowed cross-segment combinations. + +```bash +assaycode compile assay-v2.toml --out assay.plan.json +assaycode inspect assay.plan.json +``` + +The compiler validates and fingerprints every input, audits bounded target +sets, records review findings, selects a deterministic strategy per segment, +and emits a portable JSON plan. It does not yet claim a production universal +multi-segment execution runtime. + ## Commands Installing the Python distribution provides both command identities: @@ -69,6 +86,28 @@ assaycode engine validate --targets targets.tsv --reads reads.fastq.gz \ --target-length 20 ``` +## Streaming QC + +`assaycode watch` consumes assignment events as JSON Lines. It keeps bounded +state, emits periodic rate snapshots with a 95% Wilson interval, and returns +`insufficient_data`, `on_track`, or `review` decisions under explicit +thresholds. + +```bash +assaycode watch assignments.jsonl --out watch.jsonl --every 100000 +``` + +This is an experimental workflow primitive, not a sequencer-control or adaptive +sampling claim. + +## Experimental Calibration + +`dotmatch.calibration` contains a held-apart experimental decoder with +per-cycle empirical error fitting, Phred shrinkage, selective posterior and +likelihood-ratio thresholds, joint inference over allowed assay combinations, +Brier score, expected calibration error, and held-out FDR threshold selection. +Deterministic DotMatch assignment remains the production default. + ## Python Identity ```python @@ -102,9 +141,10 @@ methods should continue to cite the DotMatch release that performed assignment. ## Claim Boundary -AssayCode currently organizes existing assay-level capabilities. It does not yet -claim a general assay compiler, calibrated probabilistic decoder, production -streaming monitor, or universal sequencing platform. +AssayCode now includes an experimental multi-segment compiler, quality-aware +calibration module, joint tuple decoder, and streaming QC primitive. It does not +yet claim a production universal multi-segment runtime, calibrated public-data +superiority, sequencer control, or universal sequencing-platform coverage. Those capabilities require implementation plus independent correctness, calibration, performance, and public-data evidence before their names become From f813c4b29e97d06101aaaedd5b7e7f9573b3a668 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:25:37 +0100 Subject: [PATCH 24/93] Record AssayCode experimental evidence boundary --- docs/scientific-claims.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/scientific-claims.md b/docs/scientific-claims.md index b0e18fb5..5f3d13ce 100644 --- a/docs/scientific-claims.md +++ b/docs/scientific-claims.md @@ -59,6 +59,18 @@ Techniques deliberately not claimed: observed edited base is above a configured Phred threshold, but this is a deterministic hard filter. It is not a target posterior probability, a calibrated sequencing-error model, or a quality-weighted confidence score; +- AssayScript v2 compilation is experimental: it validates multi-read segment + declarations, fingerprints inputs, selects deterministic strategies, and + writes portable JSON plans. It is not yet a production universal + multi-segment runtime; +- the `dotmatch.calibration` module is experimental: it fits per-cycle error + summaries from independently trusted pairs, combines empirical and Phred + evidence, supports selective and allowed-tuple decoding, and reports + calibration metrics. It has not yet passed public-data calibration or + throughput gates and does not replace the deterministic CLI path; +- `assaycode watch` is an experimental bounded-memory monitor over assignment + JSONL. Its threshold decisions are workflow QC signals, not sequencer control + or validated adaptive sampling; - calibrated statistical screen interpretation is left to downstream tools such as MAGeCK, BAGEL, drugZ, or CERES because DotMatch stops at read assignment and QC; @@ -104,7 +116,7 @@ alignment or all demultiplexing tasks. | Indexed assignment preserves native exhaustive-scan semantics for `unique`, `ambiguous`, `none`, and `invalid` outcomes in the supported fixed-window lanes. | Supported | `dotmatch validate`, native assignment tests, `make native-exact-gate`, Edlib validation artifacts under `benchmarks/raw/`, Levenshtein `k=2` CLI regression cases, Hamming `k=2`/`k=3` comparator artifacts | The native gate requires zero Edlib mismatches; large-library exact rows must beat `exact_hash_lookup`; large-library indexed `k=1` rows must beat exhaustive Edlib by >10x, beat the best BK-tree/neighbor baseline, and verify no more than 1.05 candidates/read; large-library fixed-length `k=2` substitution rows must beat exhaustive Edlib by >8x while verifying no more than 1.05 candidates/read; and large-library Levenshtein `k=2` insertion/deletion rows must beat exhaustive Edlib by >8x while verifying no more than 25 candidates/read. Levenshtein `k=2` uses packed A/C/G/T hash-neighborhood pruning for windows up to 32 bases, with fallback preserving semantics for unsupported cases. Hamming `k=2`/`k=3` is same-length substitution-only fixed-window matching. Current `N`/IUPAC behavior is literal-byte matching, not wildcard expansion semantics. | | Public CRISPR guide-counting rows are validated. | Supported | `make public-crispr-evidence-gate` passes; report at `docs/benchmarks/public_crispr/README.md` | Supports the documented MAGeCK/Yusa public-data workflow, not universal CRISPR superiority. | | Extended CRISPR comparison rows are validated. | Supported | `make crispr-comparison-gate` passes; report at `docs/benchmarks/crispr_comparison/README.md` | Applies to the recorded CRISPR guide-counting lanes and their documented comparator semantics. | -| DotMatch has an experimental GPU acceleration evidence lane. | Experimental | `make bench-gpu`, `make gpu-evidence-gate`, report at `docs/benchmarks/gpu/README.md` | Current evidence is Apple Metal-only for packed Hamming `k=1`, including synthetic rows and a public CRISPR FASTQ extract-pack-dispatch-readback-count row. It is not a production speed claim. Promotion requires additional real-workload gates, non-A/C/G/T fallback, and production scheduling. | +| AssayCode exposes experimental AssayScript v2 compilation, calibrated decoding primitives, allowed-tuple inference, and sequential JSONL QC. | Experimental | Python unit tests for compiler validation/fingerprints/strategy selection, posterior calibration mathematics, joint decoding, and streaming thresholds | These are foundations, not a production universal assay runtime or public-data superiority claim. Deterministic DotMatch assignment remains authoritative. |\n| DotMatch has an experimental GPU acceleration evidence lane. | Experimental | `make bench-gpu`, `make gpu-evidence-gate`, report at `docs/benchmarks/gpu/README.md` | Current evidence is Apple Metal-only for packed Hamming `k=1`, including synthetic rows and a public CRISPR FASTQ extract-pack-dispatch-readback-count row. It is not a production speed claim. Promotion requires additional real-workload gates, non-A/C/G/T fallback, and production scheduling. | | FASTQ count and demux workflows can optionally gate one-edit substitution and read-insertion rescue by observed Sanger Phred quality. | Supported | `make cli-test`; `--max-correction-qual` CLI regression cases | This is a deterministic correction filter, not a calibrated sequencing-error probability model. Read-deletion rescue has no observed edited base to score and is not rejected by this gate. | | FASTQ count workflows can optionally reject same-length unique calls whose Phred-quality posterior is below a configured threshold. | Experimental | Python and CLI posterior regression tests | The posterior model is an opt-in conservative filter over fixed-window calls. It is not calibrated public evidence, not supported for demux output routing, and not a throughput claim. | | The Python API exposes an experimental quality-aware posterior helper for one fixed-window read against known targets. | Experimental | `python/tests/test_matcher.py` posterior-assignment cases | This is a simple Phred likelihood helper with literal-byte target comparison and optional priors. It is not yet calibrated on public datasets, not used by the high-throughput CLI, and not a speed claim. | From 087eace65c3f26f044f12a17e07172617f8d604f Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:27:27 +0100 Subject: [PATCH 25/93] Add AssayCode readiness gate --- scripts/check_assaycode_readiness.py | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 scripts/check_assaycode_readiness.py diff --git a/scripts/check_assaycode_readiness.py b/scripts/check_assaycode_readiness.py new file mode 100644 index 00000000..5e0baf72 --- /dev/null +++ b/scripts/check_assaycode_readiness.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Fail closed when AssayCode platform surfaces drift or overclaim.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def require(path: str, needles: list[str]) -> list[str]: + file_path = ROOT / path + if not file_path.is_file(): + return [f"missing required AssayCode artifact: {path}"] + text = file_path.read_text(encoding="utf-8") + return [f"{path}: missing required text: {needle}" for needle in needles if needle not in text] + + +def main() -> int: + errors: list[str] = [] + errors += require( + "pyproject.toml", + [ + 'assaycode = "assaycode.cli:main"', + 'include = ["assaycode*", "dotmatch*", "quickdna*"]', + ], + ) + errors += require( + "python/assaycode/cli.py", + ["command_compile", "command_inspect", "command_watch"], + ) + errors += require( + "python/dotmatch/assayscript.py", + ["class CompiledAssay", "source_sha256", "library_sha256", "safety_status"], + ) + errors += require( + "python/dotmatch/calibration.py", + [ + "class ErrorModel", + "def decode_joint", + "def calibration_metrics", + "def threshold_for_fdr", + "This module is experimental", + ], + ) + errors += require( + "python/dotmatch/assaywatch.py", + ["class SequentialMonitor", "assignment_rate_interval95", "insufficient_data"], + ) + for test in [ + "python/tests/test_assaycode_brand.py", + "python/tests/test_assayscript.py", + "python/tests/test_calibration.py", + "python/tests/test_assaywatch.py", + ]: + errors += require(test, ["test_"]) + errors += require( + "docs/assaycode.md", + [ + "Compatibility Contract", + "AssayScript v2 Compilation", + "Experimental Calibration", + "not yet claim", + ], + ) + errors += require( + "docs/scientific-claims.md", + [ + "AssayScript v2 compilation is experimental", + "deterministic DotMatch assignment remains authoritative", + ], + ) + errors += require( + "paper/paper.md", + [ + "# AssayCode and AssayScript", + "# Experimental uncertainty and run monitoring", + "does not yet claim", + ], + ) + errors += require( + "packaging/bioconda/meta.yaml", + ["import assaycode, dotmatch", "assaycode --version"], + ) + if errors: + for error in errors: + print(f"assaycode-readiness: {error}", file=sys.stderr) + return 1 + print("assaycode-readiness: ready") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2a57d2bfb34a7f6f65e5963dccedaeb0f932af63 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:27:46 +0100 Subject: [PATCH 26/93] Gate AssayCode release readiness --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 62cbeee2..ef09f7dd 100644 --- a/Makefile +++ b/Makefile @@ -454,7 +454,7 @@ repository-ready: python3 scripts/check_evidence_gallery.py $(MAKE) docs-ready -release-ready: python-test python-package-test docs-ready scientific-readiness-ready assay-evidence-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready evidence-gallery-ready distribution-record-ready bioconda-recipe-ready gpu-evidence-gate native-exact-gate public-crispr-evidence-gate crispr-comparison-gate barcode-comparison-gate feature-barcode-public-gate perturb-seq-public-gate amplicon-panel-public-gate bcl-tiny-public-gate oligo-adapter-public-gate reviewer-readiness-ready adoption-metrics-ready +release-ready: python-test python-package-test docs-ready scientific-readiness-ready assay-evidence-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready evidence-gallery-ready distribution-record-ready bioconda-recipe-ready gpu-evidence-gate native-exact-gate public-crispr-evidence-gate crispr-comparison-gate barcode-comparison-gate feature-barcode-public-gate perturb-seq-public-gate amplicon-panel-public-gate bcl-tiny-public-gate oligo-adapter-public-gate reviewer-readiness-ready adoption-metrics-ready assaycode-readiness-ready python3 scripts/check_release_readiness.py pretag-ready: @@ -509,6 +509,10 @@ adoption-metrics-ready: distribution-record-ready: python3 scripts/check_distribution_record.py +.PHONY: assaycode-readiness-ready +assaycode-readiness-ready: + python3 scripts/check_assaycode_readiness.py + bioconda-recipe-ready: python3 scripts/check_bioconda_recipe.py From 52449bd0b1b1847c69c3808bec82f0c7788c2ca1 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:28:06 +0100 Subject: [PATCH 27/93] Align AssayCode readiness claim check --- scripts/check_assaycode_readiness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_assaycode_readiness.py b/scripts/check_assaycode_readiness.py index 5e0baf72..d17bb771 100644 --- a/scripts/check_assaycode_readiness.py +++ b/scripts/check_assaycode_readiness.py @@ -69,7 +69,7 @@ def main() -> int: "docs/scientific-claims.md", [ "AssayScript v2 compilation is experimental", - "deterministic DotMatch assignment remains authoritative", + "Deterministic DotMatch assignment remains authoritative", ], ) errors += require( From 3dfdfcde4fb8c8b1fc2b17b8f6205db50eef9988 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:28:34 +0100 Subject: [PATCH 28/93] Align rewritten paper with JOSS structure --- paper/paper.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/paper/paper.md b/paper/paper.md index 444f2914..8f78185a 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -67,6 +67,8 @@ a guide window can be shifted by library construction; a read can be too short to contain the requested region; independently plausible barcode components can form a biologically impossible tuple. +# State of the field + These states affect downstream scientific conclusions. CRISPR screen analysis software such as MAGeCK [@li2014mageck] expects reliable guide counts. Single-cell CRISPR guide-assignment strategies can change the number of assigned @@ -83,7 +85,9 @@ analysis: extract declared regions, compare them with declared known targets, preserve ambiguity, audit correction safety, and record enough provenance for the decision to be reproduced. -# Assignment contract and algorithms +# Software design + +## Assignment contract and algorithms The deterministic assignment contract consists of a target table, read window, metric, maximum distance, and ambiguity policy. Exact, Hamming, and Levenshtein @@ -184,7 +188,7 @@ semantics separate. Accelerated or probabilistic paths are not promoted by association with deterministic evidence; each requires its own correctness, calibration, performance, and public-data gate. -# Availability and research impact +# Research impact statement DotMatch is distributed as source, Python wheels, and a Bioconda package following common life-science distribution practices [@gruning2018bioconda]. From ed89820d7f702fd6ac3445d9c23ed7e4141d0397 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:29:31 +0100 Subject: [PATCH 29/93] Repair and extend Bioconda AssayCode smoke tests --- packaging/bioconda/meta.yaml | 128 +---------------------------------- 1 file changed, 3 insertions(+), 125 deletions(-) diff --git a/packaging/bioconda/meta.yaml b/packaging/bioconda/meta.yaml index 64cc0343..49b543ab 100644 --- a/packaging/bioconda/meta.yaml +++ b/packaging/bioconda/meta.yaml @@ -35,7 +35,9 @@ test: commands: - python -c "import assaycode, dotmatch; assert assaycode.engine is dotmatch; assert dotmatch.distance('ACGT', 'AGGT') == 1" - python -c "from dotmatch.native import find_native_cli; p=find_native_cli(); assert p.name == 'dotmatch-native' and p.exists()" - - assaycode --version | grep '^assaycode {{ version }} (DotMatch engine {{ version }}) + - assaycode --version | grep '^assaycode {{ version }} (DotMatch engine {{ version }})$' + - assaycode --help | grep 'additive assay-level identity' + - dotmatch --version | grep '^dotmatch {{ version }}$' - dotmatch dist ACGT AGGT | grep '^1$' - dotmatch leq 1 ACGT AGGT | grep '^true$' - "dotmatch citation | grep 'Citation metadata: CITATION.cff'" @@ -92,130 +94,6 @@ about: dev_url: https://github.com/dnncha/dotmatch doc_url: https://dotmatch.readthedocs.io/ -extra: - additional-platforms: - - osx-arm64 - recipe-maintainers: - - dnncha - - - assaycode --help | grep 'additive assay-level identity' - - dotmatch --version | grep '^dotmatch {{ version }} - - dotmatch dist ACGT AGGT | grep '^1$' - - dotmatch leq 1 ACGT AGGT | grep '^true$' - - "dotmatch citation | grep 'Citation metadata: CITATION.cff'" - - dotmatch --help | grep 'Workflow namespaces' - - dotmatch count --help | grep 'Hamming supports k=0..3' - - dotmatch crispr-count --help | grep 'MAGeCK-ready' - - dotmatch audit --help | grep 'safe_at_hamming_k3' - - dotmatch assay --help | grep 'dotmatch assay' - - dotmatch barcode --help | grep 'dotmatch barcode' - - dotmatch panel --help | grep 'dotmatch panel' - - test -f "${PREFIX}/include/qdalign.h" - - test -f "${PREFIX}/lib/libdotmatch.a" - - test -f "${PREFIX}/lib/libdotmatch.so" || test -f "${PREFIX}/lib/libdotmatch.dylib" - - dotmatch assay init --template crispr --out assay.toml - - grep 'assay_type = "crispr"' assay.toml - - printf 'target_id\ttarget_seq\nbc0\tACGT\n' > targets.tsv - - printf '@r0\nACGT\n+\nIIII\n' > reads.fastq - - dotmatch count --targets targets.tsv --reads reads.fastq --sample-label sample --target-start 0 --target-length 4 --k 0 --metric hamming --out counts.tsv - - awk -F '\t' 'NR==2 { exit !($1=="bc0" && $2=="ACGT" && $3=="" && $4=="0" && $5=="1" && $10=="1") }' counts.tsv - - printf 'target_id\ttarget_seq\na\tAAAAAAAA\nb\tCCCCCCCC\n' > audit_targets.tsv - - dotmatch audit --targets audit_targets.tsv --k 3 --audit-mode exact --out-dir audit_out - - awk -F '\t' '$1=="safe_at_hamming_k2" { exit !($2=="yes") }' audit_out/audit_summary.tsv - - awk -F '\t' '$1=="safe_at_hamming_k3" { exit !($2=="yes") }' audit_out/audit_summary.tsv - - chmod -R a+rwX audit_out - - printf 'fastq_path,sample\nreads.fastq,treated\n' > samples.csv - - printf 'sgRNA,sgRNA_sequence,gene_symbol\ng0,ACGT,GENE0\n' > crispr_guides.csv - - dotmatch crispr-count --library crispr_guides.csv --samples samples.csv --guide-start 0 --guide-length 4 --k 2 --metric hamming --ambiguity-policy best --out crispr_counts.tsv --summary crispr_summary.json --sample-qc crispr_sample_qc.tsv - - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' crispr_counts.tsv - - awk -F '\t' 'NR==2 { exit !($1=="treated" && $2=="reads.fastq" && $3=="1" && $4=="1" && $5=="1" && $6=="1") }' crispr_sample_qc.tsv - - printf 'guide\tbases\tgene\ng0\tACGT\tGENE0\n' > guides.tsv - - dotmatch guide-counter count --input reads.fastq --samples sample --library guides.tsv --offset-sample-size 1 --offset-min-fraction 0.1 --output gc_out - - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' gc_out.counts.txt - - awk -F '\t' 'NR==1 { exit !($1=="guide" && $2=="gene" && $3=="guide_type" && $4=="sample") }' gc_out.extended-counts.txt - - awk -F '\t' 'NR==2 { exit !($1=="reads.fastq" && $2=="sample" && $3=="1" && $4=="1" && $5=="1" && $6=="1.0000") }' gc_out.stats.txt - - printf 'barcode_id\tbarcode_seq\ns1\tACGT\ns2\tTTTT\n' > barcodes.tsv - - printf '@r1\nNACGTAAAA\n+\nIIIIIIIII\n@r2\nNTTTTAAAA\n+\nIIIIIIIII\n' > barcode_reads.fastq - - dotmatch barcode infer --barcodes barcodes.tsv --reads barcode_reads.fastq --scan-starts 0:2 --barcode-length 4 --sample-reads 10 --out offset_scan.tsv --summary barcode_summary.json - - "grep '\"recommended_start\": 1' barcode_summary.json" - - dotmatch panel design --n 2 --length 4 --candidate-pool-size 100 --restarts 1 --min-hamming-distance 2 --min-levenshtein-distance 2 --out-dir panel_out - - test -f panel_out/barcodes.tsv - - test -f panel_out/design_report.json - - chmod -R a+rwX panel_out - -about: - home: https://github.com/dnncha/dotmatch - license: Apache-2.0 - license_file: LICENSE - summary: Deterministic short-DNA known-target assignment - description: | - DotMatch is a deterministic known-target short-DNA assignment engine for - CRISPR guides, barcodes, primers, panels, and whitelist-style target sets. - It is not a genome aligner and does not emit SAM/BAM or CIGAR output. - dev_url: https://github.com/dnncha/dotmatch - doc_url: https://dotmatch.readthedocs.io/ - -extra: - additional-platforms: - - osx-arm64 - recipe-maintainers: - - dnncha - - - dotmatch dist ACGT AGGT | grep '^1$' - - dotmatch leq 1 ACGT AGGT | grep '^true$' - - "dotmatch citation | grep 'Citation metadata: CITATION.cff'" - - dotmatch --help | grep 'Workflow namespaces' - - dotmatch count --help | grep 'Hamming supports k=0..3' - - dotmatch crispr-count --help | grep 'MAGeCK-ready' - - dotmatch audit --help | grep 'safe_at_hamming_k3' - - dotmatch assay --help | grep 'dotmatch assay' - - dotmatch barcode --help | grep 'dotmatch barcode' - - dotmatch panel --help | grep 'dotmatch panel' - - test -f "${PREFIX}/include/qdalign.h" - - test -f "${PREFIX}/lib/libdotmatch.a" - - test -f "${PREFIX}/lib/libdotmatch.so" || test -f "${PREFIX}/lib/libdotmatch.dylib" - - dotmatch assay init --template crispr --out assay.toml - - grep 'assay_type = "crispr"' assay.toml - - printf 'target_id\ttarget_seq\nbc0\tACGT\n' > targets.tsv - - printf '@r0\nACGT\n+\nIIII\n' > reads.fastq - - dotmatch count --targets targets.tsv --reads reads.fastq --sample-label sample --target-start 0 --target-length 4 --k 0 --metric hamming --out counts.tsv - - awk -F '\t' 'NR==2 { exit !($1=="bc0" && $2=="ACGT" && $3=="" && $4=="0" && $5=="1" && $10=="1") }' counts.tsv - - printf 'target_id\ttarget_seq\na\tAAAAAAAA\nb\tCCCCCCCC\n' > audit_targets.tsv - - dotmatch audit --targets audit_targets.tsv --k 3 --audit-mode exact --out-dir audit_out - - awk -F '\t' '$1=="safe_at_hamming_k2" { exit !($2=="yes") }' audit_out/audit_summary.tsv - - awk -F '\t' '$1=="safe_at_hamming_k3" { exit !($2=="yes") }' audit_out/audit_summary.tsv - - chmod -R a+rwX audit_out - - printf 'fastq_path,sample\nreads.fastq,treated\n' > samples.csv - - printf 'sgRNA,sgRNA_sequence,gene_symbol\ng0,ACGT,GENE0\n' > crispr_guides.csv - - dotmatch crispr-count --library crispr_guides.csv --samples samples.csv --guide-start 0 --guide-length 4 --k 2 --metric hamming --ambiguity-policy best --out crispr_counts.tsv --summary crispr_summary.json --sample-qc crispr_sample_qc.tsv - - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' crispr_counts.tsv - - awk -F '\t' 'NR==2 { exit !($1=="treated" && $2=="reads.fastq" && $3=="1" && $4=="1" && $5=="1" && $6=="1") }' crispr_sample_qc.tsv - - printf 'guide\tbases\tgene\ng0\tACGT\tGENE0\n' > guides.tsv - - dotmatch guide-counter count --input reads.fastq --samples sample --library guides.tsv --offset-sample-size 1 --offset-min-fraction 0.1 --output gc_out - - awk -F '\t' 'NR==2 { exit !($1=="g0" && $2=="GENE0" && $3=="1") }' gc_out.counts.txt - - awk -F '\t' 'NR==1 { exit !($1=="guide" && $2=="gene" && $3=="guide_type" && $4=="sample") }' gc_out.extended-counts.txt - - awk -F '\t' 'NR==2 { exit !($1=="reads.fastq" && $2=="sample" && $3=="1" && $4=="1" && $5=="1" && $6=="1.0000") }' gc_out.stats.txt - - printf 'barcode_id\tbarcode_seq\ns1\tACGT\ns2\tTTTT\n' > barcodes.tsv - - printf '@r1\nNACGTAAAA\n+\nIIIIIIIII\n@r2\nNTTTTAAAA\n+\nIIIIIIIII\n' > barcode_reads.fastq - - dotmatch barcode infer --barcodes barcodes.tsv --reads barcode_reads.fastq --scan-starts 0:2 --barcode-length 4 --sample-reads 10 --out offset_scan.tsv --summary barcode_summary.json - - "grep '\"recommended_start\": 1' barcode_summary.json" - - dotmatch panel design --n 2 --length 4 --candidate-pool-size 100 --restarts 1 --min-hamming-distance 2 --min-levenshtein-distance 2 --out-dir panel_out - - test -f panel_out/barcodes.tsv - - test -f panel_out/design_report.json - - chmod -R a+rwX panel_out - -about: - home: https://github.com/dnncha/dotmatch - license: Apache-2.0 - license_file: LICENSE - summary: Deterministic short-DNA known-target assignment - description: | - DotMatch is a deterministic known-target short-DNA assignment engine for - CRISPR guides, barcodes, primers, panels, and whitelist-style target sets. - It is not a genome aligner and does not emit SAM/BAM or CIGAR output. - dev_url: https://github.com/dnncha/dotmatch - doc_url: https://dotmatch.readthedocs.io/ - extra: additional-platforms: - osx-arm64 From b6f7b7c4651a1793ebe98bc7815ed8c793c238a7 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:30:18 +0100 Subject: [PATCH 30/93] Prepare AssayCode 0.2.0 release metadata --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 92f54c1b..a2601c10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dotmatch" -version = "0.1.9" +version = "0.2.0" description = "Deterministic known-target short-DNA assignment for CRISPR guide counting, barcode demultiplexing, and FASTQ workflows" readme = "README.md" requires-python = ">=3.9" From 94b092c7694941d941e778f3d363ac3ea6a3e615 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:30:25 +0100 Subject: [PATCH 31/93] Prepare AssayCode 0.2.0 release metadata --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 53145e46..df471e2a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dotmatch-site", - "version": "0.1.9", + "version": "0.2.0", "private": true, "scripts": { "dev": "next dev", From 150dd83fb4c7d7eccef1df5cc3efaf49cbe20090 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:30:32 +0100 Subject: [PATCH 32/93] Prepare AssayCode 0.2.0 release metadata --- codemeta.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codemeta.json b/codemeta.json index 97b81c35..e8e2cf11 100644 --- a/codemeta.json +++ b/codemeta.json @@ -9,8 +9,8 @@ "issueTracker": "https://github.com/dnncha/dotmatch/issues", "license": "https://spdx.org/licenses/Apache-2.0", "identifier": "https://doi.org/10.5281/zenodo.20541628", - "version": "0.1.9", - "softwareVersion": "0.1.9", + "version": "0.2.0", + "softwareVersion": "0.2.0", "programmingLanguage": [ "C", "Python", From 5dd8f5d3e521b07e07d77a7f40c803acc77682aa Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:30:39 +0100 Subject: [PATCH 33/93] Prepare AssayCode 0.2.0 release metadata --- .zenodo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zenodo.json b/.zenodo.json index 282ef8f1..14540baf 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,7 +1,7 @@ { "title": "DotMatch: deterministic known-target short-DNA assignment for sequencing workflows", "upload_type": "software", - "version": "0.1.9", + "version": "0.2.0", "conceptdoi": "10.5281/zenodo.20541628", "creators": [ { From 7907c92c54f213c41bcb51fba9e86e7bdfad02dc Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:31:05 +0100 Subject: [PATCH 34/93] Prepare AssayCode 0.2.0 release metadata --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index c17296c3..69db8b8d 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,7 +8,7 @@ authors: orcid: "https://orcid.org/0009-0003-5012-7229" repository-code: "https://github.com/dnncha/dotmatch" license: Apache-2.0 -version: "0.1.9" +version: "0.2.0" doi: 10.5281/zenodo.20541628 abstract: "DotMatch is a deterministic known-target short-DNA assignment engine for CRISPR guide counting, barcode demultiplexing, and fixed-target FASTQ workflows." keywords: From d95069b90be3811a24652c3a0371f0f5699ace04 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:31:14 +0100 Subject: [PATCH 35/93] Prepare AssayCode 0.2.0 release metadata --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c93415bc..5c209f04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ LABEL org.opencontainers.image.title="DotMatch" \ org.opencontainers.image.source="https://github.com/dnncha/dotmatch" \ org.opencontainers.image.url="https://dotmatch.readthedocs.io/" \ org.opencontainers.image.documentation="https://dotmatch.readthedocs.io/" \ - org.opencontainers.image.version="0.1.9" \ + org.opencontainers.image.version="0.2.0" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.authors="Donncha O'Toole" From 309d6f93971dbfa6c9fc628147a77756b58930ba Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:31:24 +0100 Subject: [PATCH 36/93] Prepare AssayCode 0.2.0 release metadata --- include/qdalign.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/qdalign.h b/include/qdalign.h index 7b78d4de..7a318787 100644 --- a/include/qdalign.h +++ b/include/qdalign.h @@ -7,7 +7,7 @@ extern "C" { #endif -#define QDALN_VERSION "0.1.9" +#define QDALN_VERSION "0.2.0" #define QDALN_ALPHABET_POLICY "literal-byte; A/C/G/T/N/IUPAC symbols are ordinary byte symbols; no wildcard expansion" enum qdaln_match_status { From ed88204d629f140bb50ad584b162a6dc4c9b89bf Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:31:30 +0100 Subject: [PATCH 37/93] Prepare AssayCode 0.2.0 release metadata --- packaging/bioconda/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/bioconda/meta.yaml b/packaging/bioconda/meta.yaml index 49b543ab..0fe8b8e2 100644 --- a/packaging/bioconda/meta.yaml +++ b/packaging/bioconda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "dotmatch" %} -{% set version = "0.1.9" %} +{% set version = "0.2.0" %} {% set sha256 = "REPLACE_WITH_RELEASE_TARBALL_SHA256" %} package: From c653ab903be01653ed1c0cc26f99da48129f8c63 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:31:37 +0100 Subject: [PATCH 38/93] Prepare AssayCode 0.2.0 release metadata --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 1c592cfa..12f0b0da 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = "DotMatch" author = "Donncha O'Toole" copyright = "2026, Donncha O'Toole" -release = "0.1.9" +release = "0.2.0" version = release extensions = [ From 62c0ce2225a1b702f44e465eddeda51b7a9be546 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:31:45 +0100 Subject: [PATCH 39/93] Prepare AssayCode 0.2.0 release metadata --- paper/paper.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper/paper.bib b/paper/paper.bib index 43306a9d..f63ffa28 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -1,7 +1,7 @@ @software{dotmatch_zenodo_017, author = {{O'Toole}, Donncha}, title = {{DotMatch: deterministic known-target short-DNA assignment for sequencing workflows}}, - version = {0.1.9}, + version = {0.2.0}, date = {2026-06-04}, publisher = {Zenodo}, doi = {10.5281/zenodo.20541628}, From 45ff5ab3b45fba50d4c4df706ede22c8cd84701f Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:32:00 +0100 Subject: [PATCH 40/93] Align AssayCode 0.2.0 version --- python/dotmatch/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dotmatch/__init__.py b/python/dotmatch/__init__.py index 11749527..2a062538 100644 --- a/python/dotmatch/__init__.py +++ b/python/dotmatch/__init__.py @@ -50,7 +50,7 @@ def _source_tree_version() -> Optional[str]: try: __version__ = _source_tree_version() or _metadata_version("dotmatch") except PackageNotFoundError: - __version__ = "0.1.9" + __version__ = "0.2.0" __all__ = [ "__version__", From c610d0e5d5f89550a3f43d9e0a713703016e390a Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:32:10 +0100 Subject: [PATCH 41/93] Align AssayCode 0.2.0 version --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b96ece17..c6fb63ea 100644 --- a/README.md +++ b/README.md @@ -291,13 +291,13 @@ documented scope are especially welcome. ## Installation -DotMatch 0.1.9 is the current release target. After the tagged release workflow +DotMatch 0.2.0 is the current release target. After the tagged release workflow publishes the package and `make distribution-channels` verifies the channel, the PyPI package includes the `dotmatch` command, Python imports, and the bundled native library. ```bash -python3 -m pip install dotmatch==0.1.9 +python3 -m pip install dotmatch==0.2.0 assaycode --version dotmatch --version dotmatch dist ACGT AGGT @@ -331,14 +331,14 @@ docker build -t dotmatch:dev . docker run --rm -v "$PWD:/work" dotmatch:dev dist ACGT AGGT ``` -Bioconda is the Conda-based bioinformatics install path. The 0.1.9 recipe update +Bioconda is the Conda-based bioinformatics install path. The 0.2.0 recipe update keeps `linux-64`, `osx-64`, and `osx-arm64` support, including Apple Silicon Macs. Treat the install command as released only after the Bioconda recipe update is accepted and `make distribution-channels` verifies that Bioconda repodata and a clean `conda create` both resolve the package: ```bash -conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.1.9 +conda create -n dotmatch -c conda-forge -c bioconda dotmatch=0.2.0 conda activate dotmatch dotmatch --version ``` @@ -350,7 +350,7 @@ in [Packaging Notes](docs/packaging.md), the available for a release after `make distribution-channels` verifies public metadata and install smoke tests. -The tagged release workflow publishes the 0.1.9 source distribution, native +The tagged release workflow publishes the 0.2.0 source distribution, native macOS wheel, and repaired manylinux/musllinux Linux wheels. PyPI trusted publishing is configured for that workflow. The GitHub release workflow builds and smoke-tests repaired manylinux/musllinux wheels before upload. PyPI wheel @@ -363,9 +363,9 @@ behavior, and DOI evidence. Raw to PyPI. BioContainers publication is expected through the Bioconda automation rather -than a separate DotMatch container submission. After the accepted Bioconda 0.1.9 +than a separate DotMatch container submission. After the accepted Bioconda 0.2.0 package is converted by BioContainers, the expected image tag shape is -`quay.io/biocontainers/dotmatch:0.1.9--`. +`quay.io/biocontainers/dotmatch:0.2.0--`. Bioconda provides the `dotmatch` command-line tool, Python workflow namespaces, Python imports, and C header/library artifacts for the published package @@ -824,7 +824,7 @@ A short JOSS software-paper draft is available in [paper/paper.md](paper/paper.m @software{dotmatch_software, author = {{O'Toole}, Donncha}, title = {{DotMatch: deterministic known-target short-DNA assignment for sequencing workflows}}, - version = {0.1.9}, + version = {0.2.0}, date = {2026-06-04}, publisher = {Zenodo}, doi = {10.5281/zenodo.20541628}, From ab5ed476f7d7dce5217d16073a338f30bda897f5 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:32:17 +0100 Subject: [PATCH 42/93] Align AssayCode 0.2.0 version --- docs/packaging.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/packaging.md b/docs/packaging.md index d353c8c8..c9f49f3a 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -18,7 +18,7 @@ The same verifier also builds the sdist, confirms it contains `src/qdalign.c` an For PyPI, upload the sdist plus the native macOS wheel built on GitHub Actions. Linux binary wheels should go to PyPI only after they are built or repaired as manylinux/musllinux wheels. The release workflow builds repaired Linux wheel artifacts with cibuildwheel for `manylinux_x86_64` and `musllinux_x86_64`, smoke-tests `import dotmatch`, the installed console script, and `dotmatch dist ACGT AGGT`, and uploads them as GitHub release artifacts. Do not upload a raw `linux_x86_64` wheel to PyPI. -DotMatch 0.1.9 is the current release target; the `v0.1.9` release workflow publishes the source distribution, the native macOS wheel, and repaired manylinux/musllinux Linux wheels. The release workflow +DotMatch 0.2.0 is the current release target; the `v0.2.0` release workflow publishes the source distribution, the native macOS wheel, and repaired manylinux/musllinux Linux wheels. The release workflow uses PyPI trusted publishing from repository `dnncha/dotmatch`, workflow `.github/workflows/release.yml`, and environment `pypi`; if that publisher is missing or mismatched, the build artifacts are created but the publish job fails @@ -37,7 +37,7 @@ not upload a Conda package directly. published DotMatch 0.1.2 as the first Bioconda package. [bioconda/bioconda-recipes#66291](https://github.com/bioconda/bioconda-recipes/pull/66291) merged the DotMatch 0.1.8 update on 2026-06-17. The public Anaconda page now -shows 0.1.9, but immutable source, checksum, repodata, and clean-install +shows 0.2.0, but immutable source, checksum, repodata, and clean-install evidence for that release still needs to be captured. Treat the Bioconda versions as available only after `https://anaconda.org/bioconda/dotmatch`, repodata, and the install smoke tests @@ -89,10 +89,10 @@ The native CLI exposes `dotmatch --version`, so the Bioconda recipe and post-release Bioconda install verifier should check version output as well as functional CLI smoke tests. -### Bioconda 0.1.9 PR changelog draft +### Bioconda 0.2.0 PR changelog draft -- Update DotMatch from the latest accepted Bioconda version to 0.1.9. -- Use the immutable v0.1.9 tag and replace the SHA256 after the release tarball +- Update DotMatch from the latest accepted Bioconda version to 0.2.0. +- Use the immutable v0.2.0 tag and replace the SHA256 after the release tarball is available. - Keep the Python console-script package scope introduced in 0.1.4: `dotmatch` exposes the native commands plus `assay`, `barcode`, `panel`, and @@ -129,15 +129,15 @@ images to `ghcr.io/dnncha/dotmatch`. BioContainers images for DotMatch are generated from the accepted Bioconda recipe; there is no separate DotMatch Dockerfile to submit to BioContainers for -the normal release path. The 0.1.9 image is expected only after the Bioconda +the normal release path. The 0.2.0 image is expected only after the Bioconda recipe is accepted and propagated. The remaining local check is Docker-backed manifest/runtime verification: ```bash -python3 scripts/check_distribution_channels.py --version 0.1.9 -docker pull quay.io/biocontainers/dotmatch:0.1.9-- -docker run --rm quay.io/biocontainers/dotmatch:0.1.9-- dotmatch dist ACGT AGGT -docker run --rm quay.io/biocontainers/dotmatch:0.1.9-- dotmatch leq 1 ACGT AGGT +python3 scripts/check_distribution_channels.py --version 0.2.0 +docker pull quay.io/biocontainers/dotmatch:0.2.0-- +docker run --rm quay.io/biocontainers/dotmatch:0.2.0-- dotmatch dist ACGT AGGT +docker run --rm quay.io/biocontainers/dotmatch:0.2.0-- dotmatch leq 1 ACGT AGGT ``` Do not publish a manual BioContainers image for DotMatch unless the Bioconda @@ -188,7 +188,7 @@ publication has actually happened. The repository includes `.zenodo.json` metadata for tagged software archives. General software citation uses DOI `10.5281/zenodo.20541628`, which resolves -through Zenodo metadata for DotMatch. The 0.1.9 archive metadata remains a +through Zenodo metadata for DotMatch. The 0.2.0 archive metadata remains a post-tag verification step. Version DOI `10.5281/zenodo.20541629` belongs to v0.1.7 and is retained only as explicit -version-specific provenance, not as the v0.1.9 DOI. +version-specific provenance, not as the v0.2.0 DOI. From f6c34edb85dfc59a6b6b827bb3a22322a7f9e0b0 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:32:45 +0100 Subject: [PATCH 43/93] Prepare 0.2.0 distribution record --- docs/distribution-release.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/distribution-release.json b/docs/distribution-release.json index 617fb6cc..91862cd3 100644 --- a/docs/distribution-release.json +++ b/docs/distribution-release.json @@ -1,16 +1,16 @@ { "schema_version": 1, "status": "not_released", - "release_version": "0.1.9", + "release_version": "0.2.0", "post_release_gate": "make distribution-channels", "channels": [ { "id": "pypi", "status": "prepared", - "expected_url": "https://pypi.org/project/dotmatch/0.1.9/", + "expected_url": "https://pypi.org/project/dotmatch/0.2.0/", "verification_command": "make distribution-channels", - "blocker": "The 0.1.9 source distribution and repaired wheel artifacts have not been published by the tagged release workflow yet.", - "next_action": "Tag v0.1.9, let the GitHub release workflow publish the source distribution and repaired wheels, then rerun make distribution-channels." + "blocker": "The AssayCode/DotMatch 0.2.0 source distribution and repaired wheel artifacts require a tagged immutable release.", + "next_action": "Pass make pretag-ready, tag v0.2.0, publish the source distribution and repaired wheels, then rerun make distribution-channels." }, { "id": "bioconda", @@ -21,36 +21,36 @@ "observed_date": "2026-07-13", "initial_recipe_pr_url": "https://github.com/bioconda/bioconda-recipes/pull/65367", "verification_command": "make distribution-channels", - "blocker": "The public package page shows 0.1.9, but this record still needs immutable source/SHA256 and clean repodata-install evidence before calling the channel verified.", - "next_action": "Capture the v0.1.9 source SHA256, verify repodata and a clean conda create, then rerun make distribution-channels and update this channel record." + "blocker": "Bioconda currently publishes dotmatch 0.1.9. Version 0.2.0 needs an immutable release tarball checksum and a recipe PR that smoke-tests both dotmatch and assaycode.", + "next_action": "After v0.2.0 is tagged, compute its SHA256, copy the prepared recipe into dnncha/bioconda-recipes, open the upstream recipe PR, and verify a clean conda create." }, { "id": "ghcr", "status": "prepared", "expected_url": "https://github.com/dnncha/dotmatch/pkgs/container/dotmatch", "verification_command": "make distribution-channels", - "blocker": "The 0.1.9 GHCR image has not been published and runtime-smoke-tested yet.", - "next_action": "Let the release workflow publish ghcr.io/dnncha/dotmatch:v0.1.9, then verify --version and CLI distance smoke tests with an OCI runtime." + "blocker": "The 0.2.0 GHCR image has not been published or runtime-smoke-tested.", + "next_action": "Publish ghcr.io/dnncha/dotmatch:v0.2.0 from the tagged workflow, then test assaycode and dotmatch identities." }, { "id": "biocontainers", "status": "prepared", "expected_url": "https://quay.io/repository/biocontainers/dotmatch", "verification_command": "make distribution-channels", - "blocker": "BioContainers 0.1.9 cannot propagate until the Bioconda recipe is accepted.", - "next_action": "After Bioconda publishes 0.1.9, verify the selected quay.io/biocontainers/dotmatch:0.1.9-- image with CLI distance and threshold smoke tests." + "blocker": "BioContainers 0.2.0 cannot propagate until the Bioconda 0.2.0 recipe is accepted.", + "next_action": "After Bioconda publishes 0.2.0, smoke-test the selected quay.io/biocontainers/dotmatch:0.2.0-- image." }, { "id": "zenodo", "status": "prepared", "expected_url": "https://zenodo.org/records/20541628", "verification_command": "make distribution-channels", - "blocker": "The Zenodo archive metadata for v0.1.9 has not been verified yet.", - "next_action": "After v0.1.9 is tagged and archived, verify the Zenodo record metadata and DOI resolution, then rerun make distribution-channels." + "blocker": "The Zenodo archive for v0.2.0 does not exist before the immutable tag.", + "next_action": "Tag v0.2.0, verify the archived release metadata and DOI resolution, then rerun make distribution-channels." } ], "blockers": [ - "DotMatch 0.1.9 has not been tagged, published to PyPI/GHCR/Zenodo, accepted by Bioconda, or propagated to BioContainers yet." + "AssayCode/DotMatch 0.2.0 is not tagged; PyPI, GHCR, Zenodo, Bioconda, and BioContainers publication cannot be claimed before immutable artifacts and channel verification exist." ], - "next_action": "Run make pretag-ready, tag v0.1.9, monitor the release workflow, submit the Bioconda recipe update, and rerun make distribution-channels after public channels propagate." + "next_action": "Pass CI and make pretag-ready, tag v0.2.0, monitor release publication, submit the prepared Bioconda recipe update, and rerun make distribution-channels." } From 1ab824e94b81cecdcc2340943ca6990bed0fa107 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:33:07 +0100 Subject: [PATCH 44/93] Align site lockfile with 0.2.0 --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2fbfee2d..39889283 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dotmatch-site", - "version": "0.1.9", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dotmatch-site", - "version": "0.1.9", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { "next": "^16.2.7", From 465915a4a4577a1eb47d91ecbd9faa42be35945a Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:33:27 +0100 Subject: [PATCH 45/93] Document AssayCode 0.2.0 changes --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e983b3d3..cbc593cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,16 @@ All notable user-facing changes are tracked here. Public statements in release notes must stay aligned with `docs/scientific-claims.md`. -## Unreleased / Next (post 0.1.7) +## 0.2.0 - Unreleased + +### AssayCode platform + +- Added the additive `assaycode` CLI and Python namespace while preserving the `dotmatch` package, executable, ABI, DOI, schemas, and citation contract. +- Added AssayScript v2 compilation for R1/R2/I1/I2 segment declarations, fixed or anchored extraction, positional jitter, orientation, per-segment matching policies, allowed-combination tables, source and library fingerprints, bounded safety findings, deterministic strategy selection, and portable JSON plans. +- Added `assaycode compile`, `assaycode inspect`, and compatibility-safe assay workflow shortcuts. +- Added an explicitly experimental calibration module with per-cycle error fitting, Phred shrinkage, selective posterior calls, likelihood-ratio abstention, joint decoding over permitted tuples, Brier score, expected calibration error, held-out FDR threshold selection, and smoothed abundance priors. +- Added `assaycode watch`, a bounded-memory JSONL monitor that emits assignment-rate confidence intervals and threshold-based sequential QC decisions. +- Added a release-blocking AssayCode readiness gate, focused tests, updated scientific claim boundaries, a rewritten paper, and Bioconda smoke tests for both command identities. ### Added - `dotmatch.tl` submodule: scverse/scanpy-style tools (`tl.assign_features`, `tl.feature_counts`, aliases for CRISPR/feature barcodes). In-place modification or copy, with provenance in `.uns`. Excellent UX for AnnData-centric workflows. From 0c3cf759cae5bc4ba67734372765713d5e083e1d Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:34:37 +0100 Subject: [PATCH 46/93] Correct 0.2.0 packaging and Bioconda handoff --- docs/packaging.md | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/docs/packaging.md b/docs/packaging.md index c9f49f3a..3cdf753a 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -36,9 +36,9 @@ not upload a Conda package directly. [bioconda/bioconda-recipes#65367](https://github.com/bioconda/bioconda-recipes/pull/65367) published DotMatch 0.1.2 as the first Bioconda package. [bioconda/bioconda-recipes#66291](https://github.com/bioconda/bioconda-recipes/pull/66291) -merged the DotMatch 0.1.8 update on 2026-06-17. The public Anaconda page now -shows 0.2.0, but immutable source, checksum, repodata, and clean-install -evidence for that release still needs to be captured. Treat the Bioconda +merged the DotMatch 0.1.8 update on 2026-06-17. The public Anaconda page currently shows 0.1.9. AssayCode is targeted for the +prepared 0.2.0 update, which still needs an immutable source tag, checksum, +recipe review, repodata visibility, and clean-install evidence. Treat the Bioconda versions as available only after `https://anaconda.org/bioconda/dotmatch`, repodata, and the install smoke tests in `make distribution-channels` all verify the release version. @@ -64,9 +64,9 @@ future upstream recipe updates unless Bioconda CI demonstrates a platform-specific blocker and the release notes clearly document that `osx-arm64` is unavailable. -The current Bioconda recipe installs the Python `dotmatch` console script as -the user-facing command, with the native executable bundled inside the Python -package as `dotmatch-native`. It also installs the public C header, static +The prepared 0.2.0 Bioconda recipe installs both the additive `assaycode` +platform command and the compatibility-stable `dotmatch` command, with the +native executable bundled inside the Python package as `dotmatch-native`. It also installs the public C header, static library, shared library, and license. Workbench and browser assets remain outside the Bioconda recipe. @@ -91,25 +91,18 @@ functional CLI smoke tests. ### Bioconda 0.2.0 PR changelog draft -- Update DotMatch from the latest accepted Bioconda version to 0.2.0. -- Use the immutable v0.2.0 tag and replace the SHA256 after the release tarball - is available. -- Keep the Python console-script package scope introduced in 0.1.4: `dotmatch` - exposes the native commands plus `assay`, `barcode`, `panel`, and - GuideCounter-compatible CRISPR counting namespaces. -- Add Hamming `k=2`/`k=3` guide-counting support and exact audit safety fields; - keep larger-radius claims bounded to same-length Hamming fixed-window - assignment. -- Add native exact-table shortcuts, indexed status paths, and bounded k=2 - single-unknown status stops for the CRISPR/counting hot paths. -- Add installed-package smoke tests for `dotmatch count --help`, - `dotmatch crispr-count --help`, `dotmatch audit --help`, Hamming `k=2` - CRISPR counting, exact Hamming `k=3` audit summaries, GuideCounter-compatible - counts/extended-counts/stats files, barcode offset inference, and panel - design. -- Opt into `osx-arm64` builds with `extra.additional-platforms`. -- Keep host `zlib` for FASTQ.gz/native linkage and let Conda export `libzlib` - at runtime. +- Update the existing `dotmatch` package from the published 0.1.9 build to + 0.2.0; do not create or rename to a second Conda package. +- Use the immutable v0.2.0 tag and replace the SHA256 placeholder only after the + release tarball exists. +- Install and smoke-test both `dotmatch` and `assaycode`; verify that the + AssayCode Python namespace exposes the same DotMatch engine and version. +- Include AssayScript v2 compilation, experimental calibrated/joint decoding, + bounded-memory sequential QC, the rewritten paper, and explicit experimental + claim boundaries. +- Preserve the native commands, C header/static/shared libraries, workflow + namespaces, GuideCounter compatibility, Hamming k=2/k=3 audit tests, + `osx-arm64` opt-in, and host-zlib linkage. ## Docker From bca13abcfa88b5624f385c203594ab96620b8fcf Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:34:58 +0100 Subject: [PATCH 47/93] Align preferred citation with rewritten paper --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 69db8b8d..ce1fe7c6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -21,7 +21,7 @@ keywords: - edit distance preferred-citation: type: article - title: "DotMatch: deterministic known-target short-DNA assignment for sequencing workflows" + title: "DotMatch: ambiguity-aware known-target DNA assignment and auditable assay workflows" authors: - given-names: "Donncha" family-names: "O'Toole" From 015622e19b97b6f2bd33b4d1c8bb47af036a16d5 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:35:55 +0100 Subject: [PATCH 48/93] Add calibrated decoding file interfaces --- python/dotmatch/calibration_io.py | 147 ++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 python/dotmatch/calibration_io.py diff --git a/python/dotmatch/calibration_io.py b/python/dotmatch/calibration_io.py new file mode 100644 index 00000000..87dd87f5 --- /dev/null +++ b/python/dotmatch/calibration_io.py @@ -0,0 +1,147 @@ +"""File interfaces for experimental calibrated decoding.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any, Mapping + +from .calibration import ErrorModel, decode, fit_error_model + + +def fit_model_tsv(path: str | Path, *, prior_strength: float = 100.0) -> ErrorModel: + rows = _dict_rows(path) + observations: list[tuple[str, str, str]] = [] + for line_number, row in enumerate(rows, start=2): + try: + observed = row["observed"].strip().upper() + expected = row["expected"].strip().upper() + quality = row["quality"].strip() + except KeyError as exc: + raise ValueError("training TSV requires observed, expected, and quality columns") from exc + if not observed or not expected or not quality: + raise ValueError(f"training TSV line {line_number} contains an empty required value") + observations.append((observed, expected, quality)) + if not observations: + raise ValueError("training TSV contains no observations") + return fit_error_model(observations, prior_strength=prior_strength) + + +def write_model(model: ErrorModel, path: str | Path) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema_version": 1, + "status": "experimental", + **model.to_dict(), + } + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return output + + +def read_model(path: str | Path) -> ErrorModel: + data = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise ValueError("unsupported calibration model schema") + substitutions = data.get("substitution_counts") + if not isinstance(substitutions, dict): + raise ValueError("calibration model substitution_counts must be an object") + normalized: dict[str, dict[str, int]] = {} + for truth, row in substitutions.items(): + if not isinstance(truth, str) or not isinstance(row, dict): + raise ValueError("invalid calibration substitution row") + normalized[truth] = {str(called): int(count) for called, count in row.items()} + return ErrorModel( + cycle_totals=tuple(int(value) for value in data["cycle_totals"]), + cycle_errors=tuple(int(value) for value in data["cycle_errors"]), + substitution_counts=normalized, + prior_strength=float(data["prior_strength"]), + ) + + +def decode_tsv( + reads_path: str | Path, + targets_path: str | Path, + model: ErrorModel, + output_path: str | Path, + *, + posterior_min: float = 0.99, + likelihood_ratio_min: float = 10.0, +) -> dict[str, int]: + targets = _read_targets(targets_path) + rows = _dict_rows(reads_path) + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + summary = {"total": 0, "unique": 0, "ambiguous": 0, "none": 0} + with output.open("wt", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + delimiter="\t", + lineterminator="\n", + fieldnames=[ + "read_id", + "observed", + "target", + "status", + "posterior", + "second_posterior", + "likelihood_ratio", + ], + ) + writer.writeheader() + for line_number, row in enumerate(rows, start=2): + observed = str(row.get("observed", "")).strip().upper() + quality = str(row.get("quality", "")).strip() + read_id = str(row.get("read_id", f"read_{line_number - 2}")).strip() + if not observed or not quality: + raise ValueError(f"reads TSV line {line_number} requires observed and quality") + call = decode( + observed, + quality, + targets, + model, + posterior_min=posterior_min, + likelihood_ratio_min=likelihood_ratio_min, + ) + summary["total"] += 1 + summary[call.status] += 1 + writer.writerow( + { + "read_id": read_id, + "observed": observed, + "target": call.target or "", + "status": call.status, + "posterior": f"{call.posterior:.12g}", + "second_posterior": f"{call.second_posterior:.12g}", + "likelihood_ratio": "inf" + if call.likelihood_ratio == float("inf") + else f"{call.likelihood_ratio:.12g}", + } + ) + return summary + + +def _dict_rows(path: str | Path) -> list[dict[str, str]]: + with Path(path).open("rt", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + if reader.fieldnames is None: + raise ValueError(f"{path}: missing TSV header") + return [dict(row) for row in reader] + + +def _read_targets(path: str | Path) -> list[str]: + rows = _dict_rows(path) + if not rows: + raise ValueError("target table contains no targets") + candidates = ["target_seq", "guide_seq", "barcode_seq", "sequence", "seq"] + fieldnames = list(rows[0]) + sequence_column = next((column for column in candidates if column in fieldnames), None) + if sequence_column is None: + if len(fieldnames) < 2: + raise ValueError("target table requires a sequence column") + sequence_column = fieldnames[1] + targets = [str(row.get(sequence_column, "")).strip().upper() for row in rows] + if any(not target for target in targets): + raise ValueError("target table contains an empty sequence") + return targets From 730665cc9fe3da3acf38ed3f79321fe57d686bf6 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:36:25 +0100 Subject: [PATCH 49/93] Expose calibrated AssayCode decoding workflow --- python/assaycode/cli.py | 63 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/python/assaycode/cli.py b/python/assaycode/cli.py index 9badad23..7a2876a4 100644 --- a/python/assaycode/cli.py +++ b/python/assaycode/cli.py @@ -12,6 +12,7 @@ from dotmatch import cli as _engine_cli from dotmatch.assayscript import AssayScriptError, load_and_compile, write_compiled_plan from dotmatch.assaywatch import WatchThresholds, watch_jsonl +from dotmatch.calibration_io import decode_tsv, fit_model_tsv, read_model, write_model _SHORTCUTS = {"new", "infer", "check", "plan", "run", "start"} @@ -77,6 +78,60 @@ def command_inspect(argv: Sequence[str]) -> int: + +def command_calibrate(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="assaycode calibrate", + description="Fit an experimental per-cycle error model from independently trusted TSV pairs.", + ) + parser.add_argument("training", help="TSV with observed, expected, and quality columns") + parser.add_argument("--out", required=True, help="model JSON") + parser.add_argument("--prior-strength", type=float, default=100.0) + args = parser.parse_args(list(argv)) + try: + model = fit_model_tsv(args.training, prior_strength=args.prior_strength) + output = write_model(model, args.out) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"assaycode calibrate: {exc}", file=sys.stderr) + return 2 + print(json.dumps({ + "status": "experimental", + "model": str(output), + "cycles": len(model.cycle_totals), + "observations": sum(model.cycle_totals), + "errors": sum(model.cycle_errors), + }, indent=2, sort_keys=True)) + return 0 + + +def command_decode_quality(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="assaycode decode-quality", + description="Apply an experimental calibrated selective decoder to short read windows.", + ) + parser.add_argument("--reads", required=True, help="TSV with read_id, observed, and quality") + parser.add_argument("--targets", required=True, help="TSV target table") + parser.add_argument("--model", required=True, help="calibration model JSON") + parser.add_argument("--out", required=True, help="calls TSV") + parser.add_argument("--posterior-min", type=float, default=0.99) + parser.add_argument("--likelihood-ratio-min", type=float, default=10.0) + args = parser.parse_args(list(argv)) + try: + model = read_model(args.model) + summary = decode_tsv( + args.reads, + args.targets, + model, + args.out, + posterior_min=args.posterior_min, + likelihood_ratio_min=args.likelihood_ratio_min, + ) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"assaycode decode-quality: {exc}", file=sys.stderr) + return 2 + print(json.dumps({"status": "experimental", **summary}, indent=2, sort_keys=True)) + return 0 + def command_watch(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser( prog="assaycode watch", @@ -124,6 +179,8 @@ def print_help() -> None: assaycode --version assaycode compile assay-v2.toml --out assay.plan.json assaycode inspect assay.plan.json + assaycode calibrate trusted.tsv --out error-model.json + assaycode decode-quality --reads windows.tsv --targets targets.tsv --model error-model.json --out calls.tsv assaycode watch assignments.jsonl --out snapshots.jsonl assaycode check assay.toml assaycode plan assay.toml @@ -135,6 +192,8 @@ def print_help() -> None: AssayScript v2: compile validate a multi-read specification and select deterministic strategies inspect summarize a compiled plan, safety status, fingerprints, and findings + calibrate fit an experimental cycle-error model from trusted pairs + decode-quality apply calibrated selective decoding with abstention watch stream assignment events into sequential QC decisions AssaySpec v1 workflow shortcuts: @@ -172,6 +231,10 @@ def main(argv: Sequence[str] | None = None) -> int: return command_compile(raw_args[1:]) if raw_args[0] == "inspect": return command_inspect(raw_args[1:]) + if raw_args[0] == "calibrate": + return command_calibrate(raw_args[1:]) + if raw_args[0] == "decode-quality": + return command_decode_quality(raw_args[1:]) if raw_args[0] == "watch": return command_watch(raw_args[1:]) if raw_args[0] == "engine": From 55ba6dc28d18f236d80e3db74e908c007fc81ec4 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:37:16 +0100 Subject: [PATCH 50/93] Test calibrated decoding CLI workflow --- python/tests/test_calibration_io.py | 105 ++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 python/tests/test_calibration_io.py diff --git a/python/tests/test_calibration_io.py b/python/tests/test_calibration_io.py new file mode 100644 index 00000000..3e5edc7e --- /dev/null +++ b/python/tests/test_calibration_io.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from assaycode import cli +from dotmatch.calibration_io import decode_tsv, fit_model_tsv, read_model, write_model + + +def _training(path: Path) -> Path: + path.write_text( + "observed\texpected\tquality\n" + + "".join("ACGT\tACGT\tIIII\n" for _ in range(20)) + + "AGGT\tACGT\tIIII\n", + encoding="utf-8", + ) + return path + + +def _targets(path: Path) -> Path: + path.write_text( + "target_id\ttarget_seq\nexact\tACGT\nneighbor\tAGGT\n", + encoding="utf-8", + ) + return path + + +def _reads(path: Path) -> Path: + path.write_text( + "read_id\tobserved\tquality\nr1\tACGT\tIIII\nr2\tANGT\t!!!!\n", + encoding="utf-8", + ) + return path + + +def test_calibration_model_round_trip(tmp_path: Path) -> None: + model = fit_model_tsv(_training(tmp_path / "training.tsv")) + output = write_model(model, tmp_path / "model.json") + restored = read_model(output) + + assert restored == model + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["status"] == "experimental" + assert payload["schema_version"] == 1 + + +def test_decode_tsv_writes_selective_calls(tmp_path: Path) -> None: + model = fit_model_tsv(_training(tmp_path / "training.tsv")) + output = tmp_path / "calls.tsv" + + summary = decode_tsv( + _reads(tmp_path / "reads.tsv"), + _targets(tmp_path / "targets.tsv"), + model, + output, + ) + + assert summary == {"total": 2, "unique": 1, "ambiguous": 1, "none": 0} + with output.open(encoding="utf-8") as handle: + rows = list(csv.DictReader(handle, delimiter="\t")) + assert rows[0]["target"] == "ACGT" + assert rows[0]["status"] == "unique" + assert rows[1]["target"] == "" + assert rows[1]["status"] == "ambiguous" + + +def test_assaycode_calibrate_and_decode_quality(tmp_path: Path, capsys) -> None: + model = tmp_path / "model.json" + assert cli.main( + [ + "calibrate", + str(_training(tmp_path / "training.tsv")), + "--out", + str(model), + ] + ) == 0 + calibration_summary = json.loads(capsys.readouterr().out) + assert calibration_summary["status"] == "experimental" + assert calibration_summary["cycles"] == 4 + + output = tmp_path / "calls.tsv" + assert cli.main( + [ + "decode-quality", + "--reads", + str(_reads(tmp_path / "reads.tsv")), + "--targets", + str(_targets(tmp_path / "targets.tsv")), + "--model", + str(model), + "--out", + str(output), + ] + ) == 0 + decode_summary = json.loads(capsys.readouterr().out) + assert decode_summary["unique"] == 1 + assert decode_summary["ambiguous"] == 1 + + +def test_training_tsv_requires_explicit_trusted_columns(tmp_path: Path) -> None: + bad = tmp_path / "bad.tsv" + bad.write_text("observed\texpected\nACGT\tACGT\n", encoding="utf-8") + + assert cli.main(["calibrate", str(bad), "--out", str(tmp_path / "model.json")]) == 2 From 357d8cc5e37cb05e25ea63a1f08e3b956e00de66 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:37:35 +0100 Subject: [PATCH 51/93] Gate calibrated AssayCode CLI workflow --- scripts/check_assaycode_readiness.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/check_assaycode_readiness.py b/scripts/check_assaycode_readiness.py index d17bb771..47053250 100644 --- a/scripts/check_assaycode_readiness.py +++ b/scripts/check_assaycode_readiness.py @@ -29,7 +29,7 @@ def main() -> int: ) errors += require( "python/assaycode/cli.py", - ["command_compile", "command_inspect", "command_watch"], + ["command_compile", "command_inspect", "command_calibrate", "command_decode_quality", "command_watch"], ) errors += require( "python/dotmatch/assayscript.py", @@ -45,6 +45,10 @@ def main() -> int: "This module is experimental", ], ) + errors += require( + "python/dotmatch/calibration_io.py", + ["fit_model_tsv", "write_model", "decode_tsv"], + ) errors += require( "python/dotmatch/assaywatch.py", ["class SequentialMonitor", "assignment_rate_interval95", "insufficient_data"], @@ -53,6 +57,7 @@ def main() -> int: "python/tests/test_assaycode_brand.py", "python/tests/test_assayscript.py", "python/tests/test_calibration.py", + "python/tests/test_calibration_io.py", "python/tests/test_assaywatch.py", ]: errors += require(test, ["test_"]) From f5aed18de8e0c83202e8b83eaa06985669d695a1 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:37:55 +0100 Subject: [PATCH 52/93] Document calibrated AssayCode evaluation CLI --- docs/assaycode.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/assaycode.md b/docs/assaycode.md index eb7fda5d..f49ad295 100644 --- a/docs/assaycode.md +++ b/docs/assaycode.md @@ -108,6 +108,22 @@ likelihood-ratio thresholds, joint inference over allowed assay combinations, Brier score, expected calibration error, and held-out FDR threshold selection. Deterministic DotMatch assignment remains the production default. +A file-backed evaluation path makes this testable without integrating it into +the production FASTQ router: + +```bash +assaycode calibrate trusted-observed-expected.tsv --out error-model.json +assaycode decode-quality \ + --reads held-out-windows.tsv \ + --targets targets.tsv \ + --model error-model.json \ + --out calls.tsv +``` + +Training rows must contain independently trusted `observed`, `expected`, and +`quality` columns. Calibration and false-discovery thresholds must be +evaluated on separate held-out truth data before any scientific claim. + ## Python Identity ```python From a080b4470dcefb26c688a1afacb49e97f9938f4a Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:41:20 +0100 Subject: [PATCH 53/93] Add deterministic assay panel simulation --- python/dotmatch/assaysim.py | 135 ++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 python/dotmatch/assaysim.py diff --git a/python/dotmatch/assaysim.py b/python/dotmatch/assaysim.py new file mode 100644 index 00000000..e3714a6a --- /dev/null +++ b/python/dotmatch/assaysim.py @@ -0,0 +1,135 @@ +"""Deterministic design-time simulation for known-target panels. + +This module is experimental. It estimates assignment behavior under an explicit +substitution-error model; it is not a substitute for held-out empirical +validation or a promise about a sequencing platform. +""" + +from __future__ import annotations + +import random +from dataclasses import asdict, dataclass +from typing import Mapping, Sequence + +DNA = "ACGT" + + +@dataclass(frozen=True) +class SimulationResult: + seed: int + total_reads: int + correct_unique: int + misassigned_unique: int + ambiguous: int + none: int + usable_yield: float + ambiguity_rate: float + no_call_rate: float + false_discovery_rate: float + confusion: Mapping[str, Mapping[str, int]] + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +def simulate_panel( + targets: Mapping[str, str], + *, + reads_per_target: int = 1000, + k: int = 1, + error_rate: float | Sequence[float] = 0.01, + seed: int = 1, +) -> SimulationResult: + """Simulate substitutions and assign each read by bounded Hamming distance. + + Every target receives the same number of reads. A call is unique only when + exactly one target is within the configured distance. Duplicate target + sequences are rejected because their truth labels are not identifiable. + """ + if not targets: + raise ValueError("targets must not be empty") + if isinstance(reads_per_target, bool) or not isinstance(reads_per_target, int) or reads_per_target <= 0: + raise ValueError("reads_per_target must be a positive integer") + if isinstance(k, bool) or not isinstance(k, int) or k < 0: + raise ValueError("k must be a non-negative integer") + + normalized = {str(name): str(sequence).upper() for name, sequence in targets.items()} + if len(normalized) != len(targets) or any(not name for name in normalized): + raise ValueError("target names must be unique and non-empty") + lengths = {len(sequence) for sequence in normalized.values()} + if len(lengths) != 1 or not lengths or 0 in lengths: + raise ValueError("all target sequences must have the same positive length") + if any(set(sequence) - set(DNA) for sequence in normalized.values()): + raise ValueError("target sequences must contain only A, C, G, and T") + if len(set(normalized.values())) != len(normalized): + raise ValueError("duplicate target sequences are not identifiable") + + length = next(iter(lengths)) + rates = _cycle_rates(error_rate, length) + rng = random.Random(seed) + counts = {"correct": 0, "misassigned": 0, "ambiguous": 0, "none": 0} + confusion: dict[str, dict[str, int]] = {name: {} for name in normalized} + + for truth_name, truth_sequence in normalized.items(): + for _ in range(reads_per_target): + observed = _mutate(truth_sequence, rates, rng) + candidates = [ + name + for name, target_sequence in normalized.items() + if _hamming(observed, target_sequence) <= k + ] + if not candidates: + counts["none"] += 1 + continue + if len(candidates) > 1: + counts["ambiguous"] += 1 + continue + called = candidates[0] + confusion[truth_name][called] = confusion[truth_name].get(called, 0) + 1 + if called == truth_name: + counts["correct"] += 1 + else: + counts["misassigned"] += 1 + + total = reads_per_target * len(normalized) + unique = counts["correct"] + counts["misassigned"] + return SimulationResult( + seed=seed, + total_reads=total, + correct_unique=counts["correct"], + misassigned_unique=counts["misassigned"], + ambiguous=counts["ambiguous"], + none=counts["none"], + usable_yield=unique / total, + ambiguity_rate=counts["ambiguous"] / total, + no_call_rate=counts["none"] / total, + false_discovery_rate=counts["misassigned"] / unique if unique else 0.0, + confusion=confusion, + ) + + +def _cycle_rates(error_rate: float | Sequence[float], length: int) -> tuple[float, ...]: + if isinstance(error_rate, (int, float)) and not isinstance(error_rate, bool): + rates = (float(error_rate),) * length + else: + rates = tuple(float(value) for value in error_rate) + if len(rates) != length: + raise ValueError("per-cycle error rates must match target length") + if any(rate < 0.0 or rate > 1.0 for rate in rates): + raise ValueError("error rates must be between 0 and 1") + return rates + + +def _mutate(sequence: str, rates: Sequence[float], rng: random.Random) -> str: + called: list[str] = [] + for base, rate in zip(sequence, rates): + if rng.random() < rate: + alternatives = [candidate for candidate in DNA if candidate != base] + called.append(alternatives[rng.randrange(3)]) + else: + called.append(base) + return "".join(called) + + +def _hamming(left: str, right: str) -> int: + return sum(a != b for a, b in zip(left, right)) From 1bd05fc2ffe66ff2ad9f3046a9b035cc0bcdf059 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:41:40 +0100 Subject: [PATCH 54/93] Test deterministic assay simulation --- python/tests/test_assaysim.py | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 python/tests/test_assaysim.py diff --git a/python/tests/test_assaysim.py b/python/tests/test_assaysim.py new file mode 100644 index 00000000..85023a4c --- /dev/null +++ b/python/tests/test_assaysim.py @@ -0,0 +1,65 @@ +import pytest + +from dotmatch.assaysim import simulate_panel + + +def test_simulation_is_reproducible_and_conserves_reads(): + targets = {"a": "AAAAAAAA", "b": "CCCCCCCC"} + first = simulate_panel(targets, reads_per_target=100, error_rate=0.05, k=1, seed=42) + second = simulate_panel(targets, reads_per_target=100, error_rate=0.05, k=1, seed=42) + + assert first == second + assert first.total_reads == 200 + assert ( + first.correct_unique + + first.misassigned_unique + + first.ambiguous + + first.none + == first.total_reads + ) + assert 0.0 <= first.usable_yield <= 1.0 + assert 0.0 <= first.false_discovery_rate <= 1.0 + + +def test_zero_error_recovers_well_separated_targets(): + result = simulate_panel( + {"a": "AAAA", "b": "CCCC"}, + reads_per_target=10, + error_rate=0.0, + k=0, + ) + assert result.correct_unique == 20 + assert result.misassigned_unique == 0 + assert result.ambiguous == 0 + assert result.none == 0 + assert result.usable_yield == 1.0 + assert result.false_discovery_rate == 0.0 + + +def test_close_targets_expose_ambiguity(): + result = simulate_panel( + {"a": "AAAA", "b": "AAAT"}, + reads_per_target=5, + error_rate=0.0, + k=1, + ) + assert result.ambiguous == 10 + assert result.usable_yield == 0.0 + + +@pytest.mark.parametrize( + ("targets", "kwargs"), + [ + ({}, {}), + ({"a": "AAAA", "b": "AAAA"}, {}), + ({"a": "AAAA", "b": "CCC"}, {}), + ({"a": "AAAN"}, {}), + ({"a": "AAAA"}, {"reads_per_target": 0}), + ({"a": "AAAA"}, {"k": -1}), + ({"a": "AAAA"}, {"error_rate": 1.1}), + ({"a": "AAAA"}, {"error_rate": [0.1, 0.1]}), + ], +) +def test_invalid_simulations_are_rejected(targets, kwargs): + with pytest.raises(ValueError): + simulate_panel(targets, **kwargs) From 9aee7640dde51c6dffa2308fd1911fb49be36efd Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:42:07 +0100 Subject: [PATCH 55/93] Expose assay panel simulation in AssayCode CLI --- python/assaycode/cli.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/python/assaycode/cli.py b/python/assaycode/cli.py index 7a2876a4..d62a03c7 100644 --- a/python/assaycode/cli.py +++ b/python/assaycode/cli.py @@ -11,6 +11,8 @@ from dotmatch import __version__ from dotmatch import cli as _engine_cli from dotmatch.assayscript import AssayScriptError, load_and_compile, write_compiled_plan +from dotmatch.assaysim import simulate_panel +from dotmatch.core import load_targets from dotmatch.assaywatch import WatchThresholds, watch_jsonl from dotmatch.calibration_io import decode_tsv, fit_model_tsv, read_model, write_model @@ -132,6 +134,43 @@ def command_decode_quality(argv: Sequence[str]) -> int: print(json.dumps({"status": "experimental", **summary}, indent=2, sort_keys=True)) return 0 +def command_simulate(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="assaycode simulate", + description="Run an experimental deterministic substitution-error simulation for a target panel.", + ) + parser.add_argument("--targets", required=True, help="TSV target table") + parser.add_argument("--out", required=True, help="simulation result JSON") + parser.add_argument("--reads-per-target", type=int, default=1000) + parser.add_argument("--error-rate", type=float, default=0.01) + parser.add_argument("-k", type=int, default=1) + parser.add_argument("--seed", type=int, default=1) + args = parser.parse_args(list(argv)) + try: + targets = dict(load_targets(args.targets)) + result = simulate_panel( + targets, + reads_per_target=args.reads_per_target, + error_rate=args.error_rate, + k=args.k, + seed=args.seed, + ) + output = Path(args.out) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(result.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + except (OSError, ValueError) as exc: + print(f"assaycode simulate: {exc}", file=sys.stderr) + return 2 + print(json.dumps({ + "status": "experimental", + "result": str(output), + "total_reads": result.total_reads, + "usable_yield": result.usable_yield, + "false_discovery_rate": result.false_discovery_rate, + }, indent=2, sort_keys=True)) + return 0 + + def command_watch(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser( prog="assaycode watch", @@ -181,6 +220,7 @@ def print_help() -> None: assaycode inspect assay.plan.json assaycode calibrate trusted.tsv --out error-model.json assaycode decode-quality --reads windows.tsv --targets targets.tsv --model error-model.json --out calls.tsv + assaycode simulate --targets targets.tsv --out simulation.json assaycode watch assignments.jsonl --out snapshots.jsonl assaycode check assay.toml assaycode plan assay.toml @@ -194,6 +234,7 @@ def print_help() -> None: inspect summarize a compiled plan, safety status, fingerprints, and findings calibrate fit an experimental cycle-error model from trusted pairs decode-quality apply calibrated selective decoding with abstention + simulate estimate yield, ambiguity, no-calls, and FDR under an error model watch stream assignment events into sequential QC decisions AssaySpec v1 workflow shortcuts: @@ -235,6 +276,8 @@ def main(argv: Sequence[str] | None = None) -> int: return command_calibrate(raw_args[1:]) if raw_args[0] == "decode-quality": return command_decode_quality(raw_args[1:]) + if raw_args[0] == "simulate": + return command_simulate(raw_args[1:]) if raw_args[0] == "watch": return command_watch(raw_args[1:]) if raw_args[0] == "engine": From 48692d4fd6a482b6d8cb6e42d6b5d064ace42fee Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:42:25 +0100 Subject: [PATCH 56/93] Exercise AssayCode simulation CLI --- python/tests/test_assaysim.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/tests/test_assaysim.py b/python/tests/test_assaysim.py index 85023a4c..3b350bd7 100644 --- a/python/tests/test_assaysim.py +++ b/python/tests/test_assaysim.py @@ -1,5 +1,8 @@ +import json + import pytest +from assaycode.cli import command_simulate from dotmatch.assaysim import simulate_panel @@ -63,3 +66,25 @@ def test_close_targets_expose_ambiguity(): def test_invalid_simulations_are_rejected(targets, kwargs): with pytest.raises(ValueError): simulate_panel(targets, **kwargs) + + +def test_simulate_cli_writes_machine_readable_result(tmp_path, capsys): + targets = tmp_path / "targets.tsv" + targets.write_text("target_id\ttarget_seq\na\tAAAA\nb\tCCCC\n", encoding="utf-8") + output = tmp_path / "simulation.json" + + assert command_simulate([ + "--targets", str(targets), + "--out", str(output), + "--reads-per-target", "5", + "--error-rate", "0", + "-k", "0", + "--seed", "7", + ]) == 0 + + payload = json.loads(output.read_text(encoding="utf-8")) + summary = json.loads(capsys.readouterr().out) + assert payload["correct_unique"] == 10 + assert payload["seed"] == 7 + assert summary["status"] == "experimental" + assert summary["usable_yield"] == 1.0 From 3cdbab6279913726564d5032a5034a2c0b113feb Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:42:44 +0100 Subject: [PATCH 57/93] Enforce integer calibration counts --- python/dotmatch/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dotmatch/calibration.py b/python/dotmatch/calibration.py index 7418b2ec..9c6ffa94 100644 --- a/python/dotmatch/calibration.py +++ b/python/dotmatch/calibration.py @@ -261,7 +261,7 @@ def smoothed_abundance_priors( raise ValueError("alpha must be positive") if not counts: raise ValueError("counts must not be empty") - if any(isinstance(value, bool) or value < 0 for value in counts.values()): + if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in counts.values()): raise ValueError("counts must be non-negative integers") total = sum(counts.values()) + alpha * len(counts) return {target: (count + alpha) / total for target, count in counts.items()} From af8ce2aa211d7aade3df403727d052c534b49303 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:43:04 +0100 Subject: [PATCH 58/93] Test strict abundance prior counts --- python/tests/test_calibration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/tests/test_calibration.py b/python/tests/test_calibration.py index 61e071ff..ce8cb2d4 100644 --- a/python/tests/test_calibration.py +++ b/python/tests/test_calibration.py @@ -154,3 +154,8 @@ def test_joint_decode_rejects_incomplete_constraints() -> None: {"sample": {"s1": 1.0}, "guide": {"g1": 1.0}}, [{"sample": "s1"}], ) + + +def test_abundance_priors_reject_fractional_counts(): + with pytest.raises(ValueError, match="non-negative integers"): + smoothed_abundance_priors({"a": 1.5}) From dc76f33f1aed9cce0490c8cc912aa5ded0f59bd8 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:44:06 +0100 Subject: [PATCH 59/93] Document experimental assay simulation --- docs/assaycode.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/assaycode.md b/docs/assaycode.md index f49ad295..cfb2c330 100644 --- a/docs/assaycode.md +++ b/docs/assaycode.md @@ -86,6 +86,27 @@ assaycode engine validate --targets targets.tsv --reads reads.fastq.gz \ --target-length 20 ``` +## Design-time simulation + +Use the experimental digital twin to stress-test a fixed-length target panel +under a declared substitution-error rate before sequencing: + +```bash +assaycode simulate \ + --targets targets.tsv \ + --reads-per-target 10000 \ + --error-rate 0.01 \ + -k 1 \ + --seed 42 \ + --out simulation.json +``` + +The JSON result reports correct and incorrect unique calls, ambiguous calls, +no-calls, usable yield, false-discovery rate, and a truth-by-call confusion +table. Results are deterministic for a seed. This substitution-only simulator +is an experimental design aid, not a replacement for held-out empirical +validation or a platform performance claim. + ## Streaming QC `assaycode watch` consumes assignment events as JSON Lines. It keeps bounded From fd26c5f65e52a5f39e658761dd1ec3e3cc2f7739 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:44:24 +0100 Subject: [PATCH 60/93] Record AssayCode digital twin --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc593cb..dc15404c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable user-facing changes are tracked here. Public statements in release n ## 0.2.0 - Unreleased +- Added a deterministic experimental panel simulator for pre-sequencing yield, ambiguity, no-call, confusion, and FDR analysis. ### AssayCode platform - Added the additive `assaycode` CLI and Python namespace while preserving the `dotmatch` package, executable, ABI, DOI, schemas, and citation contract. From df6244f36eae581bcb9dcb524c0c1d0b5fb796a6 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:44:47 +0100 Subject: [PATCH 61/93] Gate assay simulation assets --- scripts/check_assaycode_readiness.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/check_assaycode_readiness.py b/scripts/check_assaycode_readiness.py index 47053250..d0e39204 100644 --- a/scripts/check_assaycode_readiness.py +++ b/scripts/check_assaycode_readiness.py @@ -51,6 +51,7 @@ def main() -> int: ) errors += require( "python/dotmatch/assaywatch.py", + "python/dotmatch/assaysim.py", ["class SequentialMonitor", "assignment_rate_interval95", "insufficient_data"], ) for test in [ @@ -59,6 +60,7 @@ def main() -> int: "python/tests/test_calibration.py", "python/tests/test_calibration_io.py", "python/tests/test_assaywatch.py", + "python/tests/test_assaysim.py", ]: errors += require(test, ["test_"]) errors += require( From 97072a238fe3cbef6a5e776329c03b887a4179eb Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:45:21 +0100 Subject: [PATCH 62/93] Describe experimental assay digital twin --- paper/paper.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/paper/paper.md b/paper/paper.md index 8f78185a..08ec29d6 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -167,6 +167,13 @@ held-out threshold selector can maximize accepted calls subject to an empirical false-discovery ceiling. These APIs remain experimental until public datasets show calibration and yield improvements at a fixed measured error rate. +A deterministic design-time simulator perturbs fixed-length panels under a +declared scalar or per-cycle substitution model and applies the same +ambiguity-preserving radius rule. It reports usable yield, ambiguity, no-call +rate, false-discovery rate, and truth-by-call confusion. This digital twin is a +reproducible stress test for panel geometry, not a replacement for held-out +platform data. + AssayCode also includes a bounded-memory sequential monitor for assignment events. It reports assignment, ambiguity, unmatched, and invalid rates with a Wilson confidence interval and emits machine-readable on-track, review, or @@ -177,7 +184,7 @@ sequencer-control or production adaptive-sampling claim. Tests cover native kernels, deterministic fuzzing, Python APIs, CLI workflows, AssaySpec projects, AssayScript compilation, calibration mathematics, -sequential monitoring, packaging, and public workflow fixtures. The repository +panel simulation, sequential monitoring, packaging, and public workflow fixtures. The repository separates supported, experimental, and unsupported statements in a machine-checked evidence inventory. From 47b87d9516a262ad46ef3954829f30be694a7b3f Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:45:44 +0100 Subject: [PATCH 63/93] Bound assay simulation scientific claims --- docs/scientific-claims.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/scientific-claims.md b/docs/scientific-claims.md index 5f3d13ce..dd4b73f4 100644 --- a/docs/scientific-claims.md +++ b/docs/scientific-claims.md @@ -68,6 +68,10 @@ Techniques deliberately not claimed: evidence, supports selective and allowed-tuple decoding, and reports calibration metrics. It has not yet passed public-data calibration or throughput gates and does not replace the deterministic CLI path; +- `assaycode simulate` is an experimental substitution-only design aid. It + deterministically estimates panel yield, ambiguity, no-calls, false-discovery + rate, and confusion under declared error rates; it is not empirical sequencing + performance evidence or a platform model; - `assaycode watch` is an experimental bounded-memory monitor over assignment JSONL. Its threshold decisions are workflow QC signals, not sequencer control or validated adaptive sampling; @@ -116,7 +120,7 @@ alignment or all demultiplexing tasks. | Indexed assignment preserves native exhaustive-scan semantics for `unique`, `ambiguous`, `none`, and `invalid` outcomes in the supported fixed-window lanes. | Supported | `dotmatch validate`, native assignment tests, `make native-exact-gate`, Edlib validation artifacts under `benchmarks/raw/`, Levenshtein `k=2` CLI regression cases, Hamming `k=2`/`k=3` comparator artifacts | The native gate requires zero Edlib mismatches; large-library exact rows must beat `exact_hash_lookup`; large-library indexed `k=1` rows must beat exhaustive Edlib by >10x, beat the best BK-tree/neighbor baseline, and verify no more than 1.05 candidates/read; large-library fixed-length `k=2` substitution rows must beat exhaustive Edlib by >8x while verifying no more than 1.05 candidates/read; and large-library Levenshtein `k=2` insertion/deletion rows must beat exhaustive Edlib by >8x while verifying no more than 25 candidates/read. Levenshtein `k=2` uses packed A/C/G/T hash-neighborhood pruning for windows up to 32 bases, with fallback preserving semantics for unsupported cases. Hamming `k=2`/`k=3` is same-length substitution-only fixed-window matching. Current `N`/IUPAC behavior is literal-byte matching, not wildcard expansion semantics. | | Public CRISPR guide-counting rows are validated. | Supported | `make public-crispr-evidence-gate` passes; report at `docs/benchmarks/public_crispr/README.md` | Supports the documented MAGeCK/Yusa public-data workflow, not universal CRISPR superiority. | | Extended CRISPR comparison rows are validated. | Supported | `make crispr-comparison-gate` passes; report at `docs/benchmarks/crispr_comparison/README.md` | Applies to the recorded CRISPR guide-counting lanes and their documented comparator semantics. | -| AssayCode exposes experimental AssayScript v2 compilation, calibrated decoding primitives, allowed-tuple inference, and sequential JSONL QC. | Experimental | Python unit tests for compiler validation/fingerprints/strategy selection, posterior calibration mathematics, joint decoding, and streaming thresholds | These are foundations, not a production universal assay runtime or public-data superiority claim. Deterministic DotMatch assignment remains authoritative. |\n| DotMatch has an experimental GPU acceleration evidence lane. | Experimental | `make bench-gpu`, `make gpu-evidence-gate`, report at `docs/benchmarks/gpu/README.md` | Current evidence is Apple Metal-only for packed Hamming `k=1`, including synthetic rows and a public CRISPR FASTQ extract-pack-dispatch-readback-count row. It is not a production speed claim. Promotion requires additional real-workload gates, non-A/C/G/T fallback, and production scheduling. | +| AssayCode exposes experimental AssayScript v2 compilation, calibrated decoding primitives, allowed-tuple inference, deterministic panel simulation, and sequential JSONL QC. | Experimental | Python unit tests for compiler validation/fingerprints/strategy selection, posterior calibration mathematics, joint decoding, simulator outcome accounting, and streaming thresholds | These are foundations, not a production universal assay runtime or public-data superiority claim. Deterministic DotMatch assignment remains authoritative. |\n| DotMatch has an experimental GPU acceleration evidence lane. | Experimental | `make bench-gpu`, `make gpu-evidence-gate`, report at `docs/benchmarks/gpu/README.md` | Current evidence is Apple Metal-only for packed Hamming `k=1`, including synthetic rows and a public CRISPR FASTQ extract-pack-dispatch-readback-count row. It is not a production speed claim. Promotion requires additional real-workload gates, non-A/C/G/T fallback, and production scheduling. | | FASTQ count and demux workflows can optionally gate one-edit substitution and read-insertion rescue by observed Sanger Phred quality. | Supported | `make cli-test`; `--max-correction-qual` CLI regression cases | This is a deterministic correction filter, not a calibrated sequencing-error probability model. Read-deletion rescue has no observed edited base to score and is not rejected by this gate. | | FASTQ count workflows can optionally reject same-length unique calls whose Phred-quality posterior is below a configured threshold. | Experimental | Python and CLI posterior regression tests | The posterior model is an opt-in conservative filter over fixed-window calls. It is not calibrated public evidence, not supported for demux output routing, and not a throughput claim. | | The Python API exposes an experimental quality-aware posterior helper for one fixed-window read against known targets. | Experimental | `python/tests/test_matcher.py` posterior-assignment cases | This is a simple Phred likelihood helper with literal-byte target comparison and optional priors. It is not yet calibrated on public datasets, not used by the high-throughput CLI, and not a speed claim. | From ca7ec3c0df67de097815c8a29b1b6beabdbfddc8 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Tue, 14 Jul 2026 15:46:21 +0100 Subject: [PATCH 64/93] Rebrand public site around AssayCode --- app/page.tsx | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/app/page.tsx b/app/page.tsx index 91b8f904..de5e9f5b 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -25,15 +25,17 @@ const structuredData = { { "@type": "WebSite", "@id": "https://dnncha.github.io/dotmatch/#website", - name: "DotMatch", + name: "AssayCode", + alternateName: "DotMatch", url: "https://dnncha.github.io/dotmatch", description: - "DotMatch is a deterministic known-target sequencing assignment toolkit for CRISPR guides, inline barcodes, feature tags, primers, and panel targets." + "AssayCode compiles, validates, decodes, and diagnoses known-target sequencing assays using the DotMatch engine." }, { "@type": "SoftwareApplication", "@id": "https://dnncha.github.io/dotmatch/#software", - name: "DotMatch", + name: "AssayCode", + alternateName: "DotMatch", applicationCategory: "Bioinformatics software", operatingSystem: "Linux, macOS", softwareHelp: "https://dotmatch.readthedocs.io/", @@ -41,7 +43,7 @@ const structuredData = { license: `${repoUrl}/blob/main/LICENSE`, programmingLanguage: ["C", "Python", "R"], description: - "DotMatch assigns fixed read windows to known short DNA targets and reports unique, ambiguous, none, and invalid outcomes for auditable sequencing workflows." + "AssayCode is an assay reliability platform powered by DotMatch for ambiguity-aware known-target DNA assignment." } ] }; @@ -192,9 +194,9 @@ export default function Home() { dangerouslySetInnerHTML={{ __html: JSON.stringify(structuredData) }} />
      - +
-
+