From e35776b7060af520517f23f74acf8ea26a00e045 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Sat, 25 Jul 2026 19:13:28 +0100 Subject: [PATCH] Add reproducible assay handoff bundles --- README.md | 13 ++ docs/assayspec.md | 7 + docs/command-reference.md | 5 + docs/getting-started.md | 2 + docs/index.md | 2 + docs/lab-evaluation.md | 94 ++++++++++++ docs/tutorials/crispr-count-first-run.md | 11 ++ python/dotmatch/assayspec.py | 181 +++++++++++++++++++++++ python/tests/test_assayspec.py | 38 +++++ scripts/check_python_wheel.py | 29 ++++ 10 files changed, 382 insertions(+) create mode 100644 docs/lab-evaluation.md diff --git a/README.md b/README.md index 50927705..4dd9af8c 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,18 @@ Review `crispr_screen/inference_report.json` and `assay.toml`, then run: dotmatch assay start crispr_screen/assay.toml ``` +After a completed run, create a compact technical review bundle without copying +raw FASTQs: + +```bash +dotmatch assay handoff crispr_screen/assay.toml +``` + +The bundle includes configuration, QC, reports, methods, citation material, and +checksums for declared inputs and copied outputs. See the [lab evaluation and +handoff guide](https://dotmatch.readthedocs.io/en/latest/lab-evaluation.html) +for the review sequence and data-handling boundary. + For an explicit one-command run, use `dotmatch crispr-count`. The [CRISPR tutorial](https://dotmatch.readthedocs.io/en/latest/tutorials/crispr-count-first-run.html) covers both routes. @@ -212,6 +224,7 @@ replaces general alignment or every demultiplexing workflow. - [Getting started](https://dotmatch.readthedocs.io/en/latest/getting-started.html) - [Command reference](https://dotmatch.readthedocs.io/en/latest/command-reference.html) - [AssaySpec workflows](https://dotmatch.readthedocs.io/en/latest/assayspec.html) +- [Lab evaluation and handoff](https://dotmatch.readthedocs.io/en/latest/lab-evaluation.html) - [CRISPR count QC](https://dotmatch.readthedocs.io/en/latest/crispr-qc.html) - [Barcode panel design](https://dotmatch.readthedocs.io/en/latest/barcode-panel-design.html) - [Output schemas](https://dotmatch.readthedocs.io/en/latest/schemas.html) diff --git a/docs/assayspec.md b/docs/assayspec.md index 9708045d..bcdf42f3 100644 --- a/docs/assayspec.md +++ b/docs/assayspec.md @@ -21,6 +21,7 @@ dotmatch assay run assay.toml dotmatch assay init --template crispr --out assay.toml dotmatch assay infer --mode count --assay-type crispr --targets guides.csv --reads sample.fastq.gz --out assay.toml --report inference_report.json dotmatch assay autopsy assay.toml --out-dir autopsy/ +dotmatch assay handoff assay.toml ``` `start` is the default production entrypoint: it runs `check`, then `run`, and @@ -71,6 +72,12 @@ relaxed with `[reliability] fail_on_draft_inference = false`. `top_unmatched.*.tsv` files. `run` also triggers autopsy automatically when sample QC crosses conservative thresholds. +`handoff` creates `/handoff/` after a completed run. It copies a +review-sized set of reports, configuration, QC, count, methods, and citation +files; writes `SHA256SUMS`; and records SHA-256 hashes for every declared input. +It does not copy FASTQ files, so raw reads remain in the originating controlled +workspace. Use `--out-dir PATH` to write the bundle to an empty directory. + ## Count Example ```toml diff --git a/docs/command-reference.md b/docs/command-reference.md index b1c26121..9af8cfb0 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -56,12 +56,17 @@ dotmatch assay optimize assay.toml dotmatch assay plan assay.toml dotmatch assay run assay.toml dotmatch assay autopsy assay.toml --out-dir autopsy/ +dotmatch assay handoff assay.toml ``` `start` is the production entrypoint: it checks the spec, runs assignment, and prints the reliability verdict. `optimize` writes an advisory CPU/GPU backend recommendation; it does not change the count authority. +`handoff` creates a review bundle after a completed run. It includes the +configuration, reports, QC, count outputs, methods, citation material, and +SHA-256 records for inputs and copied files, without copying raw FASTQs. + Supported templates include `crispr`, `feature-barcode`, `inline-barcode-count`, `inline-barcode-demux`, `amplicon-panel`, `oligo-adapter`, and `pair-count`. diff --git a/docs/getting-started.md b/docs/getting-started.md index d8a0c498..cbded48a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -225,6 +225,8 @@ default policy. - Use [AssaySpec](assayspec.md) for the full `assay new`, `start`, `check`, and `run` command reference. +- Use [Lab Evaluation and Handoff](lab-evaluation.md) when a completed run + needs a review package without moving raw reads. - Use [Command Reference](command-reference.md) for the current command map. - Use [CRISPR Count QC](crispr-qc.md) before downstream screen statistics. - Use [Barcode Panel Design](barcode-panel-design.md) when creating or checking diff --git a/docs/index.md b/docs/index.md index 6e49b2ee..c16bdbaf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,6 +33,7 @@ the [command reference](command-reference.md). | --- | --- | | Count guides from a CRISPR screen | [CRISPR first run](tutorials/crispr-count-first-run.md) | | Build a checked assay project | [AssaySpec workflows](assayspec.md) | +| Evaluate and hand off a run | [Lab evaluation and handoff](lab-evaluation.md) | | Split reads by inline barcode | [Getting started: demultiplexing](getting-started.md#demultiplex-inline-barcodes) | | Diagnose barcode failures | [Barcode run diagnosis](getting-started.md#diagnose-a-barcode-run) | | Design or check a barcode panel | [Barcode panel design](barcode-panel-design.md) | @@ -72,6 +73,7 @@ getting-started command-reference tutorials/crispr-count-first-run tutorials/scverse-perturb-seq +lab-evaluation ``` ```{toctree} diff --git a/docs/lab-evaluation.md b/docs/lab-evaluation.md new file mode 100644 index 00000000..2cae4584 --- /dev/null +++ b/docs/lab-evaluation.md @@ -0,0 +1,94 @@ +# Lab Evaluation and Handoff + +This guide is for a bioinformatics team or core facility evaluating DotMatch on +a known-target sequencing assay. It is a local technical evaluation protocol, +not a clinical validation protocol and not a substitute for assay-specific +controls. + +## Before processing study data + +Confirm all of the following with the assay owner: + +- the target library is the intended revision and contains only the expected + fixed-window sequences; +- the FASTQ read and orientation are known, along with the start and length of + the target window; +- the permitted edit radius and ambiguity policy have been chosen deliberately; +- sample identifiers and FASTQ paths have been checked against the run sheet; +- the intended downstream consumer of the output is known. For CRISPR counts, + DotMatch writes a MAGeCK-style matrix but does not perform screen statistics. + +Create a reviewable project from the release package: + +```bash +dotmatch assay new crispr \ + --library guides.csv \ + --reads-dir fastqs/ \ + --out crispr_evaluation/ + +cd crispr_evaluation +dotmatch assay check assay.toml +``` + +`assay new` samples the input reads to propose an extraction window. Review +`inference_report.json`, the target-library audit, and the generated +`assay.toml`. Keep `status = "draft"` until a qualified reviewer has confirmed +the configuration, then set it to `ready`. + +## Run and review + +```bash +./run.sh +dotmatch assay handoff assay.toml +``` + +`./run.sh` uses `dotmatch assay start`: it runs preflight, assignment, target +audit, QC, and validation. Open these files in order: + +1. `assay_out/reliability_report.html` +2. `assay_out/sample_qc.tsv` and `assay_out/crispr_qc.html` for CRISPR runs +3. `assay_out/assay_report.html` +4. `assay_out/counts.mageck.tsv` or the primary count/demultiplexing output +5. `assay_out/methods.md` and `assay_out/CITATION.bib` + +Do not treat a `passed` DotMatch reliability verdict as proof of biological +validity. It means the configured target safety and software QC rules passed. +Review positive/negative controls, sample identity, sequencing-run metrics, +replicate agreement, and downstream analysis according to the local assay +protocol. + +## Handoff package + +`dotmatch assay handoff assay.toml` writes `assay_out/handoff/` without copying +raw reads. The bundle is suitable for an internal technical review or a +workflow-maintainer evaluation: + +- `README_FOR_REVIEW.md` states the review order and boundary; +- `handoff_manifest.json` records the configuration, verdict, input file sizes, + and input/output SHA-256 hashes; +- `SHA256SUMS` verifies the copied review files; +- `review/` contains reports, QC tables, primary outputs, methods, citation, + and software-version records. + +In the controlled workspace containing the FASTQs, recompute each input hash +in `handoff_manifest.json` before approving the handoff. Do not send raw reads +or identifiers outside the approved data-handling route merely to make a +DotMatch review bundle. + +## Decision record + +For each evaluated assay, record: + +| Item | Record | +| --- | --- | +| Assay and target-library revision | Name, source, checksum, and owner | +| Read extraction | Read, orientation, start, length, and rationale | +| Assignment rules | Metric, edit radius, ambiguity policy, and handling of ambiguous reads | +| Input identity | Sample-sheet revision and FASTQ checksums | +| QC outcome | Reliability verdict, findings reviewed, controls, and any exceptions | +| Output recipient | Count matrix/report location and downstream analysis owner | +| Software record | DotMatch version, native version, `methods.md`, and `CITATION.bib` | + +Keep this record with the project or laboratory notebook. It makes a later +rerun auditable without claiming that DotMatch replaces the remainder of the +assay or analysis workflow. diff --git a/docs/tutorials/crispr-count-first-run.md b/docs/tutorials/crispr-count-first-run.md index 69dd34b4..8f5d8207 100644 --- a/docs/tutorials/crispr-count-first-run.md +++ b/docs/tutorials/crispr-count-first-run.md @@ -48,6 +48,17 @@ Key outputs under the configured `out_dir`: CPU remains the assignment authority. GPU Metal is opt-in via `[backend]` in the assay spec and requires `--metal-validate` when enabled. +After a completed run, create a review package without copying raw FASTQs: + +```bash +dotmatch assay handoff crispr_assay.toml +``` + +This writes `assay_out/handoff/`, including reports, primary outputs, methods, +citation material, and SHA-256 records for the declared inputs. See [Lab +Evaluation and Handoff](../lab-evaluation.md) for the review order and +acceptance record. + ## 3. Direct `crispr-count` (single command) For a minimal single command without the full assay wrapper: diff --git a/python/dotmatch/assayspec.py b/python/dotmatch/assayspec.py index d09d1de8..ced21070 100644 --- a/python/dotmatch/assayspec.py +++ b/python/dotmatch/assayspec.py @@ -3,6 +3,7 @@ import argparse import csv import gzip +import hashlib import html import json import math @@ -545,6 +546,16 @@ def command_assay(argv: Sequence[str]) -> int: autopsy.add_argument("spec") autopsy.add_argument("--out-dir", required=True) + handoff = sub.add_parser( + "handoff", + help="assemble a review bundle with run outputs, input checksums, and the reliability verdict", + ) + handoff.add_argument("spec") + handoff.add_argument( + "--out-dir", + help="empty directory for the bundle (default: /handoff)", + ) + workflow_help = { "check": "validate an AssaySpec and write preflight reliability artifacts", "optimize": "write a benchmark-informed CPU/GPU backend recommendation", @@ -574,6 +585,12 @@ def command_assay(argv: Sequence[str]) -> int: if args.command == "autopsy": run_autopsy(assay, Path(args.out_dir)) return 0 + if args.command == "handoff": + plan = compile_assay_plan(assay) + out_dir = Path(args.out_dir) if args.out_dir else plan.spec.out_dir / "handoff" + _write_handoff_bundle(plan, out_dir) + print(out_dir / "README_FOR_REVIEW.md") + return 0 if args.command == "check": plan = compile_assay_plan(assay) return _command_assay_check(plan) @@ -2119,6 +2136,170 @@ def _write_citation_artifacts(plan: AssayPlan, manifest: Mapping[str, Any]) -> N _write_software_versions_yml(plan.artifacts["software_versions"], plan, manifest) +def _write_handoff_bundle(plan: AssayPlan, out_dir: Path) -> None: + """Write a portable, review-sized record for a completed AssaySpec run. + + Raw reads are deliberately not copied: they can be large or sensitive. The + bundle names and hashes every declared input so a reviewer can establish + which local data were used without moving those reads outside the lab. + """ + manifest_path = plan.artifacts["manifest"] + reliability_path = plan.artifacts["reliability_summary"] + if not manifest_path.is_file() or not reliability_path.is_file(): + raise AssaySpecError( + "handoff requires a completed assay run with assay_manifest.json and " + "reliability_summary.json; run 'dotmatch assay start ' first" + ) + if out_dir.exists() and any(out_dir.iterdir()): + raise AssaySpecError(f"handoff output directory must be empty: {out_dir}") + + manifest = _read_json_object(manifest_path, "assay manifest") + reliability = _read_json_object(reliability_path, "reliability summary") + out_dir.mkdir(parents=True, exist_ok=True) + review_dir = out_dir / "review" + review_dir.mkdir() + + copied: list[dict[str, object]] = [] + # These are the files a PI, core facility, or pipeline maintainer normally + # needs to judge configuration, QC, outputs, and citation without raw reads. + for key in ( + "normalized_spec", + "manifest", + "manifest_summary", + "reliability_summary", + "reliability_findings", + "reliability_report", + "reliability_manifest_summary", + "assay_fixes", + "methods", + "citation_bib", + "software_versions", + "assay_report", + "counts", + "sample_qc", + "summary", + "report", + "crispr_qc", + "crispr_qc_summary", + "crispr_qc_report", + "target_counts_long", + ): + source = plan.artifacts.get(key) + if source is None or not source.is_file(): + continue + destination = review_dir / source.name + shutil.copy2(source, destination) + copied.append( + { + "role": key, + "path": str(destination.relative_to(out_dir)), + "sha256": _sha256_file(destination), + "bytes": destination.stat().st_size, + } + ) + + input_files = [ + { + "role": role, + "declared_path": declared, + "sha256": _sha256_file(path), + "bytes": path.stat().st_size, + } + for role, declared, path in _handoff_inputs(plan.spec) + ] + bundle_manifest = { + "schema_version": 1, + "bundle_type": "dotmatch_assay_handoff", + "assay_type": plan.spec.assay_type, + "mode": plan.spec.mode, + "assay_status": plan.spec.status, + "reliability_status": reliability.get("overall_status", "unknown"), + "native_version": manifest.get("native_version", ""), + "inputs": input_files, + "review_files": copied, + "review_boundary": ( + "Raw reads are not copied into this bundle. Verify the declared input " + "hashes in the originating controlled workspace." + ), + } + (out_dir / "handoff_manifest.json").write_text( + json.dumps(bundle_manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + _write_handoff_readme(out_dir, bundle_manifest) + checksum_rows = [f"{entry['sha256']} {entry['path']}" for entry in copied] + (out_dir / "SHA256SUMS").write_text( + "\n".join(checksum_rows) + ("\n" if checksum_rows else ""), + encoding="utf-8", + ) + + +def _handoff_inputs(assay: AssaySpec) -> list[tuple[str, str, Path]]: + if assay.mode == "count": + items = [("targets", str(assay.data["targets"]))] + items.extend( + (f"sample:{sample['id']}", str(sample["fastq"])) for sample in _samples(assay.data) + ) + elif assay.mode == "demux": + items = [ + ("barcodes", str(assay.data["barcodes"])), + ("reads", str(assay.data["reads"])), + ] + else: + items = [ + ("left_targets", str(assay.data["left_targets"])), + ("right_targets", str(assay.data["right_targets"])), + ("reads", str(assay.data["reads"])), + ] + return [ + (role, declared, _path_from_spec(assay.path, declared, allow_absolute=True, name=role)) + for role, declared in items + ] + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for block in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _read_json_object(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise AssaySpecError(f"invalid {label}: {path}") from exc + if not isinstance(value, dict): + raise AssaySpecError(f"invalid {label}: {path}") + return value + + +def _write_handoff_readme(out_dir: Path, bundle_manifest: Mapping[str, Any]) -> None: + verdict = str(bundle_manifest["reliability_status"]) + text = "\n".join( + [ + "# DotMatch Assay Handoff", + "", + f"Reliability verdict: **{verdict}**.", + "", + "This is a review bundle for a completed known-target assay run. It includes the configuration, QC findings, reports, count outputs, methods, and citation material needed for technical review. It does not include raw reads.", + "", + "## Review order", + "", + "1. Read `review/reliability_report.html` and resolve every `error` or `blocked` finding before using counts for a downstream decision.", + "2. Inspect `review/assay.normalized.json`, `review/assay_manifest.json`, and `review/sample_qc.tsv` against the assay design and sample sheet.", + "3. Inspect `review/counts.mageck.tsv` or the equivalent primary output, then `review/assay_report.html` and `review/methods.md`.", + "4. In the originating controlled workspace, recompute the SHA-256 hashes for every input in `handoff_manifest.json`. Raw reads were intentionally left in that workspace.", + "5. Verify copied review files with `shasum -a 256 -c SHA256SUMS` from this directory.", + "", + "A `passed` verdict means the configured DotMatch checks passed; it does not replace assay-specific controls, sample identity checks, downstream screen statistics, or clinical validation.", + "", + ] + ) + (out_dir / "README_FOR_REVIEW.md").write_text(text, encoding="utf-8") + + def _project_root() -> Path: return Path(__file__).resolve().parents[2] diff --git a/python/tests/test_assayspec.py b/python/tests/test_assayspec.py index 856e3bdc..70a7d8ba 100644 --- a/python/tests/test_assayspec.py +++ b/python/tests/test_assayspec.py @@ -638,6 +638,44 @@ def test_assay_run_count_reproduces_existing_crispr_fixture(tmp_path: Path) -> N assert "sample_a.fastq" in report +def test_assay_handoff_writes_review_bundle_with_input_and_output_hashes(tmp_path: Path) -> None: + subprocess.run(["make", "dotmatch"], cwd=ROOT, check=True) + spec = _write_count_spec(tmp_path) + env = {"DOTMATCH_NATIVE_CLI": str(ROOT / "dotmatch")} + + run = _run_cli(["assay", "run", str(spec)], env=env) + assert run.returncode == 0, run.stderr + handoff = _run_cli(["assay", "handoff", str(spec)], env=env) + + assert handoff.returncode == 0, handoff.stderr + bundle = tmp_path / "assay_out" / "handoff" + record = json.loads((bundle / "handoff_manifest.json").read_text(encoding="utf-8")) + assert record["bundle_type"] == "dotmatch_assay_handoff" + assert record["reliability_status"] == "failed" + assert {item["role"] for item in record["inputs"]} == { + "targets", + "sample:sample_a", + "sample:sample_b", + } + assert all(len(item["sha256"]) == 64 for item in record["inputs"]) + assert any(item["role"] == "counts" for item in record["review_files"]) + assert (bundle / "review" / "reliability_report.html").exists() + assert (bundle / "review" / "counts.mageck.tsv").exists() + assert "It does not include raw reads" in (bundle / "README_FOR_REVIEW.md").read_text( + encoding="utf-8" + ) + assert "review/counts.mageck.tsv" in (bundle / "SHA256SUMS").read_text(encoding="utf-8") + + +def test_assay_handoff_requires_a_completed_run(tmp_path: Path) -> None: + spec = _write_count_spec(tmp_path) + + result = _run_cli(["assay", "handoff", str(spec)]) + + assert result.returncode == 2 + assert "requires a completed assay run" in result.stderr + + def test_assay_run_manifest_records_configured_reliability_thresholds(tmp_path: Path) -> None: subprocess.run(["make", "dotmatch"], cwd=ROOT, check=True) spec = _write_count_spec(tmp_path) diff --git a/scripts/check_python_wheel.py b/scripts/check_python_wheel.py index 55e1885d..9510fb13 100644 --- a/scripts/check_python_wheel.py +++ b/scripts/check_python_wheel.py @@ -4,6 +4,7 @@ import argparse from email.parser import Parser from email.message import Message +import hashlib import json import os import platform @@ -318,6 +319,34 @@ def verify_clean_install(artifact: Path, install_root: Path, expected_version: s ]: if not (reliability_dir / required).exists(): raise SystemExit(f"{artifact.name} installed assay run did not write {required}") + handoff_dir = probe_dir / "handoff" + run( + [ + str(venv_script(env_dir, "dotmatch")), + "assay", + "handoff", + str(spec), + "--out-dir", + str(handoff_dir), + ], + cwd=probe_dir, + env=env, + ) + handoff = json.loads((handoff_dir / "handoff_manifest.json").read_text(encoding="utf-8")) + if handoff.get("bundle_type") != "dotmatch_assay_handoff": + raise SystemExit(f"{artifact.name} installed assay handoff manifest is invalid: {handoff!r}") + input_roles = {str(item.get("role")) for item in handoff.get("inputs", []) if isinstance(item, dict)} + if input_roles != {"targets", "sample:sample"}: + raise SystemExit(f"{artifact.name} installed assay handoff inputs are invalid: {input_roles!r}") + review_files = [item for item in handoff.get("review_files", []) if isinstance(item, dict)] + review_paths = {str(item.get("path")) for item in review_files} + if "review/reliability_report.html" not in review_paths or "review/counts.mageck.tsv" not in review_paths: + raise SystemExit(f"{artifact.name} installed assay handoff is missing required review files: {review_paths!r}") + for item in review_files: + review_path = handoff_dir / str(item["path"]) + observed_hash = hashlib.sha256(review_path.read_bytes()).hexdigest() + if observed_hash != item.get("sha256"): + raise SystemExit(f"{artifact.name} installed assay handoff checksum is invalid for {review_path}") counts = probe_dir / "counts.mageck.tsv" sample_qc = probe_dir / "sample_qc.tsv" crispr_qc = probe_dir / "crispr_qc.json"