docs/perf: define and measure the 10x evidence gates

Makefile

@@ -27,7 +27,7 @@ DOTMATCH_SHARED_FLAGS := -shared
 QDALIGN_SHARED_FLAGS := -shared
 endif
 
-.PHONY: all clean test cli-test coverage bench bench-batch bench-small bench-native-matrix figures bench-real-report bench-gpu gpu-report gpu-evidence-gate bench-barcode-demux bench-barcode-panel-design bench-barcode-demux-competitors bench-barcode-comparison barcode-autopsy-demo barcode-panel-design-gate barcode-validation-ready barcode-comparison-report barcode-comparison-gate barcode-demux-report barcode-competitor-env fetch-barcode-demo fetch-barcode-demo-claim fetch-sanson-crispr fetch-10x-bcl-demo bench-bcl-small bench-bcl-10x bench-bcl-real bench-bcl-real-repeated bcl-figures bcl-competitor-env bcl-linux-env bcl-tiny-public-gate bcl-comparison-gate fetch-oligo-adapter-demo bench-oligo-adapter bench-oligo-adapter-public oligo-adapter-smoke-gate oligo-adapter-public-gate fetch-amplicon-panel-demo bench-amplicon-panel bench-amplicon-panel-public amplicon-panel-smoke-gate amplicon-panel-public-gate fetch-feature-barcode-demo bench-feature-barcode bench-feature-barcode-public feature-barcode-smoke-gate feature-barcode-public-gate fetch-perturb-seq-demo bench-perturb-seq bench-perturb-seq-public perturb-seq-smoke-gate perturb-seq-public-gate bench-public-crispr-small bench-public-crispr bench-public-crispr-competitors bench-public-crispr-repeated bench-public-crispr-scaling bench-real-competitors bench-crispr-comparison bench-crispr-hamming-k23 crispr-comparison-report crispr-comparison-gate count-agreement count-agreement-comparison validate-public-crispr-edlib validate-crispr-comparison-edlib public-crispr-report public-crispr-evidence-gate public-crispr-smoke-gate competitor-env edlib edlib-tools bench-edlib-native benchmark-report benchmark-report-native native-exact-gate evidence-gallery evidence-gallery-ready repro repro-small asan shared python-test python-package-test workflow-integration-test docs-ready repository-ready release-ready pretag-ready assay-evidence-ready scientific-readiness-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready workflow-adoption-status reviewer-readiness-ready distribution-record-ready bioconda-recipe-ready distribution-channels joss-paper-ready
+.PHONY: all clean test cli-test coverage bench bench-batch bench-small bench-native-matrix bench-10x-control figures bench-real-report bench-gpu gpu-report gpu-evidence-gate bench-barcode-demux bench-barcode-panel-design bench-barcode-demux-competitors bench-barcode-comparison barcode-autopsy-demo barcode-panel-design-gate barcode-validation-ready barcode-comparison-report barcode-comparison-gate barcode-demux-report barcode-competitor-env fetch-barcode-demo fetch-barcode-demo-claim fetch-sanson-crispr fetch-10x-bcl-demo bench-bcl-small bench-bcl-10x bench-bcl-real bench-bcl-real-repeated bcl-figures bcl-competitor-env bcl-linux-env bcl-tiny-public-gate bcl-comparison-gate fetch-oligo-adapter-demo bench-oligo-adapter bench-oligo-adapter-public oligo-adapter-smoke-gate oligo-adapter-public-gate fetch-amplicon-panel-demo bench-amplicon-panel bench-amplicon-panel-public amplicon-panel-smoke-gate amplicon-panel-public-gate fetch-feature-barcode-demo bench-feature-barcode bench-feature-barcode-public feature-barcode-smoke-gate feature-barcode-public-gate fetch-perturb-seq-demo bench-perturb-seq bench-perturb-seq-public perturb-seq-smoke-gate perturb-seq-public-gate bench-public-crispr-small bench-public-crispr bench-public-crispr-competitors bench-public-crispr-repeated bench-public-crispr-scaling bench-real-competitors bench-crispr-comparison bench-crispr-hamming-k23 crispr-comparison-report crispr-comparison-gate count-agreement count-agreement-comparison validate-public-crispr-edlib validate-crispr-comparison-edlib public-crispr-report public-crispr-evidence-gate public-crispr-smoke-gate competitor-env edlib edlib-tools bench-edlib-native benchmark-report benchmark-report-native native-exact-gate evidence-gallery evidence-gallery-ready repro repro-small asan shared python-test python-package-test workflow-integration-test docs-ready repository-ready release-ready pretag-ready assay-evidence-ready scientific-readiness-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready workflow-adoption-status reviewer-readiness-ready distribution-record-ready bioconda-recipe-ready distribution-channels joss-paper-ready
 
 all: dotmatch libdotmatch.a qda libqdalign.a
 
@@ -132,6 +132,12 @@ build/bench_batch: tools/bench_batch.c build/qdalign.o include/qdalign.h | build
 bench-batch: build/bench_batch
 	./build/bench_batch
 
+build/bench_hamming_control: tools/bench_hamming_control.c build/qdalign.o include/qdalign.h | build
+	$(CC) $(CFLAGS) tools/bench_hamming_control.c build/qdalign.o -o $@ $(LDFLAGS)
+
+bench-10x-control: build/bench_hamming_control
+	./build/bench_hamming_control $${DOTMATCH_10X_READS:-1000000} $${DOTMATCH_10X_TARGETS:-4096} $${DOTMATCH_10X_LENGTH:-20} $${DOTMATCH_10X_K:-1} $${DOTMATCH_10X_REPEATS:-5}
+
 ifeq ($(UNAME_S),Darwin)
 build/bench_gpu_metal: tools/bench_gpu_metal.mm build/qdalign.o include/qdalign.h | build
 	$(CXX) $(CXXFLAGS) tools/bench_gpu_metal.mm build/qdalign.o -o $@ $(LDFLAGS) -framework Foundation -framework Metal

docs/10x-evaluation-request.md

@@ -0,0 +1,71 @@
+# DotMatch evaluation request
+
+DotMatch is looking for a small number of independent evaluations on real
+known-target short-DNA workflows. This is a request for evidence, not a claim
+of adoption or performance.
+
+## Who this is for
+
+- CRISPR guide counting or screen QC;
+- barcode demultiplexing or whitelist assignment;
+- primer, amplicon, or targeted-panel matching;
+- a core facility or production pipeline that already has a deterministic
+  target set and a measurable counting or QC step.
+
+The workflow must have a declared target set and an explicit matching contract.
+DotMatch is not a genome aligner and does not emit SAM/BAM or CIGAR output.
+
+## Smallest useful evaluation
+
+1. Install a tagged DotMatch release from the documented package or source
+   route.
+2. Record the target-file format, read window, metric (`hamming` or
+   `levenshtein`), threshold, ambiguity policy, and DotMatch version.
+3. Run the equivalent existing tool or workflow on the same inputs when one
+   exists.
+4. Compare at least one outcome and one operational measure: assignment
+   agreement, ambiguous/none rates, counts, wall time, peak memory, or failure
+   rate.
+5. Keep private data private. A public-safe report can describe the workflow
+   class and aggregate results without exposing sequences or identifiers.
+
+The project can provide a command template and help interpret a result, but an
+independent evaluator should own the input, comparator, and conclusion.
+
+## Evidence record
+
+For a public-safe record, open an issue or pull request containing:
+
+```text
+DotMatch version and install route:
+Workflow/data class:
+Input scope and read window:
+Metric, threshold, and ambiguity policy:
+Comparator or truth source:
+Reads/records evaluated:
+Expected result:
+Observed result:
+Operational metric (if measured):
+Hardware/software environment:
+What this does not establish:
+Publication/permission status:
+```
+
+Do not include private sequences, patient information, proprietary sample
+identifiers, or an organization's name without permission. A project-authored
+example, an open integration submission, a download count, or a repository
+view is not an independent impact record.
+
+## How records are counted for the 10x goal
+
+Scientific records require an independently authored public URL, release and
+install route, workflow/data scope, expected-versus-observed result, and a
+clear boundary on the conclusion. An integration counts only after an upstream
+release or public external-use evidence.
+
+Industry records additionally require permission to describe the evaluation
+and a measured before/after operational metric. Confidential data may remain
+private; the existence and wording of the record still require approval.
+
+To start, use the public repository issue tracker and title the report
+`Independent evaluation: <workflow class>`. No data upload is required.

docs/10x-goal-baseline.md

@@ -0,0 +1,119 @@
+# DotMatch 10x goal: baseline and evidence gates
+
+This document turns the 10x objective into measurable project targets. It is a
+planning and evidence boundary, not a claim that any target has already been
+met. Accuracy, speed, scientific impact, and industry impact are measured
+separately; downloads, stars, open pull requests, and local synthetic tests do
+not substitute for the relevant evidence.
+
+## Baseline snapshot
+
+The baseline is the public DotMatch 0.2.2 evidence in this repository. The
+canonical working checkout is intentionally not used for this snapshot because
+it contains unrelated uncommitted changes.
+
+| Dimension | Current recorded evidence | Baseline used for the goal |
+| --- | --- | --- |
+| Scientific accuracy | The CRISPR comparison report records six public-data Edlib-oracle rows of 10,000 reads each, all with zero assignment mismatches. Count agreement, ambiguity, and outcome semantics are reported separately. | 60,000 oracle-checked reads, zero observed mismatches. Because zero is an observed floor, not a measurable positive accuracy margin, improvement is defined as 10x broader independent validation with the same zero-mismatch and semantic gates. |
+| Speed | The full Sanson/Brunello public Hamming `k=1` row records 634,950.2 reads/s and 388.9288 seconds against guide-counter at 473,609.5 reads/s. The report marks this as a single full-data row, not a repeated claim. A new deterministic indexed-Hamming control is documented in `docs/10x-performance-control.md`. | 634,950.2 reads/s and 388.9288 seconds on the recorded workload and hardware. A repeated baseline must be captured before publishing a speed multiplier. The local control is diagnostic evidence only. |
+| Scientific impact | `docs/workflow-adoption.json` is `not_ready` with no accepted external workflow entries. `docs/adopters/` contains only the record template. Open external PRs are not counted as adoption. | Zero verified independent public use records in the project evidence registry. A multiplicative ratio from zero is undefined, so the 10x operational target is ten independently authored, public-safe scientific evaluation or use records. |
+| Industry impact | There is no permissioned public industry, core-facility, or production-use record in `docs/adopters/`. Package availability and downloads are distribution evidence, not industry impact. | Zero verified industry-use records in the project evidence registry. The 10x operational target is ten permissioned evaluation or deployment records with workflow scope and at least one before/after operational metric. |
+
+### Baseline boundaries
+
+- The two public CRISPR datasets and existing benchmark lanes demonstrate
+  scoped scientific evidence; they do not establish universal accuracy or
+  superiority.
+- The local differential-fuzz suite now exercises literal ambiguity symbols,
+  lowercase bytes, mixed target lengths, and targets beyond the 32-base encoded
+  path. It is a regression guard, not independent scientific validation, and
+  does not count toward the 600,000-read target.
+- A zero-mismatch result is retained as zero mismatches. It is not converted
+  into a claim of perfect accuracy.
+- Speed comparisons must preserve input data, target set, read window, metric,
+  ambiguity policy, output contract, hardware, software versions, warm-up
+  policy, repeat count, and memory measurement.
+- Scientific and industry records require an external author or an explicitly
+  approved organization. A project-authored integration PR, download counter,
+  repository view, or unapproved logo is not an impact record.
+
+## 10x targets
+
+### 1. Scientific accuracy
+
+Pass the accuracy target only when all of the following are true:
+
+1. At least 600,000 reads are checked across at least ten public or
+   permission-cleared datasets/lanes that were not used to tune the matcher.
+2. Each lane has a declared truth source or independent edit-distance oracle,
+   recorded commands, versions, input window, target set, and ambiguity policy.
+3. The aggregate assignment mismatch count is zero, and the report retains
+   `unique`, `ambiguous`, `none`, and `invalid` outcomes rather than collapsing
+   them into a single accuracy number.
+4. Count-level agreement and relevant per-read invariants pass for every lane.
+
+With zero mismatches, the simple one-sided Rule-of-3 upper bound moves from
+approximately `3/60,000` to `3/600,000`, a tenfold tighter observed-error bound.
+This is the defensible interpretation of “10x more accurate” for a system whose
+current checked error count is already zero.
+
+### 2. Performance
+
+Pass the speed target only when a five-repeat baseline and five-repeat candidate
+are run on the same full Sanson/Brunello Hamming `k=1` workload and the same
+declared hardware/software environment:
+
+- candidate throughput: at least 6,349,502 reads/s, or
+- candidate wall time: at most 38.8929 seconds,
+
+while preserving zero oracle mismatches, count semantics, output equivalence,
+and a separately reported peak-memory budget. If the workload or comparator
+changes, it is a new benchmark lane and cannot be called a 10x improvement of
+this baseline.
+
+### 3. Scientific impact
+
+Reach ten independently authored public-safe scientific evaluation or use
+records. Each record must include a public URL, DotMatch version, install
+route, workflow/data class, expected-versus-observed result, and the boundary
+of what was actually tested. Accepted nf-core, Galaxy, MultiQC, Snakemake, or
+other workflow integrations may count as records only after upstream release
+or explicit external use evidence is public.
+
+### 4. Industry impact
+
+Reach ten permissioned records from distinct organizations, core facilities, or
+production workflow owners. Each record must document the workflow scope, the
+release used, the operational problem, and at least one measured before/after
+metric such as runtime, memory, failure rate, QC visibility, or reproducibility.
+Confidential data may remain private, but the existence and wording of the
+record must be approved before publication. Downloads alone never satisfy this
+gate.
+
+## Evidence ledger
+
+The target ledger should keep these fields separate:
+
+- accuracy: dataset, truth source, reads checked, mismatches, ambiguity/outcome
+  counts, and reproducibility command;
+- performance: workload, comparator, hardware, versions, repeat statistics,
+  throughput, wall time, peak memory, and output-equivalence status;
+- scientific impact: external author, public URL, workflow, release, result,
+  and scope boundary;
+- industry impact: permission status, organization or anonymized identifier,
+  workflow, release, before/after metric, and approved wording.
+
+Do not combine the four dimensions into a single score until the underlying
+records exist. A 10x claim is valid only when the corresponding gate above is
+green and the evidence is independently auditable.
+
+## Current next actions
+
+1. Re-run the full Sanson/Brunello Hamming `k=1` lane five times and freeze the
+   baseline artifact before optimizing.
+2. Expand held-out oracle validation to ten lanes without changing semantics.
+3. Convert open workflow submissions into accepted/released integrations before
+   counting scientific impact.
+4. Invite independent scientific and industry evaluators through
+   `docs/10x-evaluation-request.md`; do not publish private data or infer impact
+   from replies alone.

docs/10x-performance-control.md

@@ -0,0 +1,38 @@
+# Reproducible 10x performance control
+
+The `bench-10x-control` target measures the reusable indexed Hamming API on a
+deterministic synthetic workload. It is a local hot-path control, not an
+end-to-end scientific workflow claim and not a replacement for the public
+Sanson/Brunello comparison.
+
+```bash
+make bench-10x-control
+```
+
+The default control is 1,000,000 reads, 4,096 targets, 20-base windows,
+Hamming `k=1`, and five timed repeats. Override the workload when diagnosing a
+different lane:
+
+```bash
+DOTMATCH_10X_READS=1000000 \
+DOTMATCH_10X_TARGETS=4096 \
+DOTMATCH_10X_LENGTH=20 \
+DOTMATCH_10X_K=1 \
+DOTMATCH_10X_REPEATS=5 \
+make bench-10x-control
+```
+
+Before timing, the harness compares a 256-read prefix against the exhaustive
+scan API. Each repeat reports throughput, candidate rates, and a checksum;
+different checksums or a failed preflight invalidate the run. The target
+library is built once outside the timed region.
+
+For an auditable comparison, record the exact commit, compiler and flags,
+machine model, operating-system version, workload arguments, repeat rows,
+peak-memory method, and whether other CPU-heavy work was running. A noisy or
+partially captured run is a control failure, not a speed result.
+
+The project speed gate remains the repeated full Sanson/Brunello Hamming
+`k=1` lane in `docs/10x-goal-baseline.md`. A local synthetic control can
+identify a hot path and reject regressions; it cannot establish a 10x
+end-to-end improvement by itself.

docs/index.md

@@ -39,6 +39,8 @@ the [command reference](command-reference.md).
 | Use DotMatch from Python | [Streaming Python API](streaming-api.md) |
 | Add DotMatch to a pipeline | [Output schemas](schemas.md) |
 | Evaluate DotMatch for a workflow | [Bioinformatics evaluation](bioinformatics-evaluation.md) |
+| Run an independent evaluation | [10x evaluation request](10x-evaluation-request.md) |
+| Review the 10x evidence gates | [10x goal baseline](10x-goal-baseline.md) |
 | Record the software in a methods section | [Methods and citation](methods-and-citation.md) |
 
 ## Scope and limitations
@@ -92,6 +94,9 @@ schemas
 workbench
 bioinformatics-evaluation
 external-review-packet
+10x-evaluation-request
+10x-goal-baseline
+10x-performance-control
 ```
 
 ```{toctree}

tests/test_qdalign.c

@@ -1247,6 +1247,48 @@ static void index_fuzz_tests(void) {
     }
 }
 
+static void random_literal_sequence(char *sequence, size_t length) {
+    static const char alphabet[] = "ACGTNRYacgtX-";
+    for (size_t i = 0; i < length; ++i) {
+        sequence[i] = alphabet[xorshift64() % (sizeof(alphabet) - 1U)];
+    }
+    sequence[length] = '\0';
+}
+
+static void literal_alphabet_index_fuzz_tests(void) {
+    char reads_buf[8][41];
+    char targets_buf[8][41];
+    const char *reads[8];
+    const char *targets[8];
+    size_t read_lens[8];
+    size_t target_lens[8];
+    qdaln_match_result scan[8];
+    qdaln_match_result indexed[8];
+
+    for (size_t trial = 0; trial < 10000; ++trial) {
+        size_t n_reads = 1 + (size_t)(xorshift64() % 8ULL);
+        size_t n_targets = 1 + (size_t)(xorshift64() % 8ULL);
+        int k = (int)(xorshift64() % 4ULL);
+        for (size_t i = 0; i < n_reads; ++i) {
+            read_lens[i] = (size_t)(xorshift64() % 41ULL);
+            random_literal_sequence(reads_buf[i], read_lens[i]);
+            reads[i] = reads_buf[i];
+        }
+        for (size_t i = 0; i < n_targets; ++i) {
+            target_lens[i] = (size_t)(xorshift64() % 41ULL);
+            random_literal_sequence(targets_buf[i], target_lens[i]);
+            targets[i] = targets_buf[i];
+        }
+
+        qdaln_index *idx = qdaln_index_build(targets, target_lens, n_targets);
+        assert(idx != NULL);
+        assert(qdaln_match_many(reads, read_lens, n_reads, targets, target_lens, n_targets, k, scan) == 0);
+        assert(qdaln_index_assign(idx, reads, read_lens, n_reads, k, indexed) == 0);
+        for (size_t i = 0; i < n_reads; ++i) assert_match_result(indexed[i], scan[i]);
+        qdaln_index_free(idx);
+    }
+}
+
 int main(void) {
     fixed_tests();
     alphabet_policy_tests();
@@ -1276,6 +1318,7 @@ int main(void) {
     fuzz_tests();
     batch_fuzz_tests();
     index_fuzz_tests();
+    literal_alphabet_index_fuzz_tests();
     puts("qdalign tests passed");
     return 0;
 }