diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2be0e4a..729157b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,3 +35,36 @@ jobs: - name: Run tests run: uv run pytest + + # The materializer owns the `files` collection and its indexes, so its + # tests are the only guard on that index list. They are a separate job + # rather than a step above because they do not vary with the Python + # matrix -- running them per-version would compile the crate three times + # to assert the same thing. + materialize: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + # Pinned rather than inherited from the runner image: without this the + # compiler version drifts whenever the image is rebuilt, which is how a + # guard job starts failing on an unrelated PR and gets labelled flaky. + # SHA-pinned like every other action here, so the action's own code + # cannot change under us. The channel stays `stable` rather than an + # exact version: the crate declares edition 2021 and no rust-version, + # so there is no MSRV to hold it to, and pinning one would only add a + # bump to maintain. + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + with: + toolchain: stable + + - name: Cache cargo build + uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7 + with: + workspaces: materialize + + # --locked so the committed Cargo.lock is authoritative; without it a + # semver-compatible upstream release can change what CI compiles. + - name: Run materializer tests + run: cargo test --locked --manifest-path materialize/Cargo.toml diff --git a/.gitignore b/.gitignore index c864d55..cdc1371 100644 --- a/.gitignore +++ b/.gitignore @@ -77,7 +77,10 @@ uv.lock # Rust materialize/target/ -Cargo.lock +# materialize/Cargo.lock is deliberately tracked: the materializer is a +# binary crate and CI's `cargo test --locked` is the only guard on its +# index list, so an untracked lockfile let an unrelated upstream release +# turn that guard red. # Database database/ diff --git a/4DN-SUPPLEMENT.md b/4DN-SUPPLEMENT.md index f0a8ad3..a2e3afa 100644 --- a/4DN-SUPPLEMENT.md +++ b/4DN-SUPPLEMENT.md @@ -17,7 +17,16 @@ Field mapping from the 4D Nucleome (4DN) Search API and C2M2 datapackage to the | Collection | `persistent_id` contains `4DNEX*` or `4DNES*` | `accession` (e.g., `4DNEXH4ZUIH6`) | | Biosource tier | `extra.fourdn.biosource_name` | `Biosource.display_title` | -Accessions are extracted from persistent_id URLs via regex: `4DNF[A-Z0-9]+` for files, `4DNE[A-Z][A-Z0-9]+` for experiments. +Accessions are extracted from persistent_id URLs via regex: `4DNF[A-Z0-9]+` for files, `4DNE[A-Z][A-Z0-9]+` for experiments. Both are matched case-insensitively, and the extracted value is case-folded before use — an upper-case-only pattern does not merely miss a mixed-case accession, it matches the upper-case prefix and returns a truncated one. + +Both accessions are also persisted, so they can be queried directly rather than reconstructed: + +| CFDB Field | Source | Notes | +|------------|--------|-------| +| `accession_id` | `file.persistent_id` `4DNF*` | Stamped onto the raw `file` collection pre-materialization, case-folded | +| `collections[].accession_id` | `collection.persistent_id` `4DNE*` | Stamped onto the raw `collection` collection pre-materialization, case-folded | + +Both are stamped *before* materialization deliberately. The materializer rebuilds `files` from the raw collections on every run, so a value written afterwards would be erased by any standalone `make materialize-dcc DCC=4dn` — silently, leaving accession lookups returning nothing for the whole DCC. ## Materialization (Rust) diff --git a/ENCODE-SUPPLEMENT.md b/ENCODE-SUPPLEMENT.md index f114bfd..ce87a61 100644 --- a/ENCODE-SUPPLEMENT.md +++ b/ENCODE-SUPPLEMENT.md @@ -83,6 +83,7 @@ ENCODE uses human-readable strings for file formats, assay types, output types, | CFDB Field | ENCODE TSV Column | Type | Notes | |------------|-------------------|------|-------| | `local_id` | `File accession` | string | ENCODE accession (e.g., `ENCFF001ABC`) | +| `accession_id` | `File accession` | string | The same accession, case-folded to upper case. Duplicates `local_id` for ENCODE, which stores the accession there; the separate field exists for cross-DCC uniformity, since 4DN's `local_id` is an opaque UUID. Folded so an `accessionId` filter matches in any casing, which means it can legitimately differ from `local_id` in case. | | `id_namespace` | — | string | Constant: `https://www.encodeproject.org` | | `filename` | `File download URL` | string | Basename extracted from URL | | `access_url` | `File download URL` | string | Full HTTPS download URL | @@ -124,6 +125,7 @@ One collection per unique experiment accession, embedded on `file.collections[]` | CFDB Field | ENCODE TSV Column | Notes | |------------|-------------------|-------| | `local_id` | `Experiment accession` | e.g., `"ENCSR000AAA"` | +| `accession_id` | `Experiment accession` | The same accession, case-folded to upper case | | `name` | `Experiment accession` | Same as `local_id` | | `persistent_id` | `Experiment accession` | `https://www.encodeproject.org/experiments/{accession}/` | | `anatomy[]` | `Biosample term id` + `Biosample term name` | `{id, name}` object | @@ -131,7 +133,7 @@ One collection per unique experiment accession, embedded on `file.collections[]` | `subjects[]` | `Donor(s)` | Subject records (see below) | | `extra.encode` | — | Experiment-level metadata (see below) | -**Fallback**: if `Experiment accession` is missing, falls back to biosample-keyed collection (`biosample:{name}`). +**Fallback**: if `Experiment accession` is missing, falls back to biosample-keyed collection (`biosample:{name}`). That fallback collection is synthesized locally and names no ENCODE experiment, so it carries no `accession_id` rather than a fabricated one. Note also that the whole collection block is gated on `Biosample term name`: a row with an experiment accession but no biosample term produces no collection at all, so that experiment's accession is queryable nowhere. #### Collection Lab (top-level) diff --git a/README.md b/README.md index 5eb151d..06445b8 100644 --- a/README.md +++ b/README.md @@ -490,6 +490,7 @@ The central entity representing a stable digital asset. | `sha256` | string? | SHA-256 checksum (preferred) | | `md5` | string? | MD5 checksum (if SHA-256 unavailable) | | `filename` | string | Filename without path | +| `accession_id` | string? | The DCC's own accession for this file, stored upper-cased so an `accessionId` filter matches in any casing. Populated for 4DN and ENCODE; always null for HuBMAP (see the note below). | | `file_format` | FileFormat? | EDAM CV term for digital format | | `compression_format` | string? | EDAM CV term ID for compression (e.g., `format:3989` for gzip); `""` when no compression is recorded or recognized; null/absent when undetermined. Read the note below before relying on it. | | `data_type` | DataType? | EDAM CV term for data type | @@ -508,6 +509,8 @@ The central entity representing a stable digital asset. ENCODE derives the value from the download URL's filename suffix, because the ENCODE metadata TSV has no compression column. Two consequences are worth knowing. The field is **absent** (rendered as null) rather than `""` when nothing could be determined — no filename in the URL, or a compression suffix no EDAM term expresses (`.bz2`, `.xz`, `.zst`, `.zip`, `.starch`) — so treat null as "sniff the bytes", never as "uncompressed". And `format:3989` means "gzip-family stream": ENCODE names both plain gzip and BGZF `.gz` (it publishes no `.bgz` at all, and roughly a quarter of its `.gz` files are BGZF), so the value cannot distinguish them. Anything deciding on `gunzip | bgzip` must read the BGZF header — which is what `cfdb.workflows.processors.tabix` does, deliberately, and that byte-level check remains the decision of record. +**A note on `accession_id`, because a null does not mean what it looks like.** The field exists so one input works across DCCs: 4DN puts an opaque UUID in `local_id` and carries its accession only inside the `persistent_id` URL, while ENCODE stores the accession *as* `local_id`. It is stored case-folded and filter values are folded identically, so `accessionId: ["4dnfimcjxzkh"]` and `["4DNFIMCJXZKH"]` match the same file. Three different situations all render as a null field and a `totalCount` of 0, and the API cannot distinguish them for you: the accession genuinely does not exist; the DCC issues none (all of HuBMAP, which matches files by filename within a dataset — tracked in [#102](https://github.com/abdenlab/cfdb/issues/102)); or that DCC has not been synced since the field was added. **A deployment must re-sync each DCC before `accessionId` returns anything.** Each sync logs its coverage (`4DN accession coverage: 53697/53697 files carry accession_id`), which is the only place that distinction is visible. + #### Dcc A Common Fund program or Data Coordinating Center. @@ -535,6 +538,7 @@ A grouping of files, biosamples, and/or subjects. | `name` | string | Human-readable label | | `description` | string? | Human-readable description | | `lab` | string? | Lab/PI name (shared across 4DN and ENCODE) | +| `accession_id` | string? | The DCC's accession for the experiment this collection represents, stored upper-cased (shared across 4DN and ENCODE). Null on ENCODE's biosample-keyed fallback collections, which name no experiment, and on all of HuBMAP. | | `extra` | EnrichedCollection? | DCC-specific collection metadata (see EnrichedCollection) | #### Biosample diff --git a/materialize/Cargo.lock b/materialize/Cargo.lock new file mode 100644 index 0000000..9f2dd7f --- /dev/null +++ b/materialize/Cargo.lock @@ -0,0 +1,2545 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64", + "bitvec", + "chrono", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap", + "js-sys", + "once_cell", + "rand 0.9.5", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest", +] + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +dependencies = [ + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "materialize" +version = "0.1.0" +dependencies = [ + "anyhow", + "bson", + "clap", + "indicatif", + "mongodb", + "rayon", + "regex", + "serde", + "serde_json", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "mongocrypt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8426a875ded61430d4a811dbfda7633b6b8af0225c547fc6c28b8b0aa7d79a13" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.6+1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" + +[[package]] +name = "mongodb" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b814038f367d212f55de0a630cb35102a9b8ca23785a86955d62c0087c93846d" +dependencies = [ + "base64", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-net", + "hickory-proto", + "hickory-resolver", + "hmac", + "macro_magic", + "md-5", + "mongocrypt", + "mongodb-internal-macros", + "openssl", + "openssl-probe", + "pbkdf2", + "percent-encoding", + "rand 0.9.5", + "rustc_version_runtime", + "rustls", + "serde", + "serde_bytes", + "serde_with", + "sha1", + "sha2", + "socket2", + "stringprep", + "strsim", + "take_mut", + "thiserror", + "tokio", + "tokio-openssl", + "tokio-rustls", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f736d2fbc56e0011a341fbb9172bd822fda75c5f93b82fae1c7aab1e2613c810" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_with" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd" +dependencies = [ + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/materialize/src/main.rs b/materialize/src/main.rs index d6ab01b..87458cc 100644 --- a/materialize/src/main.rs +++ b/materialize/src/main.rs @@ -840,6 +840,10 @@ fn index_keys() -> Vec { vec![ doc! { "id_namespace": 1 }, doc! { "local_id": 1 }, + // Cross-DCC accession lookup. Stored already case-folded (see + // cfdb.accessions), so this plain index serves the case-insensitive + // match DocumentDB 5.0 cannot serve via collation. + doc! { "accession_id": 1 }, doc! { "id_namespace": 1, "local_id": 1 }, doc! { "persistent_id": 1 }, doc! { "filename": 1 }, @@ -854,6 +858,7 @@ fn index_keys() -> Vec { doc! { "assay_type.name": 1 }, doc! { "collections.id_namespace": 1 }, doc! { "collections.local_id": 1 }, + doc! { "collections.accession_id": 1 }, doc! { "collections.name": 1 }, // Collection anatomy indexes doc! { "collections.anatomy.id": 1 }, @@ -1031,6 +1036,62 @@ mod tests { "anatomy field should be removed when raw value is empty string"); } + #[test] + /// Test that accession_id survives materialization at both levels. + /// + /// Both 4DN accessions are stamped on the raw collections + /// pre-materialization, so both reach the files collection only + /// because enrich_file carries them: the file's by mutating the raw + /// document in place, the collection's by cloning the whole + /// collection document. Nothing else verifies either hop, and both + /// would fail silently -- the accession would simply be absent, + /// indistinguishable from a DCC that issues none. + /// + /// This is also what makes stamping the raw collections load-bearing + /// rather than incidental: the materializer rebuilds files from the + /// raw documents on every run, so a value written to files instead + /// would not survive a standalone `make materialize-dcc`. + /// + /// Given: + /// A raw file document and a raw collection document, each carrying + /// an accession_id. + /// When: + /// enrich_file processes the file. + /// Then: + /// It should carry both accessions onto the materialized document. + fn test_enrich_file_propagates_accession_id() { + // Arrange + let biosample = doc! { + "id_namespace": "4dn", + "local_id": "bio-001", + }; + let (mut file, mut lookups) = + lookups_with_biosample(biosample, HashMap::new()); + file.insert("accession_id", "4DNFIMCJXZKH"); + lookups + .collections + .get_mut(&("4dn".to_string(), "coll-001".to_string())) + .expect("collection fixture present") + .insert("accession_id", "4DNEXNHE6X77"); + + // Act + let result = enrich_file(file, &lookups); + + // Assert + assert_eq!( + result.get_str("accession_id").unwrap(), + "4DNFIMCJXZKH", + "file accession_id should survive materialization" + ); + let collections = result.get_array("collections").unwrap(); + let coll = collections[0].as_document().unwrap(); + assert_eq!( + coll.get_str("accession_id").unwrap(), + "4DNEXNHE6X77", + "collection accession_id should be carried by the document clone" + ); + } + #[test] /// Test biosample anatomy replacement when a matching anatomy document exists. /// @@ -1219,14 +1280,15 @@ mod tests { fn index_keys_returns_expected_set() { // GIVEN the index_keys function // WHEN called - // THEN it returns exactly 47 index key documents matching the expected fields + // THEN it returns exactly 49 index key documents matching the expected fields let keys = index_keys(); - assert_eq!(keys.len(), 47); + assert_eq!(keys.len(), 49); assert_eq!( keys, vec![ doc! { "id_namespace": 1 }, doc! { "local_id": 1 }, + doc! { "accession_id": 1 }, doc! { "id_namespace": 1, "local_id": 1 }, doc! { "persistent_id": 1 }, doc! { "filename": 1 }, @@ -1241,6 +1303,7 @@ mod tests { doc! { "assay_type.name": 1 }, doc! { "collections.id_namespace": 1 }, doc! { "collections.local_id": 1 }, + doc! { "collections.accession_id": 1 }, doc! { "collections.name": 1 }, doc! { "collections.anatomy.id": 1 }, doc! { "collections.anatomy.name": 1 }, diff --git a/schema.graphql b/schema.graphql index 40055ba..079dcaf 100644 --- a/schema.graphql +++ b/schema.graphql @@ -59,6 +59,7 @@ input CollectionInput { biosamples: [BiosampleInput!] = null idNamespace: [String!] = null localId: [String!] = null + accessionId: [String!] = null persistentId: [String!] = null creationTime: [String!] = null abbreviation: [String!] = null @@ -77,6 +78,7 @@ type CollectionType { biosamples: [BiosampleType!]! idNamespace: String! localId: String! + accessionId: String persistentId: String creationTime: String abbreviation: String @@ -408,6 +410,7 @@ input FileMetadataInput { project: [ProjectInput!] = null idNamespace: [String!] = null localId: [String!] = null + accessionId: [String!] = null projectIdNamespace: [String!] = null projectLocalId: [String!] = null persistentId: [String!] = null @@ -444,6 +447,7 @@ type FileMetadataType { project: ProjectType idNamespace: String! localId: String! + accessionId: String projectIdNamespace: String! projectLocalId: String! persistentId: String diff --git a/src/cfdb/accessions.py b/src/cfdb/accessions.py new file mode 100644 index 0000000..2052af7 --- /dev/null +++ b/src/cfdb/accessions.py @@ -0,0 +1,53 @@ +"""Normalization for the cross-DCC ``accession_id`` field. + +``accession_id`` gives every DCC one queryable name for the identifier +users actually recognize -- ``4DNFIMCJXZKH`` for a 4DN file, +``ENCSR918ZSJ`` for an ENCODE experiment -- independent of where each DCC +happens to keep it (4DN buries it in ``persistent_id``; ENCODE stores it +as ``local_id``). + +Callers expect that lookup to be case-insensitive, and the usual way to +get that -- a collation-bearing index -- is unavailable here. Amazon +DocumentDB 5.0, which backs the deployed environments, supports neither +the ``Case Insensitive`` index property nor ``cursor.collation()`` (both +land only in DocumentDB 8.0), and a case-insensitive ``$regex`` cannot +use an index at all. A collation-based implementation would also *pass* +against a developer's local MongoDB and fail only once deployed. + +So the field is normalized rather than collated: it is stored already +folded to :func:`normalize_accession`'s output, and filter values are +folded the same way at the API boundary, leaving an ordinary indexed +equality match that behaves identically on MongoDB and DocumentDB. + +Both sides MUST route through this one function. If the ingest form and +the query form ever diverge, nothing raises -- documents simply become +permanently unmatchable. + +Upper case is the fold direction because it is the form both DCCs +already publish, so the stored value stays the display value and no +second field is needed to recover it. +""" + +from __future__ import annotations + +__all__ = ["normalize_accession"] + + +def normalize_accession(value: str | None) -> str | None: + """Fold an accession into its canonical stored/queried form. + + Args: + value: An accession in any case, optionally surrounded by + whitespace. ``None`` and blank strings are accepted. + + Returns: + The accession stripped and upper-cased, or ``None`` when + ``value`` is ``None``, blank, or whitespace-only. Returning + ``None`` rather than ``""`` keeps an absent accession out of the + index and out of ``distinctValues``, matching how the other + optional model fields treat "no value". + """ + if value is None: + return None + normalized = value.strip().upper() + return normalized or None diff --git a/src/cfdb/api/gql/inputs.py b/src/cfdb/api/gql/inputs.py index 27974a9..2c1189d 100644 --- a/src/cfdb/api/gql/inputs.py +++ b/src/cfdb/api/gql/inputs.py @@ -2,8 +2,23 @@ import strawberry +from cfdb.accessions import normalize_accession from cfdb.api.gql.types import BigInt +#: Fully-flattened filter paths whose values are folded by +#: :func:`cfdb.accessions.normalize_accession` before becoming a MongoDB +#: predicate. Enumerated in full rather than matched on the last dotted +#: segment: the set of paths whose stored form is folded is fixed and known +#: statically, and leaf matching would silently opt in any future field +#: named ``accession_id`` at any depth. A DCC-native ``extra..accession_id`` +#: -- the shape this codebase already uses for upstream values -- would be +#: stored exactly as the DCC published it while being folded on query, +#: which is the failure ``cfdb.accessions`` warns about: nothing raises, +#: the documents simply become permanently unmatchable. Matching the whole +#: path makes a new field fail closed (unfolded, byte-exact, like every +#: other field) instead. +_NORMALIZED_PATHS = {"accession_id", "collections.accession_id"} + @strawberry.input class AnatomyInput: @@ -143,6 +158,7 @@ class CollectionInput: biosamples: list[BiosampleInput] | None = None id_namespace: list[str] | None = None local_id: list[str] | None = None + accession_id: list[str] | None = None persistent_id: list[str] | None = None creation_time: list[str] | None = None abbreviation: list[str] | None = None @@ -241,6 +257,7 @@ class FileMetadataInput: project: list[ProjectInput] | None = None id_namespace: list[str] | None = None local_id: list[str] | None = None + accession_id: list[str] | None = None project_id_namespace: list[str] | None = None project_local_id: list[str] | None = None persistent_id: list[str] | None = None @@ -289,9 +306,43 @@ def to_dict(obj): return result +def _predicate(key, value): + """Build a single equality predicate, folding normalized fields. + + Every leaf of a filter becomes a bare equality match, so a value that + does not match the stored form byte-for-byte matches nothing. For + ``accession_id`` the stored form is + :func:`cfdb.accessions.normalize_accession`'s output, so the filter + value is folded the same way here -- the one place a scalar becomes a + predicate -- rather than at each ``to_query`` call site, where a later + caller could bypass it. + + An accession that folds to ``None`` (blank or whitespace only) + contributes no clause: the empty dict returned here is dropped by + ``to_query``. Emitting ``{field: None}`` instead would match documents + whose accession is null *or absent* -- every HuBMAP file, every 4DN + file whose accession did not parse, and the whole corpus before the + first post-deploy sync. That made a blank value the only filter in the + schema that *widens* the result set, where every sibling string field + matches nothing; a search box wired straight to the variable returned + a page of unrelated files rather than no results. + """ + if key in _NORMALIZED_PATHS and isinstance(value, str): + folded = normalize_accession(value) + return {key: folded} if folded is not None else {} + return {key: value} + + def to_query(obj, prefix=""): """ Convert a nested dict/list structure into a flattened MongoDB query. + + A branch that contributes no constraint -- an unset field, an accession + that folds away, or an empty list -- yields ``{}`` and is dropped by its + parent rather than left in place. MongoDB rejects ``{"$and": []}`` and + ``{"$or": []}`` outright, so an empty clause list has to collapse to + ``{}`` (match everything) rather than be emitted; without that, a filter + whose only value folded away would 500 instead of returning rows. """ if isinstance(obj, dict): and_clause = [] @@ -301,25 +352,29 @@ def to_query(obj, prefix=""): flattened = to_query(v, key) if isinstance(flattened, dict) and "$and" in flattened: and_clause.extend(flattened["$and"]) - else: + elif flattened: and_clause.append(flattened) elif v is not None: - and_clause.append({key: v}) + predicate = _predicate(key, v) + if predicate: + and_clause.append(predicate) + if not and_clause: + return {} if len(and_clause) == 1: return and_clause[0] - else: - return {"$and": and_clause} + return {"$and": and_clause} elif isinstance(obj, list): or_clause = [] for item in obj: flattened = to_query(item, prefix) if isinstance(flattened, dict) and "$or" in flattened: or_clause.extend(flattened["$or"]) - else: + elif flattened: or_clause.append(flattened) + if not or_clause: + return {} if len(or_clause) == 1: return or_clause[0] - else: - return {"$or": or_clause} + return {"$or": or_clause} else: - return {prefix: obj} if prefix else obj + return _predicate(prefix, obj) if prefix else obj diff --git a/src/cfdb/indexes.py b/src/cfdb/indexes.py index 7bffd5a..74f534f 100644 --- a/src/cfdb/indexes.py +++ b/src/cfdb/indexes.py @@ -158,12 +158,38 @@ def operational_index_specs() -> list[IndexSpec]: ] +def materialized_files_index_specs() -> list[IndexSpec]: + """Accession indexes on the materialized ``files`` collection. + + The Rust materializer owns ``files`` and creates its full index set at + the end of every run, so this list deliberately does not mirror it -- + duplicating ~49 keys in a second writer is what + :func:`data_index_specs` exists to avoid. + + It exists for the one path that never reaches the materializer at all: + ``_sync_encode`` writes ENCODE documents straight into ``files``, so on + a database where ENCODE is the only DCC synced, ``files`` would carry + no indexes whatsoever and every accession lookup would scan the whole + collection on a public endpoint. Ensuring just the accession keys there + costs nothing when the materializer has already created them: identical + keys derive identical default names, so the create is a no-op. + """ + return [ + IndexSpec("files", [("accession_id", 1)]), + IndexSpec("files", [("collections.accession_id", 1)]), + ] + + def data_index_specs() -> list[IndexSpec]: """Query-performance indexes for the loaded C2M2 data collections. Mirrors the data-collection portion of ``scripts/create-indexes.js``. Only useful after a sync has loaded data, so these are ensured in the sync/materialize path rather than at API startup. + + Deliberately excludes the materialized ``files`` collection, which the + Rust materializer owns; see :func:`materialized_files_index_specs` for + the narrow exception and why it does not conflict. """ specs: list[IndexSpec] = [] diff --git a/src/cfdb/models.py b/src/cfdb/models.py index 5c9fc62..73ad107 100644 --- a/src/cfdb/models.py +++ b/src/cfdb/models.py @@ -354,6 +354,15 @@ class FileMetadataModel(BaseModel): An identifier representing this file, unique within this id_namespace. Part 2 of 2-component composite primary key. + accession_id: + The DCC-issued file accession users recognize this file by (e.g. + "4DNFIMCJXZKH", "ENCFF525XQX"), giving every DCC one queryable name + for it regardless of where the DCC keeps it -- 4DN carries it only + inside persistent_id, ENCODE stores it as local_id. Stored already + folded by ``cfdb.accessions.normalize_accession`` so the equality + match is case-insensitive to callers without a collation DocumentDB + 5.0 does not support. None when the DCC issues no file accession. + project_id_namespace: The id_namespace of the primary project within which this file was created. Part 1 of 2-component composite foreign key. @@ -448,6 +457,7 @@ class Config: project: Optional[Project] = None id_namespace: str = str() local_id: str = str() + accession_id: Optional[str] = None project_id_namespace: str = str() project_local_id: str = str() persistent_id: Optional[str] = None @@ -480,6 +490,7 @@ class Config: @field_validator( "file_format", "data_type", "assay_type", "project", "extra", + "accession_id", mode="before", ) @classmethod @@ -618,6 +629,15 @@ class Collection(BaseModel): An identifier representing this collection, unique within this id_namespace. Part 2 of 2-component composite primary key. + accession_id: + The DCC-issued experiment accession users recognize this collection + by (e.g. "4DNEXNHE6X77", "ENCSR918ZSJ"), giving every DCC one + queryable name for it regardless of where the DCC keeps it. Stored + already folded by ``cfdb.accessions.normalize_accession`` so the + equality match is case-insensitive to callers without a collation + DocumentDB 5.0 does not support. None when the DCC issues no + experiment accession. + persistent_id: A persistent, resolvable (not necessarily retrievable) URI or compact ID permanently attached to this collection. @@ -651,6 +671,7 @@ class Collection(BaseModel): biosamples: List[Biosample] id_namespace: str = str() local_id: str = str() + accession_id: Optional[str] = None persistent_id: Optional[str] = None creation_time: Optional[str] = None abbreviation: Optional[str] = None @@ -664,7 +685,7 @@ class Collection(BaseModel): subjects: List[Subject] = [] extra: Optional[EnrichedCollection] = None - @field_validator("extra", mode="before") + @field_validator("extra", "accession_id", mode="before") @classmethod def empty_string_to_none(cls, v): if v == "": diff --git a/src/cfdb/services/encode.py b/src/cfdb/services/encode.py index 9ef4ca0..becc4ce 100644 --- a/src/cfdb/services/encode.py +++ b/src/cfdb/services/encode.py @@ -12,7 +12,7 @@ File ~~~~ -File accession → local_id +File accession → local_id, accession_id (case-folded) File download URL → access_url, filename (derived) File download URL → compression_format (suffix-derived; the TSV carries no compression column, and the field @@ -51,7 +51,8 @@ Collection ~~~~~~~~~~ -Experiment accession → collections[].local_id, name, persistent_id +Experiment accession → collections[].local_id, name, persistent_id, + accession_id (case-folded) Lab → collections[].lab Assay → collections[].experiment_type Experiment target → collections[].experiment_target @@ -108,6 +109,7 @@ import aiohttp +from cfdb.accessions import normalize_accession from cfdb.dcc_registry import get_dcc_config from cfdb.services.ontology_mappings import ( get_assay_type, @@ -368,6 +370,11 @@ def transform_to_c2m2(row: dict) -> Optional[dict]: "submission": "encode", "id_namespace": id_namespace, "local_id": accession, + # Duplicates local_id for ENCODE, which stores the accession there. + # The point of the separate field is cross-DCC uniformity: 4DN's + # local_id is an opaque UUID, so one accession_id input works for + # both. Folded so it matches what the GraphQL layer folds filters to. + "accession_id": normalize_accession(accession), "filename": filename, "size_in_bytes": size_in_bytes, "md5": _nonempty(row.get("md5sum")), @@ -532,6 +539,12 @@ def transform_to_c2m2(row: dict) -> Optional[dict]: "biosamples": [biosample], "subjects": subjects, } + # Only the experiment-keyed branch has an accession. The + # ``biosample:``-keyed fallback collection is synthesized locally and + # names no ENCODE experiment, so it is left unset rather than given a + # fabricated accession. + if experiment_accession: + collection["accession_id"] = normalize_accession(experiment_accession) if collection_persistent_id: collection["persistent_id"] = collection_persistent_id if anatomy: diff --git a/src/cfdb/services/fourdn.py b/src/cfdb/services/fourdn.py index abc4e2f..3c4fc1c 100644 --- a/src/cfdb/services/fourdn.py +++ b/src/cfdb/services/fourdn.py @@ -24,6 +24,21 @@ File persistent_id contains 4DNF[A-Z0-9]+ accession Collection persistent_id contains 4DNE[A-Z][A-Z0-9]+ accession +Both patterns are matched case-insensitively and the extractors return the +accession already case-folded, so the value that keys the Search API round +trip is the same one stored in ``accession_id``. + +Accession Stamping (persistent_id → CFDB) +----------------------------------------- +Both run *pre*-materialization, against the raw C2M2 collections, so the +materializer carries the values into ``files`` on every rebuild. Writing +them post-materialization instead would leave them to be erased by any +standalone ``make materialize-dcc`` / ``make materialize-files``. + +file.persistent_id 4DNF* accession → file.accession_id (case-folded) +collection.persistent_id 4DNE* accession → collection.accession_id + (case-folded) + Field Mapping (4DN API → CFDB) ------------------------------- @@ -85,6 +100,7 @@ import aiohttp +from cfdb.accessions import normalize_accession from cfdb.dcc_registry import get_dcc_config from cfdb.models import ( NUMERIC_PROTOCOL_FIELDS, @@ -103,11 +119,31 @@ # requested file rather than a truncated deep-pagination window. _FILE_METADATA_BATCH_SIZE = 100 -# 4DN accession pattern: 4DNF followed by alphanumeric characters -_ACCESSION_RE = re.compile(r"4DNF[A-Z0-9]+") - -# 4DN experiment/experiment set accession pattern: 4DNEX* or 4DNES* -_EXPERIMENT_ACCESSION_RE = re.compile(r"4DNE[A-Z][A-Z0-9]+") +# 4DN accession pattern: 4DNF followed by alphanumeric characters. +# +# Case-insensitive deliberately. 4DN publishes accessions upper-cased and +# every one of the 53,697 files currently in the corpus is, but an +# upper-case-only pattern degrades badly rather than simply missing: on a +# mixed-case value it matches the upper-case prefix and returns a +# *truncated* accession (``4DNFImcjxzkh`` -> ``4DNFI``), which is a +# plausible-looking wrong answer rather than a None the callers already +# count and log. Worse, every such value truncates to the same short +# prefix, so a handful of mixed-case rows would collide onto one accession. +# +# The extractors below therefore fold what they match, making the canonical +# accession the only value any caller can obtain. Matching leniently while +# returning the raw match would have moved the failure rather than removed +# it: the extracted value is also the key for the Search API round trip, +# and the portal answers with its own upper-case form, so a mixed-case +# match would join against nothing and that file would silently lose all +# its enrichment -- while still carrying a correct accession_id, and +# without being counted in the unparseable warning that is the operator's +# only signal. +_ACCESSION_RE = re.compile(r"4DNF[A-Z0-9]+", re.IGNORECASE) + +# 4DN experiment/experiment set accession pattern: 4DNEX* or 4DNES*. +# Case-insensitive for the same reason as above. +_EXPERIMENT_ACCESSION_RE = re.compile(r"4DNE[A-Z][A-Z0-9]+", re.IGNORECASE) def extract_accession(persistent_id: str) -> Optional[str]: @@ -117,12 +153,15 @@ def extract_accession(persistent_id: str) -> Optional[str]: Handles format: https://data.4dnucleome.org/files-processed/4DNFI1234ABC/@@download/4DNFI1234ABC.mcool or: https://data.4dnucleome.org/4DNFI1234ABC - Returns accession string (e.g., "4DNFI1234ABC") or None. + Returns the accession case-folded to its canonical form (e.g., + "4DNFI1234ABC") or None. Folded here rather than at each call site so + the one value every caller holds is the one both the stored field and + the Search API are keyed on. """ if not persistent_id: return None match = _ACCESSION_RE.search(persistent_id) - return match.group(0) if match else None + return normalize_accession(match.group(0)) if match else None def extract_experiment_accession(persistent_id: str) -> Optional[str]: @@ -131,12 +170,14 @@ def extract_experiment_accession(persistent_id: str) -> Optional[str]: Handles accessions starting with 4DNEX (experiments) or 4DNES (experiment sets). - Returns accession string (e.g., "4DNEXH4ZUIH6") or None. + Returns the accession case-folded to its canonical form (e.g., + "4DNEXH4ZUIH6") or None, for the same reason as + :func:`extract_accession`. """ if not persistent_id: return None match = _EXPERIMENT_ACCESSION_RE.search(persistent_id) - return match.group(0) if match else None + return normalize_accession(match.group(0)) if match else None def parse_extra_files(extra_files_raw: list) -> list[dict]: diff --git a/src/cfdb/services/sync.py b/src/cfdb/services/sync.py index 3fa646c..884e37a 100644 --- a/src/cfdb/services/sync.py +++ b/src/cfdb/services/sync.py @@ -16,6 +16,7 @@ from typing import Optional from cfdb import api +from cfdb.accessions import normalize_accession from cfdb.dcc_registry import ( get_all_dcc_names, get_dcc_config, @@ -23,7 +24,11 @@ normalize_dcc_name, ) from cfdb.downloader import cleanup_zip, download_file, extract_zip -from cfdb.indexes import data_index_specs, ensure_indexes +from cfdb.indexes import ( + data_index_specs, + ensure_indexes, + materialized_files_index_specs, +) from cfdb.services import locks logger = logging.getLogger(__name__) @@ -138,6 +143,7 @@ async def _sync_dccs(task: SyncTask) -> None: await _sync_c2m2_zip(task, data_path, downloads_path) logger.info(f"{dcc.upper()} synced successfully") + await _log_accession_coverage(dcc) # Ensure the data-collection query indexes now that data is loaded. # Idempotent: a no-op on subsequent syncs. Kept out of API startup @@ -154,6 +160,48 @@ async def _sync_dccs(task: SyncTask) -> None: logger.info(task.progress) +async def _log_accession_coverage(dcc: str) -> None: + """Report how much of a DCC is queryable by accession after a sync. + + ``accession_id`` fails silently in both directions: a filter against an + unpopulated corpus returns ``totalCount: 0`` with no error, which reads + exactly like "no such accession", and a DCC that issues no accession at + all looks identical. Neither the API nor the client can tell those + apart, so this log is the only place the distinction is visible -- + which also makes it the signal that a standalone re-materialization + dropped the 4DN file accessions. + + Advisory only: a coverage shortfall is not a sync failure, and this + never raises into the sync path. + """ + if api.db is None: + return + + try: + total = await api.db.files.count_documents({"submission": dcc}) + covered = await api.db.files.count_documents( + {"submission": dcc, "accession_id": {"$ne": None}} + ) + except Exception as exc: # pragma: no cover - diagnostics must not break sync + logger.warning(f"{dcc.upper()} accession coverage unavailable: {exc}") + return + + if not total: + return + + pct = 100.0 * covered / total + message = ( + f"{dcc.upper()} accession coverage: {covered}/{total} files " + f"carry accession_id ({pct:.1f}%)" + ) + if covered: + logger.info(message) + else: + logger.warning( + f"{message}; accession filters will return no matches for this DCC" + ) + + async def _sync_c2m2_zip( task: SyncTask, data_path: Path, downloads_path: Path ) -> None: @@ -201,6 +249,7 @@ async def _sync_c2m2_zip( task.current_step = "enriching_collections" task.progress = "Enriching 4DN collections from experiment API..." logger.info(task.progress) + await _stamp_4dn_file_accessions() await _enrich_4dn_collections() elif dcc == "hubmap": task.current_step = "enriching_collections" @@ -233,6 +282,116 @@ async def _sync_c2m2_zip( await _enrich_hubmap_files(dataset_metadata) +async def _set_accession_ids(collection, stamps: list, label: str) -> int: + """Stamp ``accession_id`` on every document with a parsed accession. + + Kept separate from the Search API enrichment passes because it must + not inherit their matching: those only update documents the API + returned metadata for, whereas ``accession_id`` has to land on every + document whose accession parsed, or the field is queryable for only + part of the DCC. + + ``stamps`` is a list of ``(document _id, accession)`` pairs, one entry + per document -- deliberately not the accession-keyed dict the callers + also build for their Search API fetch. That dict is last-write-wins, + so two documents resolving to one accession would collapse to a single + entry and leave the loser unstamped despite having parsed cleanly, + silently contradicting the guarantee above. Which one lost would + depend on cursor order, and the end state -- a null ``accession_id`` + -- is indistinguishable from a DCC that issues no accession at all. + + Values are folded through + :func:`~cfdb.accessions.normalize_accession` so what is stored matches + what the GraphQL layer folds a filter value to. + + A failed batch is logged and skipped rather than raised. Stamping is + now the first write of each enrichment pass, so an escaping + ``BulkWriteError`` would abort the pass before any Search API + enrichment ran and fail the whole sync -- costing more than the + accessions it failed to write. The two are independent by design, so a + partial stamp degrades the field rather than the sync. + + Returns the number of documents modified. + """ + from pymongo import UpdateOne + from pymongo.errors import BulkWriteError + + operations = [ + UpdateOne({"_id": doc_id}, {"$set": {"accession_id": normalize_accession(acc)}}) + for doc_id, acc in stamps + ] + if not operations: + logger.warning(f"{label} accession_id: no documents to stamp") + return 0 + + total_modified = 0 + failed = 0 + for i in range(0, len(operations), BATCH_SIZE): + batch = operations[i : i + BATCH_SIZE] + try: + result = await collection.bulk_write(batch, ordered=False) + except BulkWriteError as exc: + failed += len(batch) + logger.error( + f"{label} accession_id: batch of {len(batch)} failed to stamp; " + f"continuing so enrichment still runs: {exc}" + ) + continue + total_modified += result.modified_count + + if failed: + logger.error( + f"{label} accession_id: {failed} of {len(operations)} documents were " + "not stamped; those files are not queryable by accession" + ) + logger.info(f"{label} accession_id: stamped {total_modified} documents") + return total_modified + + +async def _stamp_4dn_file_accessions() -> None: + """Stamp accession_id onto the raw 4DN file documents. + + Deliberately targets the raw ``file`` collection rather than the + materialized ``files``, and therefore runs pre-materialization. The + materializer rebuilds ``files`` from ``file`` on every run -- dropping + the collection outright when invoked without a submission filter -- so + a value written post-materialization survives only until the next + ``make materialize-dcc`` or ``make materialize-files``, both of which + are supported standalone operator commands. Stamping the raw document + instead lets ``enrich_file``'s in-place mutation carry the value + forward on every rebuild, which is what already makes the collection + accession durable. + + The accession is parsed from ``persistent_id``, which the raw file rows + already carry, so this needs nothing the Search API pass provides. + """ + from cfdb.services.fourdn import extract_accession + + if api.db is None: + raise RuntimeError("Database not initialized") + + stamps: list[tuple[object, str]] = [] + unparseable = 0 + cursor = api.db.file.find( + {"submission": "4dn"}, + {"_id": 1, "persistent_id": 1}, + ) + async for doc in cursor: + acc = extract_accession(doc.get("persistent_id", "")) + if acc: + stamps.append((doc["_id"], acc)) + else: + unparseable += 1 + + if unparseable: + logger.warning( + f"4DN file accession_id: {unparseable} files have no parseable " + "accession in persistent_id; accession_id left null" + ) + + await _set_accession_ids(api.db.file, stamps, "4DN file") + + async def _enrich_4dn_api_metadata() -> None: """Enrich materialized 4DN files with metadata from the 4DN Search API.""" from pymongo import UpdateOne @@ -246,10 +405,17 @@ async def _enrich_4dn_api_metadata() -> None: if api.db is None: raise RuntimeError("Database not initialized") - # Build accession -> _id lookup from existing files (avoids $regex per + # Build accession -> _ids lookup from existing files (avoids $regex per # update) first, so enrichment metadata is fetched for exactly those - # accessions. - accession_to_id: dict[str, object] = {} + # accessions. accession_id itself is stamped pre-materialization by + # _stamp_4dn_file_accessions, so this pass no longer writes it. + # + # Every accession maps to a *list* of document ids. 4DN issues one + # accession per file, but nothing here enforces that, and a plain + # accession-keyed dict is last-write-wins: two files resolving to one + # accession would silently leave the loser un-enriched, chosen by + # cursor order, with no signal that it happened. + accession_to_ids: dict[str, list] = {} cursor = api.db.files.find( {"submission": "4dn"}, {"_id": 1, "persistent_id": 1}, @@ -257,14 +423,20 @@ async def _enrich_4dn_api_metadata() -> None: async for doc in cursor: acc = extract_accession(doc.get("persistent_id", "")) if acc: - accession_to_id[acc] = doc["_id"] + accession_to_ids.setdefault(acc, []).append(doc["_id"]) - logger.info(f"4DN enrichment: {len(accession_to_id)} files in DB mapped by accession") + mapped = sum(len(ids) for ids in accession_to_ids.values()) + logger.info(f"4DN enrichment: {mapped} files in DB mapped by accession") + if mapped > len(accession_to_ids): + logger.warning( + f"4DN enrichment: {mapped - len(accession_to_ids)} files share an " + "accession with another file; all of them will be enriched alike" + ) # Fetch API data for exactly the accessions we hold. Batching the Search # API query by accession bypasses its 10k result-window cap, which a # blind deep-pagination scan would silently hit and truncate. - file_metadata = await fetch_file_metadata_bulk(accession_to_id.keys()) + file_metadata = await fetch_file_metadata_bulk(accession_to_ids.keys()) biosource_tiers = await fetch_biosource_tiers() logger.info( @@ -275,8 +447,8 @@ async def _enrich_4dn_api_metadata() -> None: # Build bulk update operations matched by _id operations = [] for accession, meta in file_metadata.items(): - doc_id = accession_to_id.get(accession) - if not doc_id: + doc_ids = accession_to_ids.get(accession) + if not doc_ids: continue update: dict = {} @@ -319,7 +491,9 @@ async def _enrich_4dn_api_metadata() -> None: if not update: continue - operations.append(UpdateOne({"_id": doc_id}, {"$set": update})) + operations.extend( + UpdateOne({"_id": doc_id}, {"$set": update}) for doc_id in doc_ids + ) if not operations: logger.warning("4DN enrichment: no updates to apply") @@ -347,23 +521,49 @@ async def _enrich_4dn_collections() -> None: if api.db is None: raise RuntimeError("Database not initialized") - # Fetch experiment metadata from 4DN API - experiment_metadata = await fetch_experiment_metadata_bulk() - - logger.info(f"4DN collection enrichment: {len(experiment_metadata)} experiment entries") - - # Build bulk updates: match collection docs by experiment accession in persistent_id - operations = [] + # Scan and stamp before any network call. An empty API result already + # left every collection queryable by accession, but a *raised* one did + # not: fetch_experiment_metadata_bulk catches only aiohttp.ClientError, + # so a TimeoutError from its 60s budget propagated and nothing was + # stamped at all. Collecting first makes that guarantee structural + # rather than a property of where the await happens to sit, and matches + # the ordering the file pass already uses. cursor = api.db.collection.find( {"submission": "4dn"}, {"_id": 1, "persistent_id": 1}, ) - matched = 0 + unparseable = 0 + stamps: list[tuple[object, str]] = [] async for doc in cursor: accession = extract_experiment_accession(doc.get("persistent_id", "")) if not accession: + unparseable += 1 continue + # One entry per document rather than an accession-keyed dict, so two + # collections sharing an accession both get stamped instead of one + # silently winning. + stamps.append((doc["_id"], accession)) + + if unparseable: + logger.warning( + f"4DN collection enrichment: {unparseable} collections have no " + "parseable accession in persistent_id; accession_id left null" + ) + + # This runs pre-materialization, so the materializer embeds the value + # into files.collections[]. + await _set_accession_ids(api.db.collection, stamps, "4DN collection") + + # Fetch experiment metadata from 4DN API + experiment_metadata = await fetch_experiment_metadata_bulk() + + logger.info(f"4DN collection enrichment: {len(experiment_metadata)} experiment entries") + + # Build bulk updates from the accessions already collected above + operations = [] + matched = 0 + for doc_id, accession in stamps: meta = experiment_metadata.get(accession) if not meta: continue @@ -383,7 +583,7 @@ async def _enrich_4dn_collections() -> None: update["extra.fourdn"] = remaining if update: - operations.append(UpdateOne({"_id": doc["_id"]}, {"$set": update})) + operations.append(UpdateOne({"_id": doc_id}, {"$set": update})) if not operations: logger.warning("4DN collection enrichment: no updates to apply") @@ -871,6 +1071,12 @@ async def _sync_encode(task: SyncTask) -> None: await api.db.files.insert_many(batch) logger.info(f"Inserted final batch, total: {count} ENCODE files") + # ENCODE writes straight into the materialized collection and never runs + # the materializer, which is the only other creator of files indexes. On + # a database where ENCODE is the only DCC synced, that leaves files with + # no indexes at all and makes every accession lookup a full scan. + await ensure_indexes(api.db, materialized_files_index_specs()) + task.progress = f"ENCODE sync complete: {count} files" logger.info(task.progress) logger.info( diff --git a/tests/conftest.py b/tests/conftest.py index 9d3d790..a711076 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -216,8 +216,30 @@ def __init__(self) -> None: # unique index behavior in tests. self._indexes: list[tuple[dict, dict]] = [] - def create_index(self, spec: dict, **opts) -> None: - self._indexes.append((spec, opts)) + def register_index(self, spec, **opts) -> str: + """Seed an index synchronously, for test arrangement. + + ``create_index`` is awaitable because Motor's is, but a test that + only needs the double pre-seeded should not have to be async to say + so. Both funnel here. + + ``spec`` mirrors pymongo's own flexibility: a dict of key/direction + pairs, or the list of ``(key, direction)`` tuples ``ensure_indexes`` + builds from an :class:`~cfdb.indexes.IndexSpec`. Both are stored as + a dict so ``insert_one``'s partial-unique check sees one shape. + """ + keys = spec if isinstance(spec, dict) else dict(spec) + self._indexes.append((keys, opts)) + return opts.get("name") or "_".join(f"{k}_{v}" for k, v in keys.items()) + + async def create_index(self, spec, **opts) -> str: + """Record an index. Async because Motor's ``create_index`` is awaited. + + Production awaits this (see ``cfdb.indexes.ensure_indexes``), so a + synchronous stub silently returned ``None`` into an ``await`` and + any code path that ensured indexes could not be tested at all. + """ + return self.register_index(spec, **opts) def with_options(self, **_kwargs): """No-op shim mirroring Motor's ``Collection.with_options``. @@ -326,6 +348,13 @@ async def find_one_and_update( return None async def bulk_write(self, operations: list, ordered: bool = True) -> _BulkWriteResult: + # Routed through ``_apply_update`` rather than assigning ``$set`` + # keys directly: a dotted key (``extra.fourdn``) must nest, not land + # as a literal flat key, or a test asserting an enrichment payload + # shape passes against a document real Mongo would have written + # differently. ``_apply_update`` also reports whether the row + # actually changed, so ``modified_count`` counts changed rows like + # Mongo does rather than merely matched ones. count = 0 for op in operations: # Support UpdateOne @@ -334,9 +363,8 @@ async def bulk_write(self, operations: list, ordered: bool = True) -> _BulkWrite update = op._doc for d in self.docs: if _match(d, filt): - for k, v in update.get("$set", {}).items(): - d[k] = v - count += 1 + if _apply_update(d, update, is_insert=False): + count += 1 break return _BulkWriteResult(count) diff --git a/tests/integration/test_executor_boundary.py b/tests/integration/test_executor_boundary.py index 24acad5..d50c245 100644 --- a/tests/integration/test_executor_boundary.py +++ b/tests/integration/test_executor_boundary.py @@ -32,7 +32,7 @@ def _install_jobs_index(mock_db) -> None: - mock_db.jobs.create_index( + mock_db.jobs.register_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"active": True}, diff --git a/tests/test_accessions.py b/tests/test_accessions.py new file mode 100644 index 0000000..67ac334 --- /dev/null +++ b/tests/test_accessions.py @@ -0,0 +1,241 @@ +import sys + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from cfdb.accessions import normalize_accession + +#: Every code point ``str.strip()`` actually removes. Enumerated rather +#: than written out as ``" \t\n\r"`` because ``strip()`` removes roughly +#: 25 characters, so a hand-picked four would leave the ones a caller is +#: most likely to paste in (NBSP in particular) unexercised. +_UNICODE_WHITESPACE = "".join( + c for c in map(chr, range(sys.maxunicode + 1)) if c.isspace() +) + +#: The alphabet the DCCs actually issue accessions from. +_ACCESSION_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + + +def test_normalize_accession_should_upper_case_a_lower_case_accession(): + """Test that a lower-case accession folds to upper case. + + Given: + A 4DN file accession typed entirely in lower case. + When: + normalize_accession is called. + Then: + It should return the upper-case form, which is how the accession + is stored and therefore what an equality match needs. + """ + # Act + result = normalize_accession("4dnfimcjxzkh") + + # Assert + assert result == "4DNFIMCJXZKH" + + +def test_normalize_accession_should_pass_an_upper_case_accession_through(): + """Test that an already-canonical accession is unchanged. + + Given: + An ENCODE accession already in the published upper-case form. + When: + normalize_accession is called. + Then: + It should return the same string. + """ + # Act + result = normalize_accession("ENCFF525XQX") + + # Assert + assert result == "ENCFF525XQX" + + +def test_normalize_accession_should_strip_surrounding_whitespace(): + """Test that padding a caller pasted in is removed. + + Given: + An accession surrounded by leading and trailing whitespace. + When: + normalize_accession is called. + Then: + It should return the accession with the whitespace removed. + """ + # Act + result = normalize_accession(" ENCSR918ZSJ\n") + + # Assert + assert result == "ENCSR918ZSJ" + + +def test_normalize_accession_should_keep_interior_whitespace(): + """Test that folding repairs padding but not a broken accession. + + Given: + An accession with whitespace in the middle as well as around it. + When: + normalize_accession is called. + Then: + It should remove only the surrounding whitespace, so a mangled + accession keeps a distinct key and fails to match rather than + being silently repaired into a different file's accession. + """ + # Act + result = normalize_accession(" ENC FF525XQX ") + + # Assert + assert result == "ENC FF525XQX" + + +def test_normalize_accession_should_return_none_when_value_is_none(): + """Test that a missing accession stays missing. + + Given: + None, as produced when a DCC issues no accession. + When: + normalize_accession is called. + Then: + It should return None. + """ + # Act + result = normalize_accession(None) + + # Assert + assert result is None + + +def test_normalize_accession_should_raise_when_value_is_not_a_string(): + """Test that a non-string accession fails loudly rather than coercing. + + No current caller can reach this: encode reads every accession cell + through ``_nonempty`` (str or None), and the sync passes regex match + results. It is a defensive pin on the input contract, not a + reachable path -- if a future caller does pass a non-string, the + alternative to raising is storing a value no filter can ever match. + + Given: + A non-string, non-None value. + When: + normalize_accession is called. + Then: + It should raise AttributeError rather than coercing. + """ + # Act & assert + with pytest.raises(AttributeError): + normalize_accession(12345) + + +@given(blank=st.text(alphabet=_UNICODE_WHITESPACE, max_size=8)) +@settings(max_examples=100) + +def test_normalize_accession_should_return_none_when_value_is_blank(blank): + """Test that a blank accession collapses to None rather than "". + + Given: + Any string built from characters str.strip() removes, including + the empty string. + When: + normalize_accession is called. + Then: + It should return None, keeping an absent accession out of the + index instead of storing an empty string. + """ + # Act + result = normalize_accession(blank) + + # Assert + assert result is None + + +@given(value=st.text()) +@settings(max_examples=200) + +def test_normalize_accession_should_be_idempotent(value): + """Test that folding an already-folded value changes nothing. + + Given: + Any text at all. + When: + normalize_accession is applied twice. + Then: + The second application should return the first's result, so a + value re-stamped by a later sync cannot drift. + """ + # Act + once = normalize_accession(value) + twice = normalize_accession(once) + + # Assert + assert twice == once + + +@given(value=st.text()) +@settings(max_examples=200) + +def test_normalize_accession_should_not_depend_on_strip_upper_order(value): + """Test that stripping and upper-casing commute for any input. + + Note this pins operation ordering, NOT case-insensitivity: the + function's last step is already upper(), so comparing against an + upper-cased input cannot fail by construction. The genuine + case-insensitivity contract is pinned over the accession alphabet + below, because outside it the fold is lossy -- "ẞ" normalizes to + itself while its lower-case "ß" normalizes to "SS". + + Given: + Any text, and the same text upper-cased. + When: + Both are normalized. + Then: + They should agree, so no input exists for which upper-casing + before stripping would strip differently. + """ + # Act + from_value = normalize_accession(value) + from_upper = normalize_accession(value.upper()) + + # Assert + assert from_value == from_upper + + +@given( + accession=st.text(alphabet=_ACCESSION_CHARS, min_size=1, max_size=16), + flips=st.lists(st.booleans(), min_size=16, max_size=16), + pad_left=st.text(alphabet=" \t", max_size=3), + pad_right=st.text(alphabet=" \t", max_size=3), +) +@settings(max_examples=200) + +def test_normalize_accession_should_fold_every_casing_to_one_value( + accession, flips, pad_left, pad_right +): + """Test the case-insensitivity contract the whole feature rests on. + + Given: + Any accession over the alphabet the DCCs issue from, plus its + lower-cased, upper-cased and arbitrarily re-cased forms, each + with arbitrary surrounding padding. + When: + All four are normalized. + Then: + They should produce one value, so which casing a caller types + cannot change which documents an accession filter matches. + """ + # Arrange + recased = "".join( + char.lower() if flip else char for char, flip in zip(accession, flips) + ) + variants = [ + accession, + accession.lower(), + accession.upper(), + f"{pad_left}{recased}{pad_right}", + ] + + # Act + folded = {normalize_accession(v) for v in variants} + + # Assert + assert folded == {accession.upper()} diff --git a/tests/test_data.py b/tests/test_data.py index f63c56c..dd1277b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -113,7 +113,7 @@ async def test_stream_file_should_dispatch_workflow_when_processor_applies( """ # Arrange mocker.patch.object(locks, "wait_for_cutover", return_value=None) - mock_db.jobs.create_index( + await mock_db.jobs.create_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"active": True}, diff --git a/tests/test_encode.py b/tests/test_encode.py index 22a5925..516e8fe 100644 --- a/tests/test_encode.py +++ b/tests/test_encode.py @@ -6,6 +6,7 @@ from hypothesis import given, settings from hypothesis import strategies as st +from cfdb.accessions import normalize_accession from cfdb.services.encode import ( COMPRESSION_SUFFIX_TO_EDAM, UNCOMPRESSED, @@ -539,3 +540,218 @@ def test_transform_to_c2m2_should_not_raise_for_any_download_url(url): # Assert assert doc.get("compression_format", UNCOMPRESSED) in DERIVED_VALUES + + +def test_transform_to_c2m2_should_set_accession_id_on_the_file(): + """Test that the ENCODE file accession lands on accession_id. + + Given: + An ENCODE row carrying a File accession. + When: + transform_to_c2m2 is called. + Then: + It should set accession_id to that accession, giving ENCODE the same + cross-DCC query field 4DN gets from its persistent_id. + """ + # Arrange + row = _encode_row() + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["accession_id"] == "ENCFF123ABC" + + +def test_transform_to_c2m2_should_set_accession_id_on_the_experiment_collection(): + """Test that the experiment accession lands on the collection. + + Given: + An ENCODE row naming an Experiment accession, and a Biosample term + name -- which the collection block is gated on, so the accession + alone would build no collection at all. + When: + transform_to_c2m2 is called. + Then: + It should set accession_id on the built collection, so a collection + accession filter resolves for ENCODE as it does for 4DN. + """ + # Arrange + row = _encode_row( + **{"Experiment accession": "ENCSR918ZSJ", "Biosample term name": "K562"} + ) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["collections"][0]["accession_id"] == "ENCSR918ZSJ" + + +def test_transform_to_c2m2_should_not_set_accession_id_on_a_biosample_collection(): + """Test that the synthesized fallback collection gets no accession. + + Given: + An ENCODE row with no Experiment accession, so the collection is keyed + on the biosample term instead. + When: + transform_to_c2m2 is called. + Then: + It should leave accession_id unset on that collection, since it names + no ENCODE experiment and a fabricated accession would be wrong. + """ + # Arrange + row = _encode_row(**{"Biosample term name": "K562"}) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["collections"] + assert "accession_id" not in doc["collections"][0] + + +def test_transform_to_c2m2_should_not_reintroduce_padding_on_the_accession_id(): + """Test that the builder does not re-pad an already-stripped accession. + + The stripping itself happens upstream in ``_nonempty``, which every + accession cell is read through, so this does not pin normalize_accession's + whitespace handling -- deleting .strip() from it leaves this test green. + That contract is pinned in tests/test_accessions.py. What this covers is + the builder: that nothing between the cell and the stored field puts the + padding back. + + Given: + A row whose File accession carries surrounding whitespace, as a + hand-edited TSV cell can. + When: + transform_to_c2m2 is called. + Then: + It should store the stripped accession. + """ + # Arrange + row = _encode_row(**{"File accession": " ENCFF123ABC "}) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["accession_id"] == "ENCFF123ABC" + + +def test_transform_to_c2m2_should_fold_accession_id_without_rewriting_local_id(): + """Test that the two fields legitimately disagree in case. + + Given: + A row whose File accession is published in lower case. + When: + transform_to_c2m2 is called. + Then: + It should fold accession_id for matching while leaving local_id as + published, since local_id is the DCC's own identifier and rewriting + it would change the document's key. + """ + # Arrange + row = _encode_row(**{"File accession": "encff123abc"}) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["accession_id"] == "ENCFF123ABC" + assert doc["local_id"] == "encff123abc" + + +def test_transform_to_c2m2_should_fold_the_experiment_collection_accession(): + """Test that the collection accession is folded like the file's. + + The upper-casing is the load-bearing half: the padding was already + removed upstream by ``_nonempty``, so only the case change is evidence + that the collection branch routes through the shared fold rather than + storing the cell as published. + + Given: + A row whose Experiment accession is lower-cased and padded, with a + biosample term present so the collection is built at all. + When: + transform_to_c2m2 is called. + Then: + It should store the stripped, upper-cased experiment accession. + """ + # Arrange + row = _encode_row( + **{ + "Experiment accession": " encsr918zsj ", + "Biosample term name": "K562", + } + ) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["collections"][0]["accession_id"] == "ENCSR918ZSJ" + + +def test_transform_to_c2m2_should_build_no_collection_without_a_biosample_term(): + """Test that an experiment accession alone yields no collection. + + The whole collection block is gated on the biosample term name, so a + row shaped like an ENCODE annotation or reference contributes no + collection and its experiment accession is not queryable anywhere. + Pre-existing behavior, pinned here because the accession field turns it + from a cosmetic gap into a data-completeness question. + + Given: + A row carrying an Experiment accession but no Biosample term name. + When: + transform_to_c2m2 is called. + Then: + It should produce an empty collections list. + """ + # Arrange + row = _encode_row(**{"Experiment accession": "ENCSR918ZSJ"}) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["collections"] == [] + + +@given( + accession=st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + min_size=1, + max_size=16, + ), + pad=st.text(alphabet=" \t", max_size=3), +) +@settings(max_examples=100) +def test_transform_to_c2m2_should_store_the_accession_case_folded(accession, pad): + """Test that the builder routes the accession through the shared fold. + + Both sides of the assertion call normalize_accession on the same input, + so this pins that transform_to_c2m2 does not *bypass* the shared fold + -- not which direction that fold goes. It cannot detect a change of + fold direction, because the expectation moves with it: inverting + normalize_accession to .lower() fails 37 tests elsewhere and leaves + this one green. The direction is pinned against literals by + ...should_fold_accession_id_without_rewriting_local_id below and by the + query-side round trip in tests/test_inputs.py. + + Given: + Any File accession in arbitrary casing with arbitrary padding. + When: + transform_to_c2m2 is called. + Then: + accession_id should equal the folded local_id. + """ + # Arrange + row = _encode_row(**{"File accession": f"{pad}{accession}{pad}"}) + + # Act + doc = transform_to_c2m2(row) + + # Assert + assert doc["accession_id"] == normalize_accession(doc["local_id"]) diff --git a/tests/test_fake_collection.py b/tests/test_fake_collection.py index ca04938..ba82eae 100644 --- a/tests/test_fake_collection.py +++ b/tests/test_fake_collection.py @@ -1,9 +1,10 @@ """Tests for the in-memory ``FakeCollection`` test double. These tests pin behavior of the FakeCollection extensions that the -workflow tests rely on — partial-filter awareness on unique indexes, -``$setOnInsert`` upserts, dotted-path resolution in queries, the ``$lt`` -operator, and ``$addToSet`` deduplication. Treating the fake as part of +workflow and sync tests rely on — partial-filter awareness on unique +indexes, ``$setOnInsert`` upserts, dotted-path resolution in queries, the +``$lt`` operator, ``$addToSet`` deduplication, and ``bulk_write``'s +dotted-key nesting and changed-row counting. Treating the fake as part of the test contract surface keeps regressions in the helper from masking real production behavior. """ @@ -13,12 +14,72 @@ from datetime import datetime, timedelta, timezone import pytest +from pymongo import UpdateOne from pymongo.errors import DuplicateKeyError from tests.conftest import FakeCollection class TestFakeCollectionContract: + @pytest.mark.asyncio + async def test_bulk_write_should_nest_a_dotted_set_key(self): + """Test that a dotted $set nests rather than landing flat. + + Real Mongo treats ``{"$set": {"extra.fourdn": ...}}`` as a path, so + a double that assigned the key literally would let a test assert an + enrichment payload shape against a document Mongo would have + written differently -- the assertion passes, production differs. + + Given: + A document and an UpdateOne carrying a dotted $set key. + When: + bulk_write applies it. + Then: + The value should be nested under the path's segments, with no + flat key left behind. + """ + # Arrange + coll = FakeCollection() + coll.docs = [{"_id": "d1"}] + + # Act + await coll.bulk_write( + [UpdateOne({"_id": "d1"}, {"$set": {"extra.fourdn": {"status": "released"}}})] + ) + + # Assert + assert coll.docs[0]["extra"] == {"fourdn": {"status": "released"}} + assert "extra.fourdn" not in coll.docs[0] + + @pytest.mark.asyncio + async def test_bulk_write_should_count_changed_rows_not_matched_rows(self): + """Test that modified_count means modified, as Mongo reports it. + + A double counting matched rows makes every ``modified_count`` + assertion in the suite fiction: a re-applied identical update would + report work it did not do, which is exactly the signal a sync uses + to report how much it stamped. + + Given: + A document already carrying the value an update would set. + When: + bulk_write applies that update. + Then: + It should match the document but report zero modifications. + """ + # Arrange + coll = FakeCollection() + coll.docs = [{"_id": "d1", "accession_id": "4DNFIMCJXZKH"}] + + # Act + result = await coll.bulk_write( + [UpdateOne({"_id": "d1"}, {"$set": {"accession_id": "4DNFIMCJXZKH"}})] + ) + + # Assert + assert result.modified_count == 0 + assert coll.docs[0]["accession_id"] == "4DNFIMCJXZKH" + @pytest.mark.asyncio async def test_insert_one_should_raise_duplicate_when_partial_filter_matches(self): """Test that the partial unique index blocks two active rows. @@ -35,7 +96,7 @@ async def test_insert_one_should_raise_duplicate_when_partial_filter_matches(sel """ # Arrange coll = FakeCollection() - coll.create_index( + await coll.create_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"status": {"$in": ["pending", "running"]}}, @@ -66,7 +127,7 @@ async def test_insert_one_should_allow_active_when_completed_exists(self): """ # Arrange coll = FakeCollection() - coll.create_index( + await coll.create_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"status": {"$in": ["pending", "running"]}}, diff --git a/tests/test_fourdn.py b/tests/test_fourdn.py index 7c4bf85..960b15a 100644 --- a/tests/test_fourdn.py +++ b/tests/test_fourdn.py @@ -11,9 +11,260 @@ from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from cfdb.accessions import normalize_accession from cfdb.models import NUMERIC_PROTOCOL_FIELDS, EnrichedFourdnCollection from cfdb.services import fourdn -from cfdb.services.fourdn import parse_experiment_metadata, parse_extra_files +from cfdb.services.fourdn import ( + extract_accession, + extract_experiment_accession, + parse_experiment_metadata, + parse_extra_files, +) + +_PORTAL = "https://data.4dnucleome.org" + + +class TestExtractAccession: + def test_extract_accession_should_return_the_accession_from_a_bare_url(self): + """Test the canonical persistent_id shape. + + Given: + A 4DN persistent_id that is the portal URL plus the accession. + When: + extract_accession is called. + Then: + It should return the accession. + """ + # Act, assert + assert extract_accession(f"{_PORTAL}/4DNFIMCJXZKH") == "4DNFIMCJXZKH" + + def test_extract_accession_should_return_the_accession_from_a_download_url(self): + """Test the longer download form, where the accession repeats. + + Given: + A download URL carrying the accession twice, once in the path + and once in the filename. + When: + extract_accession is called. + Then: + It should return the first occurrence, both being identical. + """ + # Arrange + url = f"{_PORTAL}/files-processed/4DNFI1234ABC/@@download/4DNFI1234ABC.mcool" + + # Act, assert + assert extract_accession(url) == "4DNFI1234ABC" + + @pytest.mark.parametrize( + "typed", + ["4DNFImcjxzkh", "4dnfimcjxzkh", "4DNFIMCJXZKH"], + ids=["mixed", "lower", "upper"], + ) + def test_extract_accession_should_fold_any_casing_to_one_accession(self, typed): + """Test that casing cannot truncate or lose the accession. + + An upper-case-only pattern degrades badly rather than simply + missing here: on a mixed-case value it matched only the upper-case + prefix and returned a truncated accession, which is a + plausible-looking wrong answer, and every such value truncated to + the same short prefix. + + Asserted on the extractor's own return value rather than on + ``normalize_accession(extract_accession(...))``: folding the result + before asserting would pass whether or not the extractor folds, and + the extractor's output is what keys the Search API round trip. + + Given: + The same accession published in mixed, lower and upper case. + When: + extract_accession is called. + Then: + All three should yield the one canonical accession. + """ + # Act + result = extract_accession(f"{_PORTAL}/{typed}") + + # Assert + assert result == "4DNFIMCJXZKH" + + def test_extract_accession_should_stop_at_a_file_extension(self): + """Test the token boundary against a trailing extension. + + Given: + A persistent_id whose accession is followed by a dot and an + upper-case extension. + When: + extract_accession is called. + Then: + It should stop at the separator rather than absorbing the + extension. + """ + # Act, assert + assert extract_accession(f"{_PORTAL}/4DNFIMCJXZKH.MCOOL") == "4DNFIMCJXZKH" + + def test_extract_accession_should_find_an_accession_in_a_query_string(self): + """Test that matching is not anchored to the path. + + Given: + A URL carrying the accession in a query parameter. + When: + extract_accession is called. + Then: + It should still return it, pinning the unanchored match as + intended rather than incidental. + """ + # Act, assert + assert extract_accession(f"{_PORTAL}/s/?accession=4DNFIABC") == "4DNFIABC" + + def test_extract_accession_should_return_none_for_an_experiment_accession(self): + """Test that the two extractors do not poach each other's inputs. + + Given: + A persistent_id carrying an experiment accession. + When: + extract_accession is called. + Then: + It should return None, so a collection URL cannot be stamped + with a file accession. + """ + # Act, assert + assert extract_accession(f"{_PORTAL}/4DNEXNHE6X77") is None + + @pytest.mark.parametrize("value", ["", None, f"{_PORTAL}/nothing-here"]) + def test_extract_accession_should_return_none_when_there_is_nothing_to_find( + self, value + ): + """Test the guard and no-match branches. + + Given: + An empty string, None, and a URL with no accession token. + When: + extract_accession is called. + Then: + It should return None without raising, so a malformed row is + counted and logged rather than aborting the sync. + """ + # Act, assert + assert extract_accession(value) is None + + @given( + accession=st.text( + alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", min_size=1, max_size=12 + ) + ) + @settings(max_examples=100) + def test_extract_accession_should_round_trip_through_normalization( + self, accession + ): + """Test that what the extractor emits is already in stored form. + + Given: + Any accession over the DCC alphabet, embedded in a portal URL. + When: + It is extracted. + Then: + It should already equal its own folded form, so re-stamping on + a later sync cannot change what is stored and the Search API + key and the stored value cannot diverge. + """ + # Act + extracted = extract_accession(f"{_PORTAL}/4DNF{accession}") + + # Assert + assert extracted == f"4DNF{accession}" + assert normalize_accession(extracted) == extracted + + +class TestExtractExperimentAccession: + @pytest.mark.parametrize( + "accession", + ["4DNEXNHE6X77", "4DNESQWI9K2F"], + ids=["experiment", "experiment-set"], + ) + def test_extract_experiment_accession_should_return_both_accession_kinds( + self, accession + ): + """Test that experiments and experiment sets are both matched. + + Given: + A persistent_id for an experiment and one for an experiment set. + When: + extract_experiment_accession is called. + Then: + It should return each accession. + """ + # Act, assert + assert extract_experiment_accession(f"{_PORTAL}/{accession}") == accession + + def test_extract_experiment_accession_should_fold_any_casing(self): + """Test that a lower-case experiment accession is not lost. + + Asserted on the extractor's own return value: folding it here + before asserting would pass whether or not the extractor folds. + + Given: + An experiment accession published in lower case. + When: + extract_experiment_accession is called. + Then: + It should return the canonical accession. + """ + # Act + result = extract_experiment_accession(f"{_PORTAL}/4dnexnhe6x77") + + # Assert + assert result == "4DNEXNHE6X77" + + def test_extract_experiment_accession_should_stop_at_a_suffix(self): + """Test the token boundary against a trailing suffix. + + Given: + An experiment accession followed by a hyphenated suffix. + When: + extract_experiment_accession is called. + Then: + It should stop at the hyphen. + """ + # Act, assert + assert ( + extract_experiment_accession(f"{_PORTAL}/4DNEXNHE6X77-rep2") + == "4DNEXNHE6X77" + ) + + def test_extract_experiment_accession_should_return_none_for_a_file_accession( + self, + ): + """Test the reverse cross-contamination guard. + + Given: + A persistent_id carrying a file accession. + When: + extract_experiment_accession is called. + Then: + It should return None. + """ + # Act, assert + assert extract_experiment_accession(f"{_PORTAL}/4DNFIMCJXZKH") is None + + @pytest.mark.parametrize("value", ["", None, f"{_PORTAL}/4DNE"]) + def test_extract_experiment_accession_should_return_none_when_nothing_matches( + self, value + ): + """Test the guard, no-match, and minimum-length branches. + + The bare prefix case pins an undocumented constraint: the pattern + requires at least two characters after 4DNE, so a truncated + accession yields None rather than a short false positive. + + Given: + An empty string, None, and a bare 4DNE prefix. + When: + extract_experiment_accession is called. + Then: + It should return None without raising. + """ + # Act, assert + assert extract_experiment_accession(value) is None def test_parse_extra_files_should_store_token_when_file_format_is_cv_object(): diff --git a/tests/test_index.py b/tests/test_index.py index 3dc00b0..63adf37 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1045,7 +1045,7 @@ async def test_stream_index_file_should_return_202_when_processor_applies_and_ca """ # Arrange mocker.patch.object(locks, "wait_for_cutover", return_value=None) - mock_db.jobs.create_index( + mock_db.jobs.register_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"active": True}, diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 46e04b3..a4cf19a 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -13,6 +13,7 @@ all_index_specs, data_index_specs, ensure_indexes, + materialized_files_index_specs, operational_index_specs, ) @@ -602,3 +603,94 @@ async def test_ensure_indexes_should_reraise_unexpected_failure(mocker): with pytest.raises(OperationFailure): await ensure_indexes(db, [spec]) collection.drop_index.assert_not_awaited() + + +def test_data_index_specs_should_not_target_the_materialized_files_collection(): + """Test the ownership split between the two index sources. + + This module owns the raw C2M2 collections; the Rust materializer owns + the denormalized ``files`` collection it builds, and indexes it in + ``index_keys``. The module docstring states that split in prose only, + so this pins it: adding a ``files`` spec here would create a second + writer for those indexes, silently competing with the materializer. + + Given: + The data index specs. + When: + The set of collections they target is collected. + Then: + It should include the raw ``file`` collection and not ``files``. + """ + # Act + targets = {spec.collection for spec in data_index_specs()} + + # Assert + assert "file" in targets + assert "files" not in targets + + +def test_materialized_files_index_specs_should_not_overlap_the_data_specs(): + """Test that the narrow files exception does not become a second writer. + + ``files`` has exactly one owner for its full index set -- the + materializer. The accession specs exist only because ``_sync_encode`` + writes ``files`` directly and never runs it, so an ENCODE-only database + would otherwise have no index at all. Keeping the two sources disjoint + is what stops that exception from growing into a duplicate of + ``index_keys``. + + Given: + Both Python-side index sources. + When: + Their (collection, name) pairs are compared. + Then: + They should share none, and the files specs should target only + ``files``. + """ + # Act + data = {(s.collection, s.name) for s in data_index_specs()} + files = {(s.collection, s.name) for s in materialized_files_index_specs()} + + # Assert + assert not data & files + assert {s.collection for s in materialized_files_index_specs()} == {"files"} + + +def test_materialized_files_index_specs_should_cover_both_accession_paths(): + """Test that both queryable accession paths are indexed. + + These are exactly the two predicates ``to_query`` can emit for an + accession filter; an accession lookup that missed either would scan the + whole collection on a public endpoint. + + Given: + The materialized files index specs. + When: + Their key tuples are collected. + Then: + They should cover the file-level and nested collection accessions. + """ + # Act + keys = {spec.keys for spec in materialized_files_index_specs()} + + # Assert + assert keys == {(("accession_id", 1),), (("collections.accession_id", 1),)} + + +def test_all_index_specs_should_not_repeat_a_collection_and_name(): + """Test that no index is declared twice. + + Given: + The full operational-plus-data spec list. + When: + Its (collection, name) pairs are collected. + Then: + None should repeat, so appending a field to two loops -- or to the + same loop twice -- fails here rather than issuing a redundant + createIndex against a live database. + """ + # Act + pairs = [(spec.collection, spec.name) for spec in all_index_specs()] + + # Assert + assert len(pairs) == len(set(pairs)) diff --git a/tests/test_inputs.py b/tests/test_inputs.py new file mode 100644 index 0000000..ffccfca --- /dev/null +++ b/tests/test_inputs.py @@ -0,0 +1,564 @@ +from hypothesis import given, settings +from hypothesis import strategies as st + +from cfdb.accessions import normalize_accession +from cfdb.api.gql.inputs import CollectionInput, FileMetadataInput, to_dict, to_query + +#: Alphabet the DCCs actually issue accessions from. +_ACCESSION_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + + +def test_to_dict_should_emit_every_declared_field(): + """Test that conversion yields the full field set, not just the set ones. + + Given: + A FileMetadataInput with every field left at its None default. + When: + to_dict converts it. + Then: + It should emit a key for every declared field, since to_query is + written to receive the whole set and drop the None values itself. + """ + # Arrange + declared = { + field.name + for field in FileMetadataInput.__strawberry_definition__.fields + } + + # Act + result = to_dict(FileMetadataInput()) + + # Assert + assert set(result) == declared + assert all(value is None for value in result.values()) + + +def test_to_dict_should_recurse_into_nested_input_lists(): + """Test that nested Strawberry inputs are converted, not passed through. + + Given: + A FileMetadataInput whose collections field holds a CollectionInput. + When: + to_dict converts it. + Then: + It should leave no Strawberry object in the tree, since to_query + can only walk dicts and lists. + """ + # Act + result = to_dict( + FileMetadataInput(collections=[CollectionInput(accession_id=["X"])]) + ) + + # Assert + assert isinstance(result["collections"], list) + assert result["collections"][0]["accession_id"] == ["X"] + + +def test_to_dict_should_pass_non_input_values_through_unchanged(): + """Test the passthrough branch, including that plain dicts are not walked. + + Given: + A scalar, None, and a plain dict carrying no Strawberry definition. + When: + to_dict converts each. + Then: + It should return each unchanged -- notably not recursing into the + plain dict, so to_dict is only safe on Strawberry inputs and lists + of them. + """ + # Arrange + plain = {"accession_id": ["x"]} + + # Act & assert + assert to_dict("ENCFF525XQX") == "ENCFF525XQX" + assert to_dict(None) is None + assert to_dict(plain) is plain + + +def _clauses(query: dict) -> set: + """Return a query's $and clauses as an order-insensitive set.""" + return frozenset(tuple(sorted(clause.items())) for clause in query["$and"]) + + +def test_to_dict_and_to_query_should_agree_with_a_hand_written_filter(): + """Test that the Strawberry path and a dict literal produce one query. + + Compared as a set of clauses rather than a list, because to_dict emits + fields in declaration order while a dict literal preserves the order + written -- so the two agree on the conjunction but not on its + sequence, and $and is order-insensitive in MongoDB. + + Given: + The same filter expressed as a FileMetadataInput and as a dict of + only its set fields. + When: + Each is converted to a MongoDB query. + Then: + They should produce the same set of clauses, pinning the + assumption every dict-literal test in this module rests on. + """ + # Arrange + payload = FileMetadataInput(filename=["a.bed"], accession_id=["encff1"]) + + # Act + from_input = to_query(to_dict(payload)) + from_dict = to_query({"filename": ["a.bed"], "accession_id": ["encff1"]}) + + # Assert + assert _clauses(from_input) == _clauses(from_dict) + + +def test_to_query_should_fold_accession_id_to_upper_case(): + """Test that a lower-case accession filter matches the stored form. + + Given: + A filter naming accession_id in lower case. + When: + to_query builds the MongoDB predicate. + Then: + It should emit the upper-case accession, since the stored value + is folded and the predicate is a bare equality match. + """ + # Act + query = to_query({"accession_id": ["4dnfimcjxzkh"]}) + + # Assert + assert query == {"accession_id": "4DNFIMCJXZKH"} + + +def test_to_query_should_fold_accession_id_nested_under_collections(): + """Test that the collection-level accession folds identically. + + Given: + A filter naming accession_id inside the collections sub-input. + When: + to_query builds the MongoDB predicate. + Then: + It should emit the flattened collections.accession_id path with + the value folded, matching the top-level behavior. + """ + # Act + query = to_query({"collections": [{"accession_id": ["encsr918zsj"]}]}) + + # Assert + assert query == {"collections.accession_id": "ENCSR918ZSJ"} + + +def test_to_query_should_strip_whitespace_around_an_accession(): + """Test that padding in a filter value does not defeat the match. + + Given: + An accession filter value surrounded by whitespace. + When: + to_query builds the MongoDB predicate. + Then: + It should emit the accession with the whitespace removed. + """ + # Act + query = to_query({"accession_id": [" encff525xqx "]}) + + # Assert + assert query == {"accession_id": "ENCFF525XQX"} + + +def test_to_query_should_leave_other_string_fields_untouched(): + """Test that folding is scoped to accession_id alone. + + Given: + A filter naming both accession_id and a case-sensitive sibling. + When: + to_query builds the MongoDB predicate. + Then: + It should fold only the accession, leaving filename byte-exact so + unrelated fields keep their existing matching semantics. + """ + # Act + query = to_query( + {"accession_id": ["encff525xqx"], "filename": ["MixedCase.bigBed"]} + ) + + # Assert + assert query == { + "$and": [ + {"accession_id": "ENCFF525XQX"}, + {"filename": "MixedCase.bigBed"}, + ] + } + + +def test_to_query_should_not_fold_a_field_merely_containing_the_name(): + """Test that matching is on the whole leaf segment, not a substring. + + Given: + A filter field whose name contains "accession_id" as a substring. + When: + to_query builds the MongoDB predicate. + Then: + It should leave the value unfolded, so the normalization cannot + leak onto unrelated fields as new ones are added. + """ + # Act + query = to_query({"upstream_accession_id_note": ["lower case"]}) + + # Assert + assert query == {"upstream_accession_id_note": "lower case"} + + +def test_to_query_should_fold_every_value_of_an_or_clause(): + """Test that folding survives the list-to-OR expansion. + + Given: + A filter naming several accessions in mixed casing. + When: + to_query builds the MongoDB predicate. + Then: + It should emit an $or of folded equality clauses, so no single + branch of the disjunction is left unfolded. + """ + # Act + query = to_query({"accession_id": ["4dnf1", "EnCfF2", "encsr3"]}) + + # Assert + assert query == { + "$or": [ + {"accession_id": "4DNF1"}, + {"accession_id": "ENCFF2"}, + {"accession_id": "ENCSR3"}, + ] + } + + +def test_to_query_should_leave_a_non_string_accession_value_unchanged(): + """Test that folding is guarded on the value actually being a string. + + Given: + A filter whose accession_id value is an integer, which the GraphQL + layer will not produce but a direct to_query caller can. + When: + to_query builds the MongoDB predicate. + Then: + It should emit the integer unchanged rather than raising, pinning + the isinstance guard. Targeting accession_id specifically is what + makes this test meaningful -- asserted against a field that is not + folded at all, it would pass with the guard deleted. + """ + # Act + query = to_query({"accession_id": [123]}) + + # Assert + assert query == {"accession_id": 123} + + +def test_to_query_should_leave_a_non_normalized_field_unchanged(): + """Test that an ordinary non-string field passes through. + + Given: + A filter naming an integer-valued field that is not folded. + When: + to_query builds the MongoDB predicate. + Then: + It should emit the integer unchanged. + """ + # Act + query = to_query({"size_in_bytes": [3221225472]}) + + # Assert + assert query == {"size_in_bytes": 3221225472} + + +def test_to_query_should_drop_a_blank_accession(): + """Test that a blank accession constrains nothing rather than everything. + + Emitting ``{accession_id: None}`` would match documents whose accession + is null *or absent* -- all of HuBMAP, every unparsed 4DN file, and the + whole corpus before the first post-deploy sync. That made a blank value + the only filter in the schema that widened the result set. + + Given: + A filter whose only accession value is whitespace. + When: + to_query builds the MongoDB query. + Then: + It should emit an empty query, so a search box wired straight to + the variable returns everything unfiltered rather than a page of + unrelated accession-less files. + """ + # Act + query = to_query({"accession_id": [" "]}) + + # Assert + assert query == {} + + +def test_to_query_should_keep_the_real_accession_when_one_value_is_blank(): + """Test that a blank value cannot widen a filter that also names one. + + Given: + A filter carrying a real accession alongside a blank one, as a + partly-filled multi-value input produces. + When: + to_query builds the MongoDB query. + Then: + It should emit only the real accession, rather than unioning in + every document that has none. + """ + # Act + query = to_query({"accession_id": ["4DNFIMCJXZKH", " "]}) + + # Assert + assert query == {"accession_id": "4DNFIMCJXZKH"} + + +def test_to_query_should_return_an_empty_query_for_a_filter_with_no_constraints(): + """Test that an empty clause list collapses instead of being emitted. + + MongoDB rejects ``{"$and": []}`` and ``{"$or": []}`` with BadValue, so + a filter whose every field is unset has to collapse to a query that + matches everything rather than one the server refuses. + + Given: + A filter object with no fields set. + When: + to_query builds the MongoDB query. + Then: + It should emit an empty query. + """ + # Act + query = to_query([{"filename": None, "accession_id": None}]) + + # Assert + assert query == {} + + +def test_to_query_should_not_fold_an_accession_nested_under_extra(): + """Test that folding is decided by the whole path, not the leaf name. + + ``extra.`` holds values exactly as the DCC published them, so a + DCC-native accession stored there is not folded on write. Folding it on + query would make those documents permanently unmatchable with nothing + raising -- so an unlisted path must fail closed, matching byte-exactly + like every other field. + + Given: + A filter naming accession_id under the extra.fourdn namespace. + When: + to_query builds the MongoDB predicate. + Then: + It should leave the value unfolded. + """ + # Act + query = to_query({"extra": {"fourdn": {"accession_id": ["4dnfimcjxzkh"]}}}) + + # Assert + assert query == {"extra.fourdn.accession_id": "4dnfimcjxzkh"} + + +def test_to_query_should_accept_accession_id_from_the_graphql_inputs(): + """Test the wiring from the Strawberry inputs through to the query. + + Given: + A FileMetadataInput carrying accession_id at both the file level + and inside a nested CollectionInput. + When: + The input is converted with to_dict and then to_query. + Then: + It should produce folded predicates on both paths, pinning that + the field is declared on both input classes. + """ + # Arrange + payload = FileMetadataInput( + accession_id=["4dnfimcjxzkh"], + collections=[CollectionInput(accession_id=["4dnexnhe6x77"])], + ) + + # Act + query = to_query(to_dict(payload)) + + # Assert + assert query == { + "$and": [ + {"collections.accession_id": "4DNEXNHE6X77"}, + {"accession_id": "4DNFIMCJXZKH"}, + ] + } + + +def test_to_query_should_collapse_a_single_clause(): + """Test that one set field yields a bare predicate, not a wrapper. + + Given: + A filter dict with exactly one field set. + When: + to_query flattens it. + Then: + It should return the predicate without an $and wrapper. + """ + # Act + query = to_query({"filename": ["a.bed"]}) + + # Assert + assert query == {"filename": "a.bed"} + + +def test_to_query_should_drop_fields_left_unset(): + """Test that the None fields to_dict always emits are discarded. + + Given: + A filter dict mixing set fields with explicitly-None ones, which is + exactly what to_dict produces. + When: + to_query flattens it. + Then: + It should emit only the set fields, which is what makes to_dict's + all-fields output usable as a filter. + """ + # Act + query = to_query({"filename": ["a.bed"], "md5": None, "sha256": None}) + + # Assert + assert query == {"filename": "a.bed"} + + +def test_to_query_should_flatten_nested_and_clauses(): + """Test that a nested conjunction is merged upward, not left nested. + + Given: + A sub-input contributing two fields alongside a top-level field. + When: + to_query flattens it. + Then: + It should emit one flat three-element $and rather than an $and + containing another $and. + """ + # Act + query = to_query( + {"filename": ["a.bed"], "collections": {"name": ["c"], "lab": ["l"]}} + ) + + # Assert + assert query == { + "$and": [ + {"filename": "a.bed"}, + {"collections.name": "c"}, + {"collections.lab": "l"}, + ] + } + + +def test_to_query_should_flatten_nested_or_clauses(): + """Test that a nested disjunction is merged upward. + + Given: + A list of two sub-inputs, the first of which itself expands to an + $or over two values. + When: + to_query flattens it. + Then: + It should emit one flat three-branch $or. + """ + # Act + query = to_query({"collections": [{"name": ["a", "b"]}, {"name": ["c"]}]}) + + # Assert + assert query == { + "$or": [ + {"collections.name": "a"}, + {"collections.name": "b"}, + {"collections.name": "c"}, + ] + } + + +def test_to_query_should_build_a_dotted_path_at_depth(): + """Test path construction several levels down. + + Given: + A filter reaching collections.biosamples.subjects.local_id. + When: + to_query flattens it. + Then: + It should emit the four-segment dotted path as one predicate, and + leave the value unfolded -- confirming with the substring test that + folding is decided by the last segment alone, not by depth. + """ + # Act + query = to_query( + {"collections": {"biosamples": {"subjects": {"local_id": ["Mixed-Case"]}}}} + ) + + # Assert + assert query == {"collections.biosamples.subjects.local_id": "Mixed-Case"} + + +def test_to_query_should_return_a_bare_scalar_unchanged(): + """Test the no-prefix branch, which bypasses predicate construction. + + Given: + A bare scalar passed with no prefix. + When: + to_query is called. + Then: + It should return the scalar unchanged and unfolded, since there is + no field name to decide folding by. + """ + # Act & assert + assert to_query("4dnfimcjxzkh") == "4dnfimcjxzkh" + + +@given( + accession=st.text(alphabet=_ACCESSION_CHARS, min_size=1, max_size=16), + swap=st.lists(st.booleans(), min_size=16, max_size=16), + pad=st.text(alphabet=" \t", max_size=3), +) +@settings(max_examples=200) +def test_to_query_should_build_one_predicate_for_any_casing(accession, swap, pad): + """Test that casing a caller chooses cannot change the predicate. + + Given: + Any accession over the DCC alphabet, an arbitrary per-character + re-casing of it, and arbitrary surrounding padding. + When: + to_query builds a predicate from each. + Then: + Both should equal the canonical stored form -- a stronger claim + than the two merely agreeing, which would also hold if both were + folded wrongly in the same way. + """ + # Arrange + recased = "".join( + char.lower() if flip else char for char, flip in zip(accession, swap) + ) + expected = {"accession_id": normalize_accession(accession)} + + # Act + from_canonical = to_query({"accession_id": [accession]}) + from_recased = to_query({"accession_id": [f"{pad}{recased}{pad}"]}) + + # Assert + assert from_canonical == expected + assert from_recased == expected + + +@given( + field=st.sampled_from( + ["filename", "local_id", "md5", "sha256", "persistent_id", "access_url"] + ), + value=st.text(min_size=1), +) +@settings(max_examples=100) +def test_to_query_should_leave_every_other_field_byte_identical(field, value): + """Test that no field other than the accession is ever folded. + + Given: + Any real model field name other than accession_id, with any value. + When: + to_query builds the predicate. + Then: + It should emit the value byte-identical, so no future field can be + folded by accident. + """ + # Act + query = to_query({field: [value]}) + + # Assert + assert query == {field: value} diff --git a/tests/test_metadata_endpoint.py b/tests/test_metadata_endpoint.py index d456c6a..c1739f8 100644 --- a/tests/test_metadata_endpoint.py +++ b/tests/test_metadata_endpoint.py @@ -54,6 +54,184 @@ def client_with_large_file(client): return client +@pytest.fixture() +def client_with_accession_file(client): + # Inserted through the same handle the resolvers read, so the accession + # filter runs against mongomock's real matcher rather than the + # FakeCollection double -- which matters for the nested path, since that + # double resolves dotted paths with dict lookups and cannot traverse the + # collections array at all. + asyncio.run( + api.db.files.insert_one( + { + "id_namespace": "ns", + "local_id": "51108ad5-2345-474c-a99b-0a64456b37bc", + "project_id_namespace": "ns", + "project_local_id": "proj", + "filename": "4DNFIMCJXZKH.fastq.gz", + "submission": "4dn", + "data_access_level": "public", + "accession_id": "4DNFIMCJXZKH", + "dcc": {"dcc_name": "4DN", "dcc_abbreviation": "4DN_DCIC"}, + "collections": [ + { + "id_namespace": "ns", + "local_id": "3d54d990-b73c-44a6-99b2-9054692004d6", + "name": "in situ Hi-C Experiment 4DNEXNHE6X77", + "accession_id": "4DNEXNHE6X77", + "biosamples": [], + } + ], + } + ) + ) + return client + + +def test_metadata_should_match_a_lower_case_accession_over_http( + client_with_accession_file, +): + """Test the accession round trip across the whole real stack. + + The unit tests fold on each side against an in-memory double; this + drives the same invariant through BSON, mongomock's matcher and JSON + serialization, which is where a stored form and a queried form would + actually have to meet in production. + + Given: + A 4DN file stored with the folded accession the sync writes. + When: + A lower-case accessionId filter is POSTed to /metadata. + Then: + It should return that file with the accession echoed in its stored + upper-case form. + """ + # Act + response = client_with_accession_file.post( + "/metadata", + json={ + "query": ( + '{ files(input: [{ accessionId: ["4dnfimcjxzkh"] }])' + " { totalCount items { filename accessionId } } }" + ) + }, + ) + + # Assert + assert response.status_code == 200 + body = response.json() + assert "errors" not in body + assert body["data"]["files"]["totalCount"] == 1 + item = body["data"]["files"]["items"][0] + assert item["filename"] == "4DNFIMCJXZKH.fastq.gz" + assert item["accessionId"] == "4DNFIMCJXZKH" + + +def test_metadata_should_match_a_lower_case_collection_accession_over_http( + client_with_accession_file, +): + """Test the nested accession filter against real array traversal. + + The collection accession is reached through a dotted path into an + array, so this depends on Mongo's implicit array traversal in a way + the file-level filter does not -- and is the riskier half of the + contract for that reason. + + Given: + The same file, whose nested collection carries an experiment + accession. + When: + A lower-case collections.accessionId filter is POSTed to /metadata. + Then: + It should return that file with the collection accession echoed in + its stored form. + """ + # Act + response = client_with_accession_file.post( + "/metadata", + json={ + "query": ( + "{ files(input: [{ collections: " + '[{ accessionId: ["4dnexnhe6x77"] }] }])' + " { totalCount items { collections { accessionId } } } }" + ) + }, + ) + + # Assert + assert response.status_code == 200 + body = response.json() + assert "errors" not in body + assert body["data"]["files"]["totalCount"] == 1 + collections = body["data"]["files"]["items"][0]["collections"] + assert collections[0]["accessionId"] == "4DNEXNHE6X77" + + +def test_metadata_should_serve_a_filter_whose_only_accession_is_blank( + client_with_accession_file, +): + """Test that a blank accession filter neither errors nor widens. + + Dropping the clause empties the enclosing $or, and MongoDB rejects + ``{"$or": []}`` with BadValue -- so this has to reach the real matcher + to mean anything. The in-memory double cannot detect it: its matcher + reduces an empty $and to ``all([])``, which is True. + + Given: + A stored 4DN file and a filter whose only accession is whitespace. + When: + The query is POSTed to /metadata. + Then: + It should return the file with no errors, treating the blank value + as no constraint rather than as a null match or a server error. + """ + # Act + response = client_with_accession_file.post( + "/metadata", + json={ + "query": ( + '{ files(input: [{ accessionId: [" "] }])' + " { totalCount } }" + ) + }, + ) + + # Assert + assert response.status_code == 200 + body = response.json() + assert "errors" not in body + assert body["data"]["files"]["totalCount"] == 1 + + +def test_metadata_should_serve_a_filter_with_no_fields_set( + client_with_accession_file, +): + """Test the issue #103 reproduction reaches the database. + + ``files(input: [{}])`` is a legal GraphQL document that flattened to + ``{"$and": []}``, which MongoDB and DocumentDB both reject outright, so + the query 500ed rather than returning rows. + + Given: + A stored file and a filter object with no fields set. + When: + The reported reproduction is POSTed to /metadata. + Then: + It should return the file with no errors. + """ + # Act + response = client_with_accession_file.post( + "/metadata", + json={"query": "{ files(input: [{}]) { totalCount } }"}, + ) + + # Assert + assert response.status_code == 200 + body = response.json() + assert "errors" not in body + assert body["data"]["files"]["totalCount"] == 1 + + def test_metadata_should_serve_a_multi_gigabyte_size_as_a_json_number( client_with_large_file, ): diff --git a/tests/test_models.py b/tests/test_models.py index 3cb588c..b3f342c 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,5 +1,5 @@ import pytest -from hypothesis import given +from hypothesis import given, settings from hypothesis import strategies as st from pydantic import ValidationError @@ -723,6 +723,94 @@ def test_empty_string_to_none_with_empty_encode(self): class TestFileMetadataModel: + def test___init___should_default_accession_id_to_none(self): + """Test that a file without an accession carries None. + + Given: + A document omitting accession_id, as every document does before + a sync populates it and as HuBMAP files do permanently. + When: + The model is instantiated. + Then: + It should leave accession_id as None rather than failing or + defaulting to an empty string. + """ + # Act + result = FileMetadataModel(**_minimal_file_metadata()) + + # Assert + assert result.accession_id is None + + def test___init___should_round_trip_an_accession_id(self): + """Test that a populated accession survives model construction. + + Given: + A materialized document carrying accession_id. + When: + The model is instantiated. + Then: + It should expose the accession unchanged, since the model is what + the GraphQL output type is generated from. + """ + # Arrange + doc = {**_minimal_file_metadata(), "accession_id": "4DNFIMCJXZKH"} + + # Act + result = FileMetadataModel(**doc) + + # Assert + assert result.accession_id == "4DNFIMCJXZKH" + + def test___init___should_coerce_a_blank_accession_id_to_none(self): + """Test that a blank accession reads as absent, not as empty. + + Matches normalize_accession, which already folds a blank to None on + the write side, so a document written by some other path cannot + surface an empty string the accession filter can never select. + + Given: + A document whose accession_id is the empty string. + When: + The model is instantiated. + Then: + It should coerce the value to None. + """ + # Arrange + doc = {**_minimal_file_metadata(), "accession_id": ""} + + # Act + result = FileMetadataModel(**doc) + + # Assert + assert result.accession_id is None + + @given(accession=st.text(min_size=1).filter(lambda s: s.strip())) + @settings(max_examples=100) + def test___init___should_not_refold_a_stored_accession(self, accession): + """Test that the read path leaves the stored value byte-identical. + + Deliberately no folding validator here: the model is read-path + only, so folding on read would make a mis-stored lower-case value + display correctly while remaining permanently unfindable, turning a + loud bug into a silent one. The fold belongs at the write and query + boundaries, where it is. + + Given: + Any non-blank accession, in any casing. + When: + The model is instantiated. + Then: + It should expose exactly what was stored. + """ + # Arrange + doc = {**_minimal_file_metadata(), "accession_id": accession} + + # Act + result = FileMetadataModel(**doc) + + # Assert + assert result.accession_id == accession + def test___init___should_preserve_the_uncompressed_sentinel(self): """Test that the uncompressed sentinel is not collapsed into None. @@ -1073,6 +1161,56 @@ def test_empty_string_to_none_with_valid_file_format(self): class TestCollection: + def test___init___should_default_accession_id_to_none(self): + """Test that a collection without an accession carries None. + + Given: + A Collection constructed without accession_id, as ENCODE's + biosample-keyed fallback collection is. + When: + The model is instantiated. + Then: + It should leave accession_id as None. + """ + # Act + result = Collection(biosamples=[]) + + # Assert + assert result.accession_id is None + + def test___init___should_round_trip_an_accession_id(self): + """Test that a populated experiment accession survives construction. + + Given: + A Collection carrying a 4DN experiment accession. + When: + The model is instantiated. + Then: + It should expose the accession unchanged, since the nested GraphQL + collection type is generated from this model. + """ + # Act + result = Collection(biosamples=[], accession_id="4DNEXNHE6X77") + + # Assert + assert result.accession_id == "4DNEXNHE6X77" + + def test___init___should_coerce_a_blank_accession_id_to_none(self): + """Test that a blank collection accession reads as absent. + + Given: + A Collection whose accession_id is the empty string. + When: + The model is instantiated. + Then: + It should coerce the value to None. + """ + # Act + result = Collection(biosamples=[], accession_id="") + + # Assert + assert result.accession_id is None + def test_empty_string_to_none_with_empty_extra(self): """Test empty string coercion on the extra field. diff --git a/tests/test_schema.py b/tests/test_schema.py index 22898a9..78c5a4b 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -6,6 +6,7 @@ import logging import pytest +from bson import ObjectId from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st from mongomock_motor import AsyncMongoMockClient @@ -54,9 +55,29 @@ def test_from_pydantic_should_convert_nested_model_lists_and_leave_json_untouche def _make_file_doc( - local_id: str, submission: str = "hubmap", size_in_bytes: int | None = None + local_id: str, + submission: str = "hubmap", + size_in_bytes: int | None = None, + accession_id: str | None = None, + collection_accession_id: str | None = None, ) -> dict: - """Return a minimal file document that satisfies FileMetadataModel.""" + """Return a minimal file document that satisfies FileMetadataModel. + + ``accession_id`` and ``collection_accession_id`` default to None so + existing callers are unaffected; pass them in the stored (already + case-folded) form the sync pipeline writes. + """ + collections = [] + if collection_accession_id is not None: + collections = [ + { + "id_namespace": "ns", + "local_id": "coll1", + "name": "a collection", + "accession_id": collection_accession_id, + "biosamples": [], + } + ] return { "id_namespace": "ns", "local_id": local_id, @@ -66,11 +87,12 @@ def _make_file_doc( "submission": submission, "data_access_level": "public", "size_in_bytes": size_in_bytes, + "accession_id": accession_id, "dcc": { "dcc_name": submission.upper(), "dcc_abbreviation": submission, }, - "collections": [], + "collections": collections, } @@ -105,6 +127,153 @@ def _patch_cutover(self, mocker): """No-op ``locks.wait_for_cutover`` for every test in this class.""" mocker.patch.object(locks, "wait_for_cutover", return_value=None) + @pytest.mark.asyncio + @pytest.mark.parametrize( + "typed", + ["4dnfimcjxzkh", "4DNFIMCJXZKH", "4DnFiMcJxZkH", " 4dnfimcjxzkh "], + ids=["lower", "upper", "mixed", "padded"], + ) + async def test_files_should_match_a_stored_accession_in_any_casing( + self, mock_db, typed + ): + """Test the round trip the accession field exists to provide. + + This is the one invariant no other test covers: the ingest side and + the query side each fold, and every other test pins only one of + them. If either fold changed, they would all still pass while every + accession lookup silently returned nothing. + + Given: + A file stored with the folded accession the sync pipeline writes. + When: + The files query filters on that accession as a caller might + type it -- lower, upper, mixed case, or padded. + Then: + It should return the file in every casing. + """ + # Arrange + mock_db.files.docs = [_make_file_doc("f1", accession_id="4DNFIMCJXZKH")] + + # Act + result = await schema.execute( + "query($a: [String!]) { files(input: [{accessionId: $a}]) " + "{ totalCount items { accessionId } } }", + variable_values={"a": [typed]}, + ) + + # Assert + assert result.errors is None + assert result.data["files"]["totalCount"] == 1 + assert result.data["files"]["items"][0]["accessionId"] == "4DNFIMCJXZKH" + + @pytest.mark.asyncio + async def test_files_should_return_the_nested_collection_accession(self, mock_db): + """Test that the accession survives onto the generated collection type. + + The nested output type is built by build_strawberry_type from the + pydantic model and populated through from_pydantic's wrapper + peeling, so it is a separate path from the file-level field. + + Given: + A file whose nested collection carries an accession. + When: + The files query selects collections { accessionId }. + Then: + It should return the stored collection accession. + """ + # Arrange + mock_db.files.docs = [ + _make_file_doc("f1", collection_accession_id="4DNEXNHE6X77") + ] + + # Act + result = await schema.execute( + "{ files { items { collections { accessionId } } } }" + ) + + # Assert + assert result.errors is None + collections = result.data["files"]["items"][0]["collections"] + assert collections[0]["accessionId"] == "4DNEXNHE6X77" + + @pytest.mark.asyncio + async def test_files_should_serialize_a_missing_accession_as_null(self, mock_db): + """Test that an accession-less file is not an error. + + Given: + A file with no accession at either level, as every HuBMAP file + permanently is. + When: + The files query selects both accession fields. + Then: + It should serialize the file-level field as null and return the + empty collection list, without a GraphQL error. + """ + # Arrange + mock_db.files.docs = [_make_file_doc("f1")] + + # Act + result = await schema.execute( + "{ files { items { accessionId collections { accessionId } } } }" + ) + + # Assert + assert result.errors is None + assert result.data["files"]["items"][0]["accessionId"] is None + assert result.data["files"]["items"][0]["collections"] == [] + + @pytest.mark.asyncio + async def test_files_should_return_no_matches_for_an_unknown_accession( + self, mock_db + ): + """Test that a folded predicate matching nothing is a clean miss. + + Given: + A file stored under one accession. + When: + The files query filters on a different accession. + Then: + It should return zero matches with no error, rather than a + malformed query. + """ + # Arrange + mock_db.files.docs = [_make_file_doc("f1", accession_id="ENCFF525XQX")] + + # Act + result = await schema.execute( + '{ files(input: [{accessionId: ["4DNFIMCJXZKH"]}]) { totalCount } }' + ) + + # Assert + assert result.errors is None + assert result.data["files"]["totalCount"] == 0 + + @pytest.mark.asyncio + async def test_file_should_return_the_accession_for_a_single_lookup(self, mock_db): + """Test the second, independent from_pydantic call site. + + Given: + One accession-bearing file addressed by its ObjectId. + When: + The single-file query selects accessionId. + Then: + It should return the stored accession, since the file resolver + builds its type through a separate conversion from files. + """ + # Arrange + doc = _make_file_doc("f1", accession_id="ENCFF525XQX") + doc["_id"] = ObjectId() + mock_db.files.docs = [doc] + + # Act + result = await schema.execute( + '{ file(id: "%s") { accessionId } }' % doc["_id"] + ) + + # Assert + assert result.errors is None + assert result.data["file"]["accessionId"] == "ENCFF525XQX" + @pytest.mark.asyncio async def test_files_should_return_page_size_items_when_more_documents_match( self, mock_db @@ -1129,6 +1298,41 @@ def _patch_cutover(self, mocker): """No-op ``locks.wait_for_cutover`` for every test in this class.""" mocker.patch.object(locks, "wait_for_cutover", return_value=None) + @pytest.mark.asyncio + @pytest.mark.parametrize( + "field", ["accession_id", "collections.accession_id"] + ) + async def test_distinct_values_should_reject_the_accession_fields( + self, mock_db, field + ): + """Test that accessions stay out of the distinct-values allowlist. + + That allowlist is for low-cardinality facet fields a client can + enumerate in a filter UI. An accession is unique per document, so + distinct over it would return one value per row -- hundreds of + thousands on the live corpus. The exclusion is deliberate, and + this pins it against a later well-meaning addition. + + Given: + Neither accession field is in ALLOWED_DISTINCT_FIELDS. + When: + The distinctValues query requests one of them. + Then: + It should return an error naming the rejected field. + """ + # Arrange + mock_db.files.docs = [_make_distinct_doc("f1", "4DN", "4dn")] + + # Act + result = await schema.execute( + "query($f: [String!]!) { distinctValues(fields: $f) { field values } }", + variable_values={"f": [field]}, + ) + + # Assert + assert result.errors is not None + assert field in str(result.errors[0].message) + @pytest.mark.asyncio async def test_distinct_values_returns_all_unique_values_for_single_field( self, mock_db @@ -1507,6 +1711,36 @@ def _patch_cutover(self, mocker): """No-op ``locks.wait_for_cutover`` for every test in this class.""" mocker.patch.object(locks, "wait_for_cutover", return_value=None) + @pytest.mark.asyncio + async def test_file_count_should_fold_a_lower_case_accession_filter(self, mock_db): + """Test that folding reaches the count resolver, not just the paged one. + + fileCount builds its query through a separate to_query call site, + so a fold wired only into the files resolver would leave counts + disagreeing with the page they describe. + + Given: + Two files, one stored under an accession and one without. + When: + fileCount filters on that accession in lower case. + Then: + It should return 1. + """ + # Arrange + mock_db.files.docs = [ + _make_file_doc("f1", accession_id="ENCFF525XQX"), + _make_file_doc("f2"), + ] + + # Act + result = await schema.execute( + '{ fileCount(input: [{accessionId: ["encff525xqx"]}]) }' + ) + + # Assert + assert result.errors is None + assert result.data["fileCount"] == 1 + @pytest.mark.asyncio async def test_file_count_should_return_total_when_no_filter(self, mock_db): """Test the file count reflects every document when no filter is supplied. @@ -2430,3 +2664,58 @@ def test_render_should_match_the_checked_in_sdl(): assert SCHEMA_PATH.read_text(encoding="utf-8") == generated, ( "schema.graphql is stale — run `make schema` to regenerate it." ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("type_name", "kind"), + [ + ("FileMetadataType", "output"), + ("CollectionType", "output"), + ("FileMetadataInput", "input"), + ("CollectionInput", "input"), + ], +) +async def test_schema_should_publish_accession_id_on_all_four_surfaces( + type_name, kind +): + """Test the accession's declared shape on every type that carries it. + + The SDL drift guard above cannot cover this: regenerating a wrong + schema makes it pass. This pins intent instead -- an accession is + optional on output because most DCCs issue none, and a list on input + because the filter convention expands a list into an OR. + + Given: + The published schema, where the accession appears on two output + types and two input types. + When: + Each type is introspected. + Then: + It should expose accessionId as a nullable String on the output + types and a list of non-null String on the input types. + """ + # Act + result = await schema.execute( + """ + query($n: String!) { + __type(name: $n) { + fields { name type { kind name ofType { kind name } } } + inputFields { name type { kind name ofType { kind name ofType { kind name } } } } + } + } + """, + variable_values={"n": type_name}, + ) + + # Assert + assert result.errors is None + declared = result.data["__type"]["fields" if kind == "output" else "inputFields"] + field = next(f for f in declared if f["name"] == "accessionId") + if kind == "output": + # Nullable: the named scalar is reached without a NON_NULL wrapper. + assert field["type"]["kind"] == "SCALAR" + assert field["type"]["name"] == "String" + else: + assert field["type"]["kind"] == "LIST" + assert _named_type(field["type"]) == "String" diff --git a/tests/test_sync.py b/tests/test_sync.py index ff9b78c..46e36a6 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -2,18 +2,24 @@ from __future__ import annotations +import asyncio import logging import pytest from cfdb.services import encode as encode_module +from cfdb.services import fourdn as fourdn_module from cfdb.services import sync as sync_module from cfdb.services.sync import ( SyncTask, + _enrich_4dn_api_metadata, + _enrich_4dn_collections, _enrich_hubmap_collections_and_subjects, _enrich_hubmap_files, _load_dataset_async, _prune_non_public_hubmap_raw_records, + _set_accession_ids, + _stamp_4dn_file_accessions, _sync_dccs, _sync_encode, ) @@ -387,6 +393,41 @@ async def test__load_dataset_async_should_not_add_compression_format_when_absent class TestSyncEncode: + @pytest.mark.asyncio + async def test__sync_encode_should_populate_accession_id_on_inserted_docs( + self, mock_db, mocker + ): + """Test that the accession survives the path that reaches the database. + + ENCODE writes the files collection directly rather than through the + materializer, so this pipeline is the only writer of both accession + levels for that DCC. + + Given: + An ENCODE row carrying a file accession, an experiment accession + and a biosample term. + When: + _sync_encode runs the fetch-transform-insert pipeline. + Then: + The inserted document should carry the folded accession at both + the file and the collection level. + """ + # Arrange + row = _encode_metadata_row("encff001aaa", "encff001aaa.bed.gz") + row["Experiment accession"] = "encsr918zsj" + row["Biosample term name"] = "K562" + mocker.patch.object( + encode_module, "fetch_encode_metadata", lambda: _async_iter([row]) + ) + + # Act + await _sync_encode(SyncTask(id="t1", dcc_names=["encode"])) + + # Assert + doc = next(d for d in mock_db.files.docs if d["submission"] == "encode") + assert doc["accession_id"] == "ENCFF001AAA" + assert doc["collections"][0]["accession_id"] == "ENCSR918ZSJ" + @pytest.mark.asyncio async def test__sync_encode_should_populate_compression_format_on_inserted_docs( self, mock_db, mocker @@ -504,3 +545,714 @@ async def test__sync_encode_should_leave_other_dcc_documents_unchanged( # Assert survivors = [d for d in mock_db.files.docs if d["submission"] != "encode"] assert survivors == others + + +class TestStamp4dnFileAccessions: + @pytest.mark.asyncio + async def test__stamp_4dn_file_accessions_should_write_the_raw_file_collection( + self, mock_db + ): + """Test that the accession survives a re-materialization. + + The materializer rebuilds ``files`` from ``file`` on every run, so a + value written to ``files`` lasts only until the next + ``make materialize-dcc``. Writing the raw document instead lets + ``enrich_file``'s in-place mutation carry it forward, which is what + already makes the collection accession durable. + + Given: + A raw 4DN file document whose persistent_id carries an accession. + When: + _stamp_4dn_file_accessions runs. + Then: + It should stamp the raw file collection, leaving the derived + files collection untouched. + """ + # Arrange + mock_db.file.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMCJXZKH", + } + ] + mock_db.files.docs = [{"_id": "f1", "submission": "4dn"}] + + # Act + await _stamp_4dn_file_accessions() + + # Assert + assert mock_db.file.docs[0]["accession_id"] == "4DNFIMCJXZKH" + assert "accession_id" not in mock_db.files.docs[0] + + @pytest.mark.asyncio + async def test__stamp_4dn_file_accessions_should_stamp_without_the_search_api( + self, mock_db + ): + """Test that accession_id does not depend on the Search API at all. + + Given: + Two raw 4DN files with parseable accessions and no Search API + stub of any kind, since this pass predates the fetch. + When: + _stamp_4dn_file_accessions runs. + Then: + It should stamp both, so every 4DN file is queryable by accession + rather than only the API-matched subset. + """ + # Arrange + mock_db.file.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMCJXZKH", + }, + { + "_id": "f2", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMEMLGM5", + }, + ] + + # Act + await _stamp_4dn_file_accessions() + + # Assert + assert [d["accession_id"] for d in mock_db.file.docs] == [ + "4DNFIMCJXZKH", + "4DNFIMEMLGM5", + ] + + @pytest.mark.asyncio + async def test__stamp_4dn_file_accessions_should_warn_when_an_accession_is_unparseable( + self, mock_db, caplog + ): + """Test the operator's only signal that the field is partial. + + Given: + Two raw 4DN files, one of whose persistent_ids carries no + accession. + When: + _stamp_4dn_file_accessions runs. + Then: + It should log a warning naming the unparseable count, since a + null accession_id is otherwise indistinguishable from a DCC + that issues no accession. + """ + # Arrange + mock_db.file.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMCJXZKH", + }, + {"_id": "f2", "submission": "4dn", "persistent_id": "https://x/nope"}, + ] + + # Act + with caplog.at_level(logging.WARNING): + await _stamp_4dn_file_accessions() + + # Assert + assert "1 files have no parseable accession" in caplog.text + + @pytest.mark.asyncio + async def test__stamp_4dn_file_accessions_should_leave_accession_id_unset_when_unparseable( + self, mock_db + ): + """Test that a file with no parseable accession is skipped, not failed. + + Given: + A raw 4DN file whose persistent_id carries no 4DNF accession. + When: + _stamp_4dn_file_accessions runs. + Then: + It should leave accession_id absent and complete without raising, + so one malformed row cannot abort the sync. + """ + # Arrange + mock_db.file.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/no-accession-here", + } + ] + + # Act + await _stamp_4dn_file_accessions() + + # Assert + assert "accession_id" not in mock_db.file.docs[0] + + +class TestEnrich4dnApiMetadata: + @pytest.mark.asyncio + async def test__enrich_4dn_api_metadata_should_enrich_both_files_sharing_an_accession( + self, mocker, mock_db + ): + """Test that a duplicate accession does not cost a file its metadata. + + The lookup was an accession-keyed dict, which is last-write-wins: + of two files resolving to one accession, whichever the cursor + yielded second overwrote the first and only it was enriched. Which + one lost depended on cursor order, and nothing reported it. + + Given: + Two 4DN files whose persistent_ids carry the same accession, + and a Search API returning metadata for it. + When: + _enrich_4dn_api_metadata runs. + Then: + It should enrich both. + """ + # Arrange + mock_db.files.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMCJXZKH", + }, + { + "_id": "f2", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMCJXZKH/@@download", + }, + ] + mocker.patch.object( + fourdn_module, + "fetch_file_metadata_bulk", + mocker.AsyncMock( + return_value={"4DNFIMCJXZKH": {"genome_assembly": "GRCh38"}} + ), + ) + mocker.patch.object( + fourdn_module, "fetch_biosource_tiers", mocker.AsyncMock(return_value={}) + ) + + # Act + await _enrich_4dn_api_metadata() + + # Assert + assert [d.get("genome_assembly") for d in mock_db.files.docs] == [ + "GRCh38", + "GRCh38", + ] + + @pytest.mark.asyncio + async def test__enrich_4dn_api_metadata_should_enrich_a_mixed_case_persistent_id( + self, mocker, mock_db + ): + """Test that the extracted accession joins the Search API response. + + The extracted value is the key for the API round trip, and the + portal answers with its own upper-case form. While the extractor + returned the raw match, a mixed-case persistent_id produced a key + that joined against nothing: the file kept a correct accession_id + and silently lost every enriched field, without being counted in + the unparseable warning that is the operator's only signal. + + Given: + A 4DN file whose persistent_id carries a mixed-case accession, + and a Search API keyed on the canonical upper-case form. + When: + _enrich_4dn_api_metadata runs. + Then: + It should apply the enrichment, not merely fail quietly. + """ + # Arrange + mock_db.files.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFImcjxzkh", + } + ] + mocker.patch.object( + fourdn_module, + "fetch_file_metadata_bulk", + mocker.AsyncMock( + return_value={"4DNFIMCJXZKH": {"genome_assembly": "GRCh38"}} + ), + ) + mocker.patch.object( + fourdn_module, "fetch_biosource_tiers", mocker.AsyncMock(return_value={}) + ) + + # Act + await _enrich_4dn_api_metadata() + + # Assert + assert mock_db.files.docs[0]["genome_assembly"] == "GRCh38" + + @pytest.mark.asyncio + async def test__enrich_4dn_api_metadata_should_enrich_only_the_matched_file( + self, mocker, mock_db + ): + """Test that enrichment applies to exactly the API-matched subset. + + Given: + Two materialized 4DN files with parseable accessions, where the + Search API returns metadata for only one of them. + When: + _enrich_4dn_api_metadata runs. + Then: + It should apply the enrichment fields to the matched file only, + leaving the unmatched one untouched. + """ + # Arrange + mock_db.files.docs = [ + { + "_id": "f1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMCJXZKH", + }, + { + "_id": "f2", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNFIMEMLGM5", + }, + ] + mocker.patch.object( + fourdn_module, + "fetch_file_metadata_bulk", + mocker.AsyncMock( + return_value={"4DNFIMCJXZKH": {"genome_assembly": "GRCh38"}} + ), + ) + mocker.patch.object( + fourdn_module, "fetch_biosource_tiers", mocker.AsyncMock(return_value={}) + ) + + # Act + await _enrich_4dn_api_metadata() + + # Assert + matched, unmatched = mock_db.files.docs + assert matched["genome_assembly"] == "GRCh38" + assert "genome_assembly" not in unmatched + + +class TestEnrich4dnCollections: + @pytest.mark.asyncio + async def test__enrich_4dn_collections_should_stamp_accession_id_when_api_returns_nothing( + self, mocker, mock_db + ): + """Test that the collection accession does not depend on an API match. + + Given: + A raw 4DN collection whose persistent_id carries an experiment + accession, and a Search API that returns no experiments. + When: + _enrich_4dn_collections runs. + Then: + It should still set accession_id, so the value is present for the + materializer to embed into files.collections[]. + """ + # Arrange + mock_db.collection.docs = [ + { + "_id": "c1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNEXNHE6X77", + } + ] + mocker.patch.object( + fourdn_module, + "fetch_experiment_metadata_bulk", + mocker.AsyncMock(return_value={}), + ) + + # Act + await _enrich_4dn_collections() + + # Assert + assert mock_db.collection.docs[0]["accession_id"] == "4DNEXNHE6X77" + + @pytest.mark.asyncio + async def test__enrich_4dn_collections_should_stamp_accession_id_when_the_api_raises( + self, mocker, mock_db + ): + """Test that the stamp survives an API failure, not just an empty result. + + fetch_experiment_metadata_bulk catches only aiohttp.ClientError, so + a TimeoutError from its 60-second budget propagates. While the + fetch ran before the scan, that aborted the pass with nothing + stamped -- the same guarantee the file pass makes, but only against + the gentler failure. + + Given: + A raw 4DN collection with a parseable accession, and a Search + API that raises rather than returning an empty result. + When: + _enrich_4dn_collections runs. + Then: + It should have stamped accession_id before the failure + propagates. + """ + # Arrange + mock_db.collection.docs = [ + { + "_id": "c1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNEXNHE6X77", + } + ] + mocker.patch.object( + fourdn_module, + "fetch_experiment_metadata_bulk", + mocker.AsyncMock(side_effect=asyncio.TimeoutError()), + ) + + # Act + with pytest.raises(asyncio.TimeoutError): + await _enrich_4dn_collections() + + # Assert + assert mock_db.collection.docs[0]["accession_id"] == "4DNEXNHE6X77" + + @pytest.mark.asyncio + async def test__enrich_4dn_collections_should_leave_accession_id_unset_when_unparseable( + self, mocker, mock_db + ): + """Test that a collection with no parseable accession is skipped. + + Given: + A 4DN collection whose persistent_id carries no 4DNE accession. + When: + _enrich_4dn_collections runs. + Then: + It should leave accession_id absent and complete without raising. + """ + # Arrange + mock_db.collection.docs = [ + { + "_id": "c1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/nothing-here", + } + ] + mocker.patch.object( + fourdn_module, + "fetch_experiment_metadata_bulk", + mocker.AsyncMock(return_value={}), + ) + + # Act + await _enrich_4dn_collections() + + # Assert + assert "accession_id" not in mock_db.collection.docs[0] + + @pytest.mark.asyncio + async def test__enrich_4dn_collections_should_stamp_accession_id_alongside_api_metadata( + self, mocker, mock_db + ): + """Test that stamping does not displace the existing enrichment. + + Given: + A 4DN collection the Search API does return experiment metadata for. + When: + _enrich_4dn_collections runs. + Then: + It should set accession_id and the promoted experiment fields + together, so the new write does not regress the existing pass. + """ + # Arrange + mock_db.collection.docs = [ + { + "_id": "c1", + "submission": "4dn", + "persistent_id": "https://data.4dnucleome.org/4DNEXNHE6X77", + } + ] + mocker.patch.object( + fourdn_module, + "fetch_experiment_metadata_bulk", + mocker.AsyncMock( + return_value={ + "4DNEXNHE6X77": { + "lab": "Some Lab", + "experiment_type": "in situ Hi-C", + # Lands under extra.fourdn via a dotted $set, so this + # also pins that the nested write still nests. + "status": "released", + } + } + ), + ) + + # Act + await _enrich_4dn_collections() + + # Assert + doc = mock_db.collection.docs[0] + assert doc["accession_id"] == "4DNEXNHE6X77" + assert doc["lab"] == "Some Lab" + assert doc["experiment_type"] == "in situ Hi-C" + assert doc["extra"]["fourdn"] == {"status": "released"} + + +class TestLogAccessionCoverage: + @pytest.mark.asyncio + async def test__log_accession_coverage_should_report_partial_coverage( + self, mock_db, caplog + ): + """Test that the operator can see how much of a DCC is queryable. + + Given: + Three 4DN files of which two carry an accession. + When: + _log_accession_coverage runs for that DCC. + Then: + It should log the covered and total counts. + """ + # Arrange + mock_db.files.docs = [ + {"_id": "f1", "submission": "4dn", "accession_id": "4DNFAAA"}, + {"_id": "f2", "submission": "4dn", "accession_id": "4DNFBBB"}, + {"_id": "f3", "submission": "4dn"}, + ] + + # Act + with caplog.at_level(logging.INFO): + await sync_module._log_accession_coverage("4dn") + + # Assert + assert "2/3" in caplog.text + + @pytest.mark.asyncio + async def test__log_accession_coverage_should_warn_when_nothing_is_covered( + self, mock_db, caplog + ): + """Test the signal that distinguishes empty from unpopulated. + + A filter against an unstamped corpus returns totalCount 0 with no + error, which reads exactly like "no such accession". Zero coverage + is the one case an operator has to be told about -- it is also what + a standalone re-materialization leaves behind. + + Given: + A DCC whose files carry no accession at all. + When: + _log_accession_coverage runs. + Then: + It should warn that accession filters will not match. + """ + # Arrange + mock_db.files.docs = [{"_id": "f1", "submission": "hubmap"}] + + # Act + with caplog.at_level(logging.WARNING): + await sync_module._log_accession_coverage("hubmap") + + # Assert + assert "will return no matches" in caplog.text + +class TestSyncEncodeIndexes: + @pytest.mark.asyncio + async def test__sync_encode_should_ensure_the_accession_indexes( + self, mocker, mock_db + ): + """Test that an ENCODE-only database is not left unindexed. + + The materializer creates the files indexes at the end of its run, + and _sync_encode never invokes it -- it writes documents straight + into files. On a database where ENCODE is the only DCC synced, that + left files with no index at all, so every accession lookup scanned + the whole collection on a public endpoint. + + Given: + An ENCODE sync over a single row. + When: + _sync_encode completes. + Then: + It should have ensured both accession indexes on files. + """ + # Arrange + row = _encode_metadata_row("encff001aaa", "encff001aaa.bed.gz") + mocker.patch.object( + encode_module, "fetch_encode_metadata", lambda: _async_iter([row]) + ) + + # Act + await _sync_encode(SyncTask(id="t1", dcc_names=["encode"])) + + # Assert + indexed = {tuple(keys.items()) for keys, _ in mock_db.files._indexes} + assert (("accession_id", 1),) in indexed + assert (("collections.accession_id", 1),) in indexed + +class TestSetAccessionIds: + @pytest.mark.asyncio + async def test__set_accession_ids_should_continue_when_a_batch_fails( + self, mocker, mock_db, caplog + ): + """Test that a failed stamp degrades the field, not the whole sync. + + Stamping is the first write of each enrichment pass, so an escaping + BulkWriteError would abort the pass before any Search API + enrichment ran and fail the sync -- costing more than the + accessions it failed to write. The two are independent by design. + + Given: + A bulk_write that raises BulkWriteError. + When: + _set_accession_ids runs. + Then: + It should report zero modifications, log the shortfall at + ERROR, and return rather than propagate, so the caller's + enrichment pass still runs. + """ + # Arrange + from pymongo.errors import BulkWriteError + + mocker.patch.object( + mock_db.files, + "bulk_write", + mocker.AsyncMock(side_effect=BulkWriteError({"writeErrors": []})), + ) + + # Act + with caplog.at_level(logging.ERROR): + modified = await _set_accession_ids( + mock_db.files, [("f0", "4DNFAAA")], "test" + ) + + # Assert + assert modified == 0 + assert "not stamped" in caplog.text + + @pytest.mark.asyncio + async def test__set_accession_ids_should_stamp_every_document(self, mock_db): + """Test that each pair produces its own stamp. + + Given: + Three documents, each with its own accession. + When: + _set_accession_ids runs. + Then: + It should stamp all three and report three modifications. + """ + # Arrange + mock_db.files.docs = [{"_id": f"f{i}"} for i in range(3)] + stamps = [("f0", "4DNFAAA"), ("f1", "4DNFBBB"), ("f2", "4DNFCCC")] + + # Act + modified = await _set_accession_ids(mock_db.files, stamps, "test") + + # Assert + assert modified == 3 + assert [d["accession_id"] for d in mock_db.files.docs] == [ + "4DNFAAA", + "4DNFBBB", + "4DNFCCC", + ] + + @pytest.mark.asyncio + async def test__set_accession_ids_should_stamp_both_documents_sharing_an_accession( + self, mock_db + ): + """Test that a duplicate accession does not cost a document its stamp. + + This is why the helper takes a list of pairs rather than the + accession-keyed dict the callers also build: that dict is + last-write-wins, so one of these two documents would be dropped + before any update was issued, left with a null accession despite + having parsed cleanly, and which one lost would depend on cursor + order. + + Given: + Two distinct documents whose persistent_ids resolve to the same + accession. + When: + _set_accession_ids runs. + Then: + It should stamp both. + """ + # Arrange + mock_db.files.docs = [{"_id": "f1"}, {"_id": "f2"}] + stamps = [("f1", "4DNFIMCJXZKH"), ("f2", "4DNFIMCJXZKH")] + + # Act + modified = await _set_accession_ids(mock_db.files, stamps, "test") + + # Assert + assert modified == 2 + assert all(d["accession_id"] == "4DNFIMCJXZKH" for d in mock_db.files.docs) + + @pytest.mark.asyncio + async def test__set_accession_ids_should_fold_the_accession(self, mock_db): + """Test that the stored form matches what a filter folds to. + + Given: + A pair whose accession is lower-cased and padded. + When: + _set_accession_ids runs. + Then: + It should store the stripped, upper-cased form. + """ + # Arrange + mock_db.files.docs = [{"_id": "f1"}] + + # Act + await _set_accession_ids(mock_db.files, [("f1", " 4dnfimcjxzkh ")], "test") + + # Assert + assert mock_db.files.docs[0]["accession_id"] == "4DNFIMCJXZKH" + + @pytest.mark.asyncio + async def test__set_accession_ids_should_stamp_across_batch_boundaries( + self, mocker, mock_db + ): + """Test that batching does not drop documents at the seams. + + Given: + Five documents and a batch size of two, so the final batch is + partial. + When: + _set_accession_ids runs. + Then: + It should issue three unordered bulk writes and stamp all five, + so no document is lost to a boundary. + """ + # Arrange + mocker.patch.object(sync_module, "BATCH_SIZE", 2) + mock_db.files.docs = [{"_id": f"f{i}"} for i in range(5)] + stamps = [(f"f{i}", f"4DNF{i}") for i in range(5)] + spy = mocker.spy(mock_db.files, "bulk_write") + + # Act + modified = await _set_accession_ids(mock_db.files, stamps, "test") + + # Assert + assert modified == 5 + assert spy.call_count == 3 + assert all(call.kwargs["ordered"] is False for call in spy.call_args_list) + + @pytest.mark.asyncio + async def test__set_accession_ids_should_do_nothing_when_given_no_pairs( + self, mock_db, caplog + ): + """Test the empty guard. + + Given: + No pairs at all, as happens when a DCC yields no parseable + accession. + When: + _set_accession_ids runs. + Then: + It should report zero, issue no write, and warn under a label + naming which pass was empty. + """ + # Arrange + mock_db.files.docs = [{"_id": "f1"}] + + # Act + with caplog.at_level(logging.WARNING): + modified = await _set_accession_ids(mock_db.files, [], "4DN file") + + # Assert + assert modified == 0 + assert "accession_id" not in mock_db.files.docs[0] + assert "4DN file" in caplog.text diff --git a/tests/test_workflows/test_executor.py b/tests/test_workflows/test_executor.py index a33a86b..b3d0071 100644 --- a/tests/test_workflows/test_executor.py +++ b/tests/test_workflows/test_executor.py @@ -112,7 +112,7 @@ def _file_meta() -> dict[str, Any]: def _install_jobs_index(mock_db) -> None: - mock_db.jobs.create_index( + mock_db.jobs.register_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"active": True}, diff --git a/tests/test_workflows/test_lock.py b/tests/test_workflows/test_lock.py index af7a9ae..0a8c025 100644 --- a/tests/test_workflows/test_lock.py +++ b/tests/test_workflows/test_lock.py @@ -45,7 +45,7 @@ def _insert_job( def _install_jobs_index(mock_db) -> None: """Register the partial unique index on the FakeDB jobs collection.""" - mock_db.jobs.create_index( + mock_db.jobs.register_index( {"workflow_key": 1}, unique=True, # Match production: the mutex partial-unique index filters on the diff --git a/tests/test_workflows/test_s3_contract.py b/tests/test_workflows/test_s3_contract.py index 3f3a5f4..06f99b9 100644 --- a/tests/test_workflows/test_s3_contract.py +++ b/tests/test_workflows/test_s3_contract.py @@ -76,7 +76,7 @@ async def run(self, file_meta, workdir, cache) -> AsyncIterator[WorkflowEvent]: def _install_jobs_index(mock_db) -> None: - mock_db.jobs.create_index( + mock_db.jobs.register_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"active": True},