Ingest ENCODE annotation files during sync — Closes #94

README.md

@@ -560,11 +560,13 @@ Anatomy, FileFormat, DataType, and AssayType share an identical schema: `id` (st
 | Entity | Ontology Source |
 |--------|----------------|
 | Anatomy | UBERON (Uber-anatomy ontology) |
-| FileFormat | EDAM CV `format:` terms |
+| FileFormat | EDAM CV `format:` terms, or a `cfdb:`-prefixed token minted where EDAM has none |
 | DataType | EDAM CV `data:` terms |
 | AssayType | OBI (Ontology for Biomedical Investigations) |
 | NcbiTaxonomy | NCBI Taxonomy Database |
 
+Every `id` above resolves in its source ontology except the minted `FileFormat` tokens. EDAM has no term for a few formats cfdb ingests — `bedpe` and `bigInteract` at present — and aliasing them onto the nearest EDAM term would make each indistinguishable from the format it was aliased to, so a `cfdb:` token is minted instead. A client resolving `file_format.id` against EDAM should skip ids carrying that prefix rather than treat them as resolvable.
+
 #### Subject
 
 A human or organism from which biosamples are derived.

schema.graphql

@@ -156,6 +156,9 @@ type EnrichedCollectionType {
 
 input EnrichedEncodeBiosampleInput {
   biosampleType: [String!] = null
+  lifeStage: [String!] = null
+  age: [String!] = null
+  ageUnits: [String!] = null
   biosampleTreatments: [String!] = null
   biosampleTreatmentsAmount: [String!] = null
   biosampleTreatmentsDuration: [String!] = null
@@ -172,6 +175,9 @@ input EnrichedEncodeBiosampleInput {
 
 type EnrichedEncodeBiosampleType {
   biosampleType: String
+  lifeStage: String
+  age: String
+  ageUnits: String
   biosampleTreatments: String
   biosampleTreatmentsAmount: String
   biosampleTreatmentsDuration: String
@@ -192,17 +198,25 @@ input EnrichedEncodeCollectionInput {
   platform: [String!] = null
   dbxrefs: [String!] = null
   rbnsProteinConcentration: [String!] = null
+  annotationType: [String!] = null
+  softwareUsed: [String!] = null
+  encyclopediaVersion: [String!] = null
 }
 
 type EnrichedEncodeCollectionType {
   project: String
   platform: String
   dbxrefs: String
   rbnsProteinConcentration: String
+  annotationType: String
+  softwareUsed: String
+  encyclopediaVersion: String
 }
 
 input EnrichedEncodeFileInput {
   assembly: [String!] = null
+  annotationType: [String!] = null
+  organism: [String!] = null
   fileFormatType: [String!] = null
   outputType: [String!] = null
   genomeAnnotation: [String!] = null
@@ -225,6 +239,8 @@ input EnrichedEncodeFileInput {
 
 type EnrichedEncodeFileType {
   assembly: String
+  annotationType: String
+  organism: String
   fileFormatType: String
   outputType: String
   genomeAnnotation: String

src/cfdb/api/gql/inputs.py

@@ -70,6 +70,9 @@ class SubjectInput:
 @strawberry.input
 class EnrichedEncodeBiosampleInput:
     biosample_type: list[str] | None = None
+    life_stage: list[str] | None = None
+    age: list[str] | None = None
+    age_units: list[str] | None = None
     biosample_treatments: list[str] | None = None
     biosample_treatments_amount: list[str] | None = None
     biosample_treatments_duration: list[str] | None = None
@@ -134,6 +137,9 @@ class EnrichedEncodeCollectionInput:
     platform: list[str] | None = None
     dbxrefs: list[str] | None = None
     rbns_protein_concentration: list[str] | None = None
+    annotation_type: list[str] | None = None
+    software_used: list[str] | None = None
+    encyclopedia_version: list[str] | None = None
 
 
 @strawberry.input
@@ -216,6 +222,8 @@ class EnrichedFourdnFileInput:
 @strawberry.input
 class EnrichedEncodeFileInput:
     assembly: list[str] | None = None
+    annotation_type: list[str] | None = None
+    organism: list[str] | None = None
     file_format_type: list[str] | None = None
     output_type: list[str] | None = None
     genome_annotation: list[str] | None = None

src/cfdb/api/gql/schema.py

@@ -78,6 +78,18 @@ def process_errors(
         "output_type",
         "status",
         "data_access_level",
+        # ENCODE annotation facets. All three are small closed vocabularies
+        # -- a handful of annotation types, organisms and assemblies --
+        # which is what this allowlist is for; unlike accession_id,
+        # deliberately absent because enumerating it would return the whole
+        # corpus. Without annotation_type here a client can discover which
+        # assemblies exist but not which annotation types do, which is the
+        # facet the annotation ingest exists to expose (issue #94).
+        # ``assembly`` mirrors the core ``genome_assembly`` above; both are
+        # written by the ENCODE ingest and either may be filtered on.
+        "extra.encode.annotation_type",
+        "extra.encode.organism",
+        "extra.encode.assembly",
     }
 )
 

src/cfdb/indexes.py

@@ -173,10 +173,30 @@ def materialized_files_index_specs() -> list[IndexSpec]:
     collection on a public endpoint. Ensuring just the accession keys there
     costs nothing when the materializer has already created them: identical
     keys derive identical default names, so the create is a no-op.
+
+    The ``extra.encode`` keys are here for the same reason and are likewise
+    not the materializer's: they are written only by the ENCODE ingest, and
+    ``annotation_type`` in particular is the filter the whole annotation
+    corpus is meant to be reached through (issue #94). Without an index that
+    is a full scan of ~300k documents on an unauthenticated endpoint.
+
+    ``genome_assembly`` is the exception to "not the materializer's": it is
+    a core field every DCC populates and the materializer does index it.
+    It is repeated here because an ENCODE-only database never runs the
+    materializer, and assembly is the other half of the same acceptance
+    criterion as organism -- narrowing to GRCh38 is the first move a client
+    makes. The `extra.encode.assembly` mirror is indexed alongside it since
+    the ENCODE ingest writes both and either may be filtered on. Repeating
+    a key the materializer also creates costs nothing: identical keys derive
+    identical default names, so the create is a no-op.
     """
     return [
         IndexSpec("files", [("accession_id", 1)]),
         IndexSpec("files", [("collections.accession_id", 1)]),
+        IndexSpec("files", [("extra.encode.annotation_type", 1)]),
+        IndexSpec("files", [("extra.encode.organism", 1)]),
+        IndexSpec("files", [("extra.encode.assembly", 1)]),
+        IndexSpec("files", [("genome_assembly", 1)]),
     ]
 
 

src/cfdb/models.py

@@ -130,9 +130,33 @@ class EnrichedFourdnFile(BaseModel):
 
 
 class EnrichedEncodeFile(BaseModel):
-    """ENCODE file-level metadata from metadata TSV."""
+    """ENCODE file-level metadata from metadata TSV.
+
+    Populated from either the Experiment or the Annotation metadata TSV.
+    The two share only part of their column sets, so a field sourced from a
+    column the other TSV does not publish is None on those documents rather
+    than derived from something else -- see the annotation mapping in
+    :mod:`cfdb.services.encode`.
+
+    Attributes:
+        annotation_type:
+            The kind of annotation this file belongs to (e.g. "candidate
+            Cis-Regulatory Elements"). The field that gives an annotation
+            file its meaning, and the one a client filters on to ask for
+            cCREs without string-matching filenames. None on experiment
+            files, whose TSV has no such column.
+
+        organism:
+            Scientific name of the source organism (e.g. "Homo sapiens").
+            Also reaches ``subjects[].taxonomy`` on experiment files, but
+            annotation rows name no donor and so build no subject -- this is
+            the only place the organism of a multi-organism annotation
+            result set is queryable.
+    """
 
     assembly: Optional[str] = None
+    annotation_type: Optional[str] = None
+    organism: Optional[str] = None
     file_format_type: Optional[str] = None
     output_type: Optional[str] = None
     genome_annotation: Optional[str] = None
@@ -227,12 +251,37 @@ def empty_string_to_none(cls, v):
 
 
 class EnrichedEncodeCollection(BaseModel):
-    """ENCODE experiment-level metadata from metadata TSV."""
+    """ENCODE dataset-level metadata from metadata TSV.
+
+    A "dataset" is an Experiment or an Annotation depending on which TSV the
+    document came from; ``platform`` and ``rbns_protein_concentration`` are
+    experiment-only, ``annotation_type``, ``software_used`` and
+    ``encyclopedia_version`` annotation-only.
+
+    Attributes:
+        annotation_type:
+            The annotation kind this dataset publishes, mirroring
+            :attr:`EnrichedEncodeFile.annotation_type`. Held on the dataset
+            as well as the file because that is the entity the property
+            actually describes.
+
+        software_used:
+            Software that produced the annotation, as a comma-separated
+            list (e.g. "ABC-Enhancer-Gene-Prediction, Distal regulation
+            ENCODE-rE2G"). Blank for some annotation types.
+
+        encyclopedia_version:
+            The ENCODE Encyclopedia release the annotation belongs to (e.g.
+            "ENCODE v4", "ENCODE v3, current").
+    """
 
     project: Optional[str] = None
     platform: Optional[str] = None
     dbxrefs: Optional[str] = None
     rbns_protein_concentration: Optional[str] = None
+    annotation_type: Optional[str] = None
+    software_used: Optional[str] = None
+    encyclopedia_version: Optional[str] = None
 
 
 class EnrichedFourdnCollection(BaseModel):
@@ -298,9 +347,31 @@ class EnrichedEncodeBiosample(BaseModel):
     ENCODE biosample-level metadata from metadata TSV.
 
     Contains biosample classification, treatment, and library information.
+
+    Attributes:
+        life_stage:
+            Developmental stage of the source organism when the biosample
+            was taken (e.g. "embryonic", "adult", "young adult", "unknown").
+
+        age:
+            Age of the source organism at sampling, in ``age_units``, kept
+            as the upstream string. NOT parsed to a number: the released
+            annotation corpus contains "2-4" and "unknown" alongside plain
+            decimals, and it distinguishes "10.5" from "10.50". This is also
+            why the value does not go to :attr:`Subject.age_at_sampling`,
+            which is a float in years -- that field could represent neither
+            the ranges nor the sentinels, and an annotation row names no
+            donor to build a Subject from in the first place.
+
+        age_units:
+            Unit for :attr:`age` ("year", "month", "week", "day"). Blank
+            when ``age`` is absent or a sentinel.
     """
 
     biosample_type: Optional[str] = None
+    life_stage: Optional[str] = None
+    age: Optional[str] = None
+    age_units: Optional[str] = None
     biosample_treatments: Optional[str] = None
     biosample_treatments_amount: Optional[str] = None
     biosample_treatments_duration: Optional[str] = None
@@ -393,7 +464,9 @@ class FileMetadataModel(BaseModel):
 
         file_format:
             An EDAM CV term identifying the digital format of this file
-            (e.g., TSV or FASTQ). If compressed, this is the uncompressed format.
+            (e.g., TSV or FASTQ), or a ``cfdb:``-prefixed token where EDAM
+            has no term for it; see :class:`FileFormat`. If compressed,
+            this is the uncompressed format.
 
         compression_format:
             An EDAM CV term ID identifying compression that is extrinsic to
@@ -571,20 +644,34 @@ class AssayType(BaseModel):
 
 class FileFormat(BaseModel):
     """
-    An EDAM CV 'format:' term.
+    An EDAM CV 'format:' term, or a term minted here where EDAM has none.
 
     Describes the digital format of C2M2 files.
 
+    Most ids are EDAM CV ``format:`` terms and resolve at edamontology.org.
+    A few formats EDAM does not cover -- ``bedpe`` and ``bigInteract`` at
+    the time of writing -- carry a token minted here instead, prefixed
+    ``cfdb:`` (:data:`cfdb.services.ontology_mappings.MINTED_FORMAT_PREFIX`,
+    the stable discriminator to test against). Those ids resolve nowhere:
+    aliasing such a format onto the nearest EDAM term would make it
+    indistinguishable from the format it was aliased to, and the processor
+    claiming that term would pick it up and mangle it. A consumer resolving
+    ids against EDAM must therefore skip the minted prefix rather than
+    assume every id is resolvable.
+
     Attributes:
         id:
-            An EDAM CV format term identifier.
+            An EDAM CV format term identifier, or a ``cfdb:``-prefixed
+            token minted where EDAM has no term for the format.
 
         name:
-            A short, human-readable, machine-read-friendly label for this EDAM
-            format term.
+            A short, human-readable, machine-read-friendly label for this
+            format term. Distinct per format even where the id is minted:
+            workflow processor routing keys on this field, so two formats
+            sharing a name share a pipeline.
 
         description:
-            A human-readable description of this EDAM format term.
+            A human-readable description of this format term.
     """
 
     id: str = str()