From efaa0c6a500f5350dec39ae8f8a2c21411a101f7 Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 29 May 2026 13:26:08 +0200 Subject: [PATCH 001/102] docs(ops): document constructor params on node-facing sources/ops Add Google-style Args: to HuggingFaceSource, Flux, JointFlux, FilterOp, WrappedOp, Tee, StandardizeOp, ThresholdOp, ConnectedComponentsOp, ToTensorOp, resolve_expression. These surface as FluxStudio widget tooltips and navigaitor form-spec field descriptions via confluid.parse_param_docs. New tests/test_node_docs.py pins full coverage; AGENTS mandate added. --- AGENTS.md | 2 ++ dataflux/core.py | 55 ++++++++++++++++++++++++++++++++++++++--- dataflux/ops/numpy.py | 17 +++++++++++++ dataflux/ops/tee.py | 7 +++++- dataflux/ops/torch.py | 7 ++++++ dataflux/sources.py | 9 +++++++ tests/test_node_docs.py | 52 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 tests/test_node_docs.py diff --git a/AGENTS.md b/AGENTS.md index 6b9f4c1..ca1e339 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,9 @@ - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. +- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. +- **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them: `Flux` / `JointFlux` are `category="dataset"` (generic, task-agnostic engines); `FilterOp` / `WrappedOp` are `category="op"`. Preserve these tags when adding or renaming the classes — `tests/test_categories.py` pins them, and dropping a tag silently empties the corresponding picker. - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. diff --git a/dataflux/core.py b/dataflux/core.py index 971fb9c..c17faf7 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -3,7 +3,20 @@ import multiprocessing from contextlib import nullcontext from functools import lru_cache -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Collection, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + Union, + cast, +) import torch.utils.data from confluid import configurable @@ -93,7 +106,11 @@ def _check_ops_materialized(ops: List[Any]) -> None: @configurable(category="op") class FilterOp: - """Configurable filter operation.""" + """Configurable filter operation. + + Args: + p: Predicate ``Sample -> bool``; the sample passes through when it returns ``True``, else is dropped. + """ def __init__(self, p: Callable[[Sample], bool]): self.p = p @@ -104,7 +121,13 @@ def __call__(self, s: Sample) -> Optional[Sample]: @configurable(category="op") class WrappedOp: - """Configurable transformation wrapper with smart mapping.""" + """Configurable transformation wrapper with smart mapping. + + Args: + f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). + s: Which Sample slot to transform — ``"input"``, ``"target"``, or ``"all"`` (the whole Sample). + kw: Extra keyword arguments forwarded to the wrapped callable on every call. + """ def __init__(self, f: Union[str, Callable], s: str, kw: Dict[str, Any]): from dataflux.discovery import get_callable_path @@ -154,6 +177,9 @@ class JointFlux: """ Aggregates multiple Flux streams into a single joint stream. Each sub-flux maintains its own unique transformation chain. + + Args: + fluxes: The Flux streams to concatenate; iteration walks them in order and length is their sum. """ def __init__(self, fluxes: List["Flux"]) -> None: @@ -188,6 +214,11 @@ class Flux(torch.utils.data.Dataset[Sample]): serialization (e.g. shared-source dataset-split patterns in navigaitor) works correctly — see ``confluid/pydantic_export.py:_ITER_TYPES_AS_ANY``. + + Args: + source: Any iterable or indexable dataset (duck-typed) to wrap; ``None`` yields an empty stream. + ops: Ordered callables ``Sample -> Optional[Sample]`` applied lazily on access (``None`` = no ops). + chunk_size: Parallel-processing chunk size; ``0`` (the default) processes sequentially. """ def __init__( @@ -435,3 +466,21 @@ def _iter_parallel(self) -> Iterator[Sample]: def collect(self) -> List[Sample]: """Materialize the full flux into a list.""" return list(self) + + def project(self, fields: Collection[str]) -> Iterator[Sample]: + """Yield pipeline-output Samples carrying only ``fields`` (the projection primitive). + + Implements :class:`dataflux.projection.SupportsProjection`. Flux must run + its op chain to produce each Sample (an op may consume the input), so this + is the generic "iterate, then drop unrequested fields" form — it cannot + skip input construction the way a leaf source (e.g. an image dataset that + reads only the label column) can. Lazy: a generator. ``fields`` is a + subset of ``{"input", "target", "metadata"}``. + """ + want = frozenset(fields) + for sample in self: + yield Sample( + input=sample.input if "input" in want else None, + target=sample.target if "target" in want else None, + metadata=sample.metadata if "metadata" in want else {}, + ) diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index fd37034..a22842c 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -25,6 +25,11 @@ def resolve_expression(value: str, sample: Sample) -> str: Returns the substituted string verbatim — the caller is responsible for any further casting (e.g. ``float(...)`` for a numeric expression). + Args: + value: Expression string with ``{meta_key}`` and/or ``$ENV_VAR`` placeholders + (a plain literal returns unchanged). + sample: The Sample whose ``metadata`` supplies the ``{key}`` substitutions. + Examples: ``"5.5"`` → ``"5.5"`` (no substitution) ``"{reference_snr_level}"`` → ``str(metadata["reference_snr_level"])`` @@ -64,6 +69,10 @@ class StandardizeOp: per-channel values that broadcasts over [C, H, W] format. Handles PIL images by converting to ndarray first. + + Args: + mean: Mean to subtract — a single float (uniform) or a per-channel sequence broadcasting over [C, H, W]. + std: Standard deviation to divide by — a single float (uniform) or a per-channel sequence. """ ACCEPTS = SampleType(input=_NUMERIC_OR_PIL) @@ -254,6 +263,10 @@ class ThresholdOp: * ``"$REF_SNR"`` / ``"-$REF_SNR"`` — environment-variable lookup Records the resolved threshold under ``metadata["threshold"]`` for traceability. + + Args: + value: Threshold as a numeric literal or a string expression resolved against + ``sample.metadata`` / ``os.environ``. """ ACCEPTS = SampleType(input=_NDARRAY) @@ -300,6 +313,10 @@ class ConnectedComponentsOp: components merge. Requires ``scipy`` (install via ``pip install data-flux[vision]``). + + Args: + min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). + connectivity: Pixel neighborhood — ``4`` (orthogonal only) or ``8`` (orthogonal + diagonal). """ ACCEPTS = SampleType(input=ArrayType(ndim=2, dtype="bool", frameworks={"numpy"})) diff --git a/dataflux/ops/tee.py b/dataflux/ops/tee.py index 9a70fed..6476944 100644 --- a/dataflux/ops/tee.py +++ b/dataflux/ops/tee.py @@ -19,7 +19,12 @@ @configurable class Tee: - """Run N op-list branches sequentially on the same sample / metadata.""" + """Run N op-list branches sequentially on the same sample / metadata. + + Args: + branches: A list of op-lists; each inner list is a chain of callables + ``Sample -> Optional[Sample]`` run in order. + """ def __init__(self, branches: List[List[Any]]) -> None: self.branches = [list(b) for b in branches] diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index c4e7443..50fed5a 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -15,6 +15,9 @@ class ToTensorOp: """ Converts input (PIL Image, NumPy array, etc.) to a Torch Tensor. + + Args: + normalize: When ``True``, scale integer pixel inputs into the ``[0, 1]`` float range during conversion. """ ACCEPTS = SampleType(input=UnionType((PythonType("PIL.Image.Image"), ArrayType(frameworks={"numpy"})))) @@ -112,6 +115,10 @@ class StandardizeOp: mean/std can be a single float (applied uniformly) or a sequence of per-channel values that broadcasts over [C, H, W] format. + + Args: + mean: Mean to subtract — a single float (uniform) or a per-channel sequence broadcasting over [C, H, W]. + std: Standard deviation to divide by — a single float (uniform) or a per-channel sequence. """ ACCEPTS = SampleType(input=_TORCH) diff --git a/dataflux/sources.py b/dataflux/sources.py index 1dff834..ece29b9 100644 --- a/dataflux/sources.py +++ b/dataflux/sources.py @@ -14,6 +14,15 @@ class HuggingFaceSource: """ DataFlux Source for Hugging Face Datasets. Configurable mapping of dataset features to DataFlux Sample triplets. + + Args: + path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. + split: HF split name (``train`` / ``validation`` / ``test`` / etc.). + input_feature: Dataset feature column to map onto ``Sample.input``. + target_feature: Dataset feature column to map onto ``Sample.target``. + metadata_features: Feature columns to preserve on ``Sample.metadata`` (``None`` = none). + count: Optional cap on the number of samples yielded (useful for fast smoke runs). + name: Optional HF subset/config name (e.g. for multi-config datasets). """ def __init__( diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py new file mode 100644 index 0000000..bc37326 --- /dev/null +++ b/tests/test_node_docs.py @@ -0,0 +1,52 @@ +"""Guard: every node-facing dataflux Source/Op documents all its constructor params. + +These classes surface in FluxStudio (as widget tooltips) and navigaitor (as +pydantic ``Field(description=...)`` in the form-spec) purely from their docstring +``Args:`` block — see ``confluid.parse_param_docs``. A param that loses its doc +silently loses its tooltip/description, so this pins the coverage. +""" + +import inspect +from typing import List + +import pytest +from confluid import parse_param_docs # type: ignore[import-not-found] + +from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from dataflux.ops.numpy import ConnectedComponentsOp, StandardizeOp, ThresholdOp +from dataflux.ops.tee import Tee +from dataflux.ops.torch import StandardizeOp as TorchStandardizeOp +from dataflux.ops.torch import ToTensorOp +from dataflux.sources import HuggingFaceSource + +_NODE_CLASSES = [ + HuggingFaceSource, + Flux, + JointFlux, + FilterOp, + WrappedOp, + Tee, + StandardizeOp, + ThresholdOp, + ConnectedComponentsOp, + ToTensorOp, + TorchStandardizeOp, +] + + +def _constructor_params(cls: type) -> List[str]: + # signature(cls) is the constructor signature (no ``self``), and it survives + # confluid's @configurable __init__ wrapping (verified against real classes). + sig = inspect.signature(cls) + return [ + name + for name, p in sig.parameters.items() + if p.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) + ] + + +@pytest.mark.parametrize("cls", _NODE_CLASSES, ids=lambda c: c.__name__) +def test_all_constructor_params_documented(cls: type) -> None: + docs = parse_param_docs(cls) + missing = [p for p in _constructor_params(cls) if not docs.get(p)] + assert not missing, f"{cls.__name__} is missing Args docs for: {missing}" From 703933365c568e3dc33674d868989cb6c4bfcca7 Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 29 May 2026 15:19:34 +0200 Subject: [PATCH 002/102] feat: Flux.from_ops_yaml + role-split discovery categories + field projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Flux.from_ops_yaml(path, source): attach a FluxStudio-exported ops-only Confluid YAML ({ops: [!class:...()]}) to any source. Routes the op-list through confluid.materialize before attaching — confluid.load leaves !class: markers nested under a mapping key deferred, which Flux rejects at iteration by design. - Discovery categories split by ROLE: Flux/JointFlux/DatasetSplit -> 'engine', HuggingFaceSource -> 'source' (replaces the inverted 'dataset'=engine scheme); test_categories + AGENTS updated. - Field projection surface exported from package root: SupportsProjection / project / iter_inputs / iter_targets / num_classes. --- AGENTS.md | 6 +- README.md | 37 +++++++ dataflux/__init__.py | 6 ++ dataflux/core.py | 27 +++++- dataflux/projection.py | 144 +++++++++++++++++++++++++++ dataflux/sources.py | 4 +- tests/test_categories.py | 20 +++- tests/test_from_ops_yaml.py | 46 +++++++++ tests/test_projection.py | 188 ++++++++++++++++++++++++++++++++++++ 9 files changed, 468 insertions(+), 10 deletions(-) create mode 100644 dataflux/projection.py create mode 100644 tests/test_from_ops_yaml.py create mode 100644 tests/test_projection.py diff --git a/AGENTS.md b/AGENTS.md index ca1e339..ff08e2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,11 @@ - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. - **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). -- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them: `Flux` / `JointFlux` are `category="dataset"` (generic, task-agnostic engines); `FilterOp` / `WrappedOp` are `category="op"`. Preserve these tags when adding or renaming the classes — `tests/test_categories.py` pins them, and dropping a tag silently empties the corresponding picker. +- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": + - `category="engine"` — the generic, task-agnostic **engines/containers** that compose sources + ops: `Flux`, `JointFlux`, `DatasetSplit`. These are NOT datasets — a `Flux` *happens* to implement the `Dataset` interface but conceptually it's the pipeline engine. FluxStudio hides them (you wire `Source → Op` and the canvas builds the `Flux` implicitly); navigaitor does NOT offer them in dataset slots. + - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`). These ARE datasets/sources — they appear as FluxStudio source nodes and as navigaitor dataset-slot options. + - `category="op"` — pipeline ops: `FilterOp` / `WrappedOp`. + Rationale (history): these were ALL once `category="dataset"` (engine) vs uncategorised (sources) — backwards and confusing (`Flux` the engine was the "dataset", while `HuggingFaceSource` the actual dataset had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and dropping/renaming a tag silently empties the corresponding picker. - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. diff --git a/README.md b/README.md index 0f5a02e..e8be962 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,30 @@ class StandardizeOp: Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime against a concrete inferred type); `compatible(consumer, producer)` is permissive (used at edit time — `Any`/unknown on either side passes). A `Sample`'s own type comes from `sample.describe()` — it returns a type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict) + `__spec__` (sidecar refinements), or infers one from the live data; attach a stored type with `sample.with_type(SampleType(...))`. +## 🔎 Field Projection & Class Counting + +Walking a source for a single field (the classic case: counting classes from +*targets*) shouldn't pay to build the fields you don't need. `dataflux.projection` +adds an opt-in protocol plus lazy helpers: + +```python +from dataflux import project, iter_targets, num_classes + +# A source MAY implement SupportsProjection (`project(fields)`) to skip building +# unrequested fields — e.g. an image dataset reads only the label column for a +# target-only walk, never decoding an image. +for sample in project(my_source, ("target",)): + ... # sample.input is None; sample.target populated + +labels = list(iter_targets(my_source)) # lazy +n = num_classes(my_source) # max(class_id) + 1 — always walks +``` + +Sources that don't implement `SupportsProjection` still work via a correct +full-iteration fallback (just without the skip-decode speedup). `num_classes` is +a free function, not a `Flux` method: integer class-id semantics are +classification-specific, so the task-agnostic engine doesn't advertise it. + ## 📦 Storage Integration DataFlux makes it easy to move data between different formats: @@ -143,6 +167,19 @@ Flux.from_source(HDF5Source("input.h5")) \ > **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key, so a single `HuggingFaceSource` is loaded once and shared by both splits. Use `!clone:` when you want an independent deep copy instead. +## 🔁 Reattach an ops-only YAML (`Flux.from_ops_yaml`) + +A `{ops: [!class:…()]}` document — e.g. one exported from a FluxStudio canvas (`fluxstudio export …`) — can be attached to any source: + +```python +from dataflux import Flux +from dataflux.sources import HuggingFaceSource + +flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) +``` + +The helper **materializes** the deferred `!class:` markers before attaching (via `confluid.materialize`) — necessary because `confluid.load` leaves markers nested under a mapping key deferred, and a `Flux` rejects deferred markers at iteration by design. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. + ## 🔗 Paired Join (Binary ↔ Annotations) `PairedSource` joins a primary `DataSource` (e.g. raw binary samples) with a secondary mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern. Three join policies cover the scenarios we actually see in ML research: diff --git a/dataflux/__init__.py b/dataflux/__init__.py index 3a7c6de..ca29bc8 100644 --- a/dataflux/__init__.py +++ b/dataflux/__init__.py @@ -5,6 +5,7 @@ from dataflux.core import Flux, JointFlux, WrappedOp from dataflux.ops import RescaleOp, StandardizeOp, ToTensorOp from dataflux.paired import PairedSource +from dataflux.projection import SupportsProjection, iter_inputs, iter_targets, num_classes, project from dataflux.sample import Sample from dataflux.sources import DatasetSplit, HuggingFaceSource from dataflux.typespec import ( @@ -37,10 +38,15 @@ "Sample", "SampleType", "StandardizeOp", + "SupportsProjection", "ToTensorOp", "UnionType", "WrappedOp", "infer_sample_type", "infer_type", + "iter_inputs", + "iter_targets", + "num_classes", + "project", "typed", ] diff --git a/dataflux/core.py b/dataflux/core.py index c17faf7..3bd67ec 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -20,6 +20,8 @@ import torch.utils.data from confluid import configurable +from confluid import load as _confluid_load +from confluid import materialize as _confluid_materialize from confluid.fluid import Fluid as _ConfluidFluid from logflow import get_logger @@ -172,7 +174,7 @@ def _worker_task(sample: Sample, ops: List[Any]) -> Optional[Sample]: return current_sample -@configurable(category="dataset") +@configurable(category="engine") class JointFlux: """ Aggregates multiple Flux streams into a single joint stream. @@ -195,7 +197,7 @@ def __len__(self) -> int: return sum(len(f) for f in self.fluxes) -@configurable(category="dataset") +@configurable(category="engine") class Flux(torch.utils.data.Dataset[Sample]): """ The primary stream engine for DataFlux. @@ -258,6 +260,27 @@ def joint(cls, fluxes: List["Flux"]) -> "Flux": """Create a new Flux that aggregates multiple other Flux streams.""" return cls(source=JointFlux(fluxes)) + @classmethod + def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": + """Attach an ops-only Confluid YAML (e.g. exported from FluxStudio) to ``source``. + + ``path`` is the ``{ops: [!class:...()]}`` document produced by + :func:`fluxstudio.export.export_ops_yaml` (the ``fluxstudio export`` CLI or the + canvas Export button). It also accepts an inline YAML string (``confluid.load`` + handles both). + + The op markers are **materialized to live callables** before being attached: + ``confluid.load`` leaves ``!class:`` markers nested under a mapping key deferred + (its final flow pass doesn't descend dict→list), so a plain ``load(path)["ops"]`` + would hand :class:`Flux` deferred ``Instance`` markers — which iteration rejects by + design (see :meth:`_guard_live_source` / ``_check_ops_materialized``). Routing through + :func:`confluid.materialize` flows the top-level list of markers into live ops. + """ + loaded = _confluid_load(path) + raw_ops = loaded.get("ops", []) if isinstance(loaded, dict) else [] + ops = list(_confluid_materialize(raw_ops)) + return cls(source=source, ops=ops) + def __len__(self) -> int: """Return the length of the underlying source if available. diff --git a/dataflux/projection.py b/dataflux/projection.py new file mode 100644 index 0000000..0110677 --- /dev/null +++ b/dataflux/projection.py @@ -0,0 +1,144 @@ +"""Field projection for DataFlux sources — read only the input or only the target. + +Walking a source for a single field (the canonical case: counting classes from +*targets*) should not pay for constructing the fields you don't need — e.g. +decoding image inputs you are about to throw away. This module adds an **opt-in** +projection protocol plus walk helpers that any consumer can use against any +source, with a correct (if unoptimized) fallback for sources that don't +implement the protocol. + +The primitive is deliberately general (``input`` / ``target`` / ``metadata`` +selection); :func:`num_classes` is one helper built on top of it. + +Design notes +------------ +* :class:`SupportsProjection` is a ``Protocol`` (never a base class), so it + composes with the DataFlux **Functional Purity** mandate — a source opts in by + *defining* ``project``, not by inheriting. +* Every public function is a lazy generator (**Lazy Evaluation** mandate) — + nothing materializes the whole source. +* :func:`num_classes` (integer class-id semantics) is a free function, *not* a + method on the generic :class:`~dataflux.core.Flux` engine — counting classes is + a classification concern, and bolting it onto the task-agnostic engine would + make every ``Flux`` look classification-capable to duck-typed consumers. +""" + +from typing import Any, Collection, Iterator, Protocol, runtime_checkable + +from dataflux.sample import Sample + +INPUT = "input" +TARGET = "target" +METADATA = "metadata" +_FIELDS = (INPUT, TARGET, METADATA) + + +@runtime_checkable +class SupportsProjection(Protocol): + """A source that can yield partial :class:`~dataflux.sample.Sample` records. + + Implementers SHOULD avoid building unrequested fields — e.g. skip decoding the + input image when only ``target`` is asked for; that efficiency is the whole + point of the protocol. ``fields`` is a subset of + ``{"input", "target", "metadata"}``; unrequested fields come back as ``None`` + (``{}`` for ``metadata``). + """ + + def project(self, fields: Collection[str]) -> Iterator[Sample]: ... + + +def project(source: Any, fields: Collection[str]) -> Iterator[Sample]: + """Yield :class:`Sample` records from ``source`` carrying only ``fields``. + + Uses the source's own ``project`` when it implements + :class:`SupportsProjection` (the efficient path that skips building + unrequested fields); otherwise falls back to a full iteration that builds + every field and nulls the unrequested ones — always correct, just not faster. + Lazy: a generator that never materializes the source. + """ + want = frozenset(fields) + unknown = want - frozenset(_FIELDS) + if unknown: + raise ValueError(f"Unknown projection field(s): {sorted(unknown)}; valid fields are {list(_FIELDS)}.") + if isinstance(source, SupportsProjection): + yield from source.project(want) + return + for raw in source: + s = Sample.from_any(raw) + yield Sample( + input=s.input if INPUT in want else None, + target=s.target if TARGET in want else None, + metadata=s.metadata if METADATA in want else {}, + ) + + +def iter_inputs(source: Any) -> Iterator[Any]: + """Lazily yield each sample's ``input`` (skipping target construction when supported).""" + for s in project(source, (INPUT,)): + yield s.input + + +def iter_targets(source: Any) -> Iterator[Any]: + """Lazily yield each sample's ``target`` (skipping input construction when supported).""" + for s in project(source, (TARGET,)): + yield s.target + + +def _to_int(value: Any) -> int: + """Coerce a single target into a Python ``int`` class id. + + Handles plain ``int``, numpy scalars, and 0-d / single-element torch tensors + (via ``.item()``). Rejects ``bool`` (an ``int`` subclass — accepting it would + silently turn a boolean target into class 0/1) and anything that isn't a + scalar so callers fail loudly instead of miscounting. + """ + if isinstance(value, bool): + raise TypeError(f"target {value!r} is a bool, not a class id") + if isinstance(value, int): + return value + item = getattr(value, "item", None) + if callable(item): + try: + result = item() + except Exception as exc: # pragma: no cover - exotic array/tensor types + raise TypeError(f"could not read a scalar class id from target {value!r}: {exc}") from exc + if isinstance(result, bool): + raise TypeError(f"target {value!r} resolved to a bool, not a class id") + if isinstance(result, int): + return result + if isinstance(result, float) and result.is_integer(): + return int(result) + raise TypeError(f"target {value!r} did not yield an integer class id (got {result!r})") + raise TypeError(f"target {value!r} of type {type(value).__name__} is not a scalar class id") + + +def num_classes(source: Any) -> int: + """Derive the number of classes by walking **every** target in ``source``. + + Always walks the full target stream (target-only, so inputs are never + constructed when the source supports projection) and returns + ``max(class_id) + 1`` — the classifier-head size needed to cover the largest + label, robust to a class id that happens not to appear in this split. Raises + ``ValueError`` if the source yields no targets (or a ``None`` target). + + This is the engine behind a dataset's lazy ``num_classes()`` method. + """ + highest = -1 + for target in iter_targets(source): + if target is None: + raise ValueError("num_classes: encountered a sample with no target — cannot derive a class count.") + cid = _to_int(target) + if cid > highest: + highest = cid + if highest < 0: + raise ValueError("num_classes: source yielded no targets — cannot derive a class count.") + return highest + 1 + + +__all__ = [ + "SupportsProjection", + "project", + "iter_inputs", + "iter_targets", + "num_classes", +] diff --git a/dataflux/sources.py b/dataflux/sources.py index ece29b9..6c0f04a 100644 --- a/dataflux/sources.py +++ b/dataflux/sources.py @@ -9,7 +9,7 @@ logger = get_logger(__name__) -@configurable +@configurable(category="source") class HuggingFaceSource: """ DataFlux Source for Hugging Face Datasets. @@ -90,7 +90,7 @@ def __len__(self) -> int: return self.count or len(self._dataset) -@configurable +@configurable(category="engine") class DatasetSplit: """ Selects a subset view of an indexable source (e.g. ``HuggingFaceSource``). diff --git a/tests/test_categories.py b/tests/test_categories.py index 4197e8f..97a05ee 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,12 +10,21 @@ from confluid.registry import get_registry from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from dataflux.sources import DatasetSplit, HuggingFaceSource -def test_dataset_classes_tagged() -> None: - """``Flux`` / ``JointFlux`` are the generic (task-agnostic) dataset engines.""" - assert Flux.__confluid_category__ == "dataset" - assert JointFlux.__confluid_category__ == "dataset" +def test_engine_classes_tagged() -> None: + """``Flux`` / ``JointFlux`` / ``DatasetSplit`` are the generic, task-agnostic *engines* + (containers that compose sources + ops), NOT datasets. They are excluded from FluxStudio's + node palette (you wire Source -> Op instead) and are not offered in navigaitor dataset slots.""" + assert Flux.__confluid_category__ == "engine" + assert JointFlux.__confluid_category__ == "engine" + assert DatasetSplit.__confluid_category__ == "engine" + + +def test_source_classes_tagged() -> None: + """``HuggingFaceSource`` is a concrete data *source* (it loads a dataset).""" + assert HuggingFaceSource.__confluid_category__ == "source" def test_op_classes_tagged() -> None: @@ -31,5 +40,6 @@ def test_categories_enumerable_via_registry() -> None: not just the class attribute — has to carry the tag. """ registry = get_registry() - assert {"Flux", "JointFlux"} <= registry.list_classes(category="dataset") + assert {"Flux", "JointFlux", "DatasetSplit"} <= registry.list_classes(category="engine") + assert {"HuggingFaceSource"} <= registry.list_classes(category="source") assert {"FilterOp", "WrappedOp"} <= registry.list_classes(category="op") diff --git a/tests/test_from_ops_yaml.py b/tests/test_from_ops_yaml.py new file mode 100644 index 0000000..0dfb547 --- /dev/null +++ b/tests/test_from_ops_yaml.py @@ -0,0 +1,46 @@ +"""Tests for ``Flux.from_ops_yaml`` — attach an exported ops-only YAML to a source. + +The YAML shape is what ``fluxstudio.export`` emits: a ``{ops: [!class:...()]}`` document. +The key behaviour under test is that the helper **materializes** the deferred ``!class:`` +markers (which ``confluid.load`` leaves un-flowed when nested under a mapping key) before +attaching them, so iteration sees live callables rather than ``Instance`` markers. +""" + +from pathlib import Path + +import torch + +from dataflux import Flux, Sample +from dataflux.ops.torch import RescaleOp # noqa: F401 - import registers the @configurable for !class: resolution + +OPS_YAML = """ops: +- !class:dataflux.ops.torch.RescaleOp() + in_min: 0.0 + in_max: 255.0 +- !class:dataflux.ops.torch.RescaleOp() + in_min: 0.0 + in_max: 1.0 + out_max: 10.0 +""" + + +def test_from_ops_yaml_materializes_and_attaches(tmp_path: Path) -> None: + path = tmp_path / "ops.yaml" + path.write_text(OPS_YAML) + src = [Sample(input=torch.tensor([0.0, 255.0]), target=None, metadata={})] + + flux = Flux.from_ops_yaml(str(path), source=src) + + # Materialized to live ops — NOT deferred Instance markers (which iteration would reject). + assert [type(o).__name__ for o in flux.ops] == ["RescaleOp", "RescaleOp"] + out = list(flux)[0].input + assert torch.allclose(out, torch.tensor([0.0, 10.0])) + + +def test_from_ops_yaml_without_ops_key_is_empty(tmp_path: Path) -> None: + path = tmp_path / "empty.yaml" + path.write_text("other: 1\n") + + flux = Flux.from_ops_yaml(str(path), source=[]) + + assert flux.ops == [] diff --git a/tests/test_projection.py b/tests/test_projection.py new file mode 100644 index 0000000..67af618 --- /dev/null +++ b/tests/test_projection.py @@ -0,0 +1,188 @@ +"""Tests for the field-projection primitive and the num_classes helper.""" + +import itertools +from typing import Collection, Iterator, List + +import numpy as np +import pytest +import torch + +from dataflux.core import Flux +from dataflux.projection import SupportsProjection, _to_int, iter_inputs, iter_targets, num_classes, project +from dataflux.sample import Sample + +# --------------------------------------------------------------------------- # +# Fallback path (sources that do NOT implement SupportsProjection) +# --------------------------------------------------------------------------- # + + +def _plain_source() -> List[Sample]: + return [ + Sample(input=np.array([1, 2]), target=0, metadata={"i": 0}), + Sample(input=np.array([3, 4]), target=2, metadata={"i": 1}), + ] + + +def test_project_fallback_nulls_unrequested_fields() -> None: + out = list(project(_plain_source(), ("target",))) + assert [s.target for s in out] == [0, 2] + assert all(s.input is None for s in out) + assert all(s.metadata == {} for s in out) + + +def test_project_fallback_input_only() -> None: + out = list(project(_plain_source(), ("input",))) + assert all(s.target is None for s in out) + assert np.array_equal(out[0].input, np.array([1, 2])) + + +def test_project_rejects_unknown_field() -> None: + with pytest.raises(ValueError, match="Unknown projection field"): + list(project(_plain_source(), ("bogus",))) + + +def test_iter_helpers() -> None: + assert list(iter_targets(_plain_source())) == [0, 2] + inputs = list(iter_inputs(_plain_source())) + assert np.array_equal(inputs[1], np.array([3, 4])) + + +# --------------------------------------------------------------------------- # +# Efficient path (sources that DO implement SupportsProjection) +# --------------------------------------------------------------------------- # + + +class _ProjectableSource: + """A source that records which fields were requested and only builds those. + + Building the input increments ``input_builds`` — the test asserts a + target-only walk never touches it, proving the efficient path skips + unrequested-field construction. + """ + + def __init__(self, targets: List[int]) -> None: + self.targets = targets + self.input_builds = 0 + + def _build_input(self, i: int) -> np.ndarray: + self.input_builds += 1 + return np.full((2, 2), i) + + def __len__(self) -> int: + return len(self.targets) + + def __iter__(self) -> Iterator[Sample]: + for i, t in enumerate(self.targets): + yield Sample(input=self._build_input(i), target=t, metadata={}) + + def project(self, fields: Collection[str]) -> Iterator[Sample]: + want = frozenset(fields) + for i, t in enumerate(self.targets): + yield Sample( + input=self._build_input(i) if "input" in want else None, + target=t if "target" in want else None, + metadata={} if "metadata" not in want else {"i": i}, + ) + + +def test_projectable_source_is_recognized_by_protocol() -> None: + src = _ProjectableSource([0, 1]) + assert isinstance(src, SupportsProjection) + + +def test_efficient_target_only_skips_input_construction() -> None: + src = _ProjectableSource([0, 1, 2]) + targets = list(iter_targets(src)) + assert targets == [0, 1, 2] + assert src.input_builds == 0 # never decoded an input + + +# --------------------------------------------------------------------------- # +# num_classes +# --------------------------------------------------------------------------- # + + +def test_num_classes_int_targets_is_max_plus_one() -> None: + # max id 2 even though id 1 absent from this "split" -> head size 3. + assert num_classes(_ProjectableSource([0, 2, 0, 2])) == 3 + + +def test_num_classes_walks_via_projection_without_inputs() -> None: + src = _ProjectableSource([0, 1, 2, 3]) + assert num_classes(src) == 4 + assert src.input_builds == 0 + + +def test_num_classes_torch_scalar_tensor_targets() -> None: + src = [Sample(input=None, target=torch.tensor(k, dtype=torch.int64)) for k in (0, 4, 1)] + assert num_classes(src) == 5 + + +def test_num_classes_numpy_scalar_targets() -> None: + src = [Sample(input=None, target=np.int64(k)) for k in (0, 1, 2)] + assert num_classes(src) == 3 + + +def test_num_classes_empty_source_raises() -> None: + with pytest.raises(ValueError, match="no targets"): + num_classes([]) + + +def test_num_classes_none_target_raises() -> None: + with pytest.raises(ValueError, match="no target"): + num_classes([Sample(input=np.array([1]), target=None)]) + + +# --------------------------------------------------------------------------- # +# _to_int coercion +# --------------------------------------------------------------------------- # + + +def test_to_int_rejects_bool() -> None: + with pytest.raises(TypeError, match="bool"): + _to_int(True) + + +def test_to_int_rejects_non_scalar() -> None: + with pytest.raises(TypeError): + _to_int("3") + with pytest.raises(TypeError): + _to_int(torch.tensor([1, 2, 3])) # .item() on a multi-element tensor raises + + +def test_to_int_accepts_integer_valued_float_tensor() -> None: + assert _to_int(torch.tensor(2.0)) == 2 + + +# --------------------------------------------------------------------------- # +# Laziness +# --------------------------------------------------------------------------- # + + +def test_project_is_lazy() -> None: + def infinite() -> Iterator[Sample]: + for i in itertools.count(): + yield Sample(input=np.array([i]), target=i) + + first_two = list(itertools.islice(iter_targets(infinite()), 2)) + assert first_two == [0, 1] # never exhausts the infinite source + + +# --------------------------------------------------------------------------- # +# Flux.project +# --------------------------------------------------------------------------- # + + +def test_flux_project_runs_pipeline_then_drops_fields() -> None: + flux = Flux([Sample(input=np.array([1]), target=7, metadata={"k": "v"})]) + assert isinstance(flux, SupportsProjection) + out = list(flux.project(("target",))) + assert out[0].target == 7 + assert out[0].input is None + # routed through the module-level project() too + assert list(iter_targets(flux)) == [7] + + +def test_flux_num_classes_via_helper() -> None: + flux = Flux([Sample(input=np.array([1]), target=t) for t in (0, 1, 2, 1)]) + assert num_classes(flux) == 3 From 55535fabc9c1c6288856678356dac410bf166aee Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 30 May 2026 12:35:03 +0200 Subject: [PATCH 003/102] feat(categories): tag concrete ops category=op; FilterOp/WrappedOp -> engine Every Sample->Sample op (ops/{copy,numpy,parallel,stash,swap,tee,torch}) now carries category=op so FluxStudio's positive {op,source,dataset} allowlist surfaces it; FilterOp/WrappedOp move op->engine (raw-callable wrappers, not GUI nodes). Updates test_categories + AGENTS Discovery Categories. --- AGENTS.md | 8 ++++---- dataflux/core.py | 4 ++-- dataflux/ops/copy.py | 8 ++++---- dataflux/ops/numpy.py | 12 ++++++------ dataflux/ops/parallel.py | 2 +- dataflux/ops/stash.py | 4 ++-- dataflux/ops/swap.py | 2 +- dataflux/ops/tee.py | 2 +- dataflux/ops/torch.py | 6 +++--- tests/test_categories.py | 24 ++++++++++++++++-------- 10 files changed, 40 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ff08e2c..35a26b5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,10 +9,10 @@ - **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. - **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — the generic, task-agnostic **engines/containers** that compose sources + ops: `Flux`, `JointFlux`, `DatasetSplit`. These are NOT datasets — a `Flux` *happens* to implement the `Dataset` interface but conceptually it's the pipeline engine. FluxStudio hides them (you wire `Source → Op` and the canvas builds the `Flux` implicitly); navigaitor does NOT offer them in dataset slots. - - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`). These ARE datasets/sources — they appear as FluxStudio source nodes and as navigaitor dataset-slot options. - - `category="op"` — pipeline ops: `FilterOp` / `WrappedOp`. - Rationale (history): these were ALL once `category="dataset"` (engine) vs uncategorised (sources) — backwards and confusing (`Flux` the engine was the "dataset", while `HuggingFaceSource` the actual dataset had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and dropping/renaming a tag silently empties the corresponding picker. + - `category="engine"` — generic **composition primitives** wired in code, NOT canvas nodes: the dataset engines `Flux` / `JointFlux` / `DatasetSplit` (compose sources + ops; a `Flux` *implements* the `Dataset` interface but is conceptually the engine) AND the higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` (take a *raw Python callable* — nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`). FluxStudio source nodes; navigaitor dataset-slot options. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, and the waivefront signal/target ops + `Enable`). FluxStudio uses a POSITIVE allowlist `{op, source, dataset}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. + Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. diff --git a/dataflux/core.py b/dataflux/core.py index 3bd67ec..a05c9b7 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -106,7 +106,7 @@ def _check_ops_materialized(ops: List[Any]) -> None: raise TypeError(_fluid_op_guidance(op, i)) -@configurable(category="op") +@configurable(category="engine") class FilterOp: """Configurable filter operation. @@ -121,7 +121,7 @@ def __call__(self, s: Sample) -> Optional[Sample]: return s if self.p(s) else None -@configurable(category="op") +@configurable(category="engine") class WrappedOp: """Configurable transformation wrapper with smart mapping. diff --git a/dataflux/ops/copy.py b/dataflux/ops/copy.py index 6a4c333..2c97b77 100644 --- a/dataflux/ops/copy.py +++ b/dataflux/ops/copy.py @@ -14,7 +14,7 @@ from dataflux.sample import Sample -@configurable +@configurable(category="op") class CopySampleOp: """Deepcopy of input, target, and metadata.""" @@ -26,7 +26,7 @@ def __call__(self, sample: Sample) -> Sample: ) -@configurable +@configurable(category="op") class CopyInputOp: """Deepcopy ``sample.input``.""" @@ -34,7 +34,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=copy.deepcopy(sample.input)) -@configurable +@configurable(category="op") class CopyTargetOp: """Deepcopy ``sample.target``.""" @@ -42,7 +42,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(target=copy.deepcopy(sample.target)) -@configurable +@configurable(category="op") class CopyMetadataOp: """Deepcopy ``sample.metadata``. diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index a22842c..5559e25 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -58,7 +58,7 @@ def _repl(match: "re.Match[str]") -> str: return _EXPR_PATTERN.sub(_repl, value) -@configurable +@configurable(category="op") class StandardizeOp: """ Standardizes ndarray values with given mean and standard deviation. @@ -123,7 +123,7 @@ def _require_ndarray(sample: Sample, op_name: str) -> np.ndarray: return arr -@configurable +@configurable(category="op") class ClipPercentilesOp: """Clip ``sample.input`` to ``[p_low, p_high]`` percentiles of finite values. @@ -158,7 +158,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=np.clip(arr, lo, hi)) -@configurable +@configurable(category="op") class RescaleOp: """Affine rescale ``sample.input`` from ``[in_min, in_max]`` to ``[out_min, out_max]``. @@ -210,7 +210,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=out) -@configurable +@configurable(category="op") class ReplaceNonFiniteOp: """Replace ``inf`` / ``-inf`` / ``nan`` entries in ``sample.input``. @@ -249,7 +249,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=np.where(non_finite, repl, arr)) -@configurable +@configurable(category="op") class ThresholdOp: """Threshold ``sample.input`` (ndarray) into a boolean mask: ``input > value``. @@ -297,7 +297,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=arr > threshold) -@configurable +@configurable(category="op") class ConnectedComponentsOp: """Label connected ``True`` regions of a boolean mask into bin-bbox tuples. diff --git a/dataflux/ops/parallel.py b/dataflux/ops/parallel.py index ac57f69..2196d53 100644 --- a/dataflux/ops/parallel.py +++ b/dataflux/ops/parallel.py @@ -29,7 +29,7 @@ from dataflux.sample import Sample -@configurable +@configurable(category="op") class Parallel: """Run an inner op sub-pipeline in a worker pool with bounded prefetch. diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py index 9adcfef..45a2f93 100644 --- a/dataflux/ops/stash.py +++ b/dataflux/ops/stash.py @@ -19,7 +19,7 @@ from dataflux.sample import Sample -@configurable +@configurable(category="op") class StashInputOp: """Copy ``sample.input`` into ``metadata[key]``; ``sample.input`` unchanged. @@ -40,7 +40,7 @@ def __call__(self, sample: Sample) -> Sample: return sample -@configurable +@configurable(category="op") class UnstashInputOp: """Set ``sample.input := metadata[key]``. diff --git a/dataflux/ops/swap.py b/dataflux/ops/swap.py index 142e85a..757d1cd 100644 --- a/dataflux/ops/swap.py +++ b/dataflux/ops/swap.py @@ -9,7 +9,7 @@ from dataflux.sample import Sample -@configurable +@configurable(category="op") class SwapInputTargetOp: """Exchange ``sample.input`` ↔ ``sample.target``. Metadata unchanged.""" diff --git a/dataflux/ops/tee.py b/dataflux/ops/tee.py index 6476944..878565f 100644 --- a/dataflux/ops/tee.py +++ b/dataflux/ops/tee.py @@ -17,7 +17,7 @@ from dataflux.sample import Sample -@configurable +@configurable(category="op") class Tee: """Run N op-list branches sequentially on the same sample / metadata. diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index 50fed5a..56907b1 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -11,7 +11,7 @@ _TORCH_FLOAT = ArrayType(dtype="floating", frameworks={"torch"}) -@configurable +@configurable(category="op") class ToTensorOp: """ Converts input (PIL Image, NumPy array, etc.) to a Torch Tensor. @@ -56,7 +56,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=tensor) -@configurable +@configurable(category="op") class RescaleOp: """Affine rescale a torch.Tensor from ``[in_min, in_max]`` to ``[out_min, out_max]``. @@ -106,7 +106,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=out) -@configurable +@configurable(category="op") class StandardizeOp: """ Standardizes tensor values with given mean and standard deviation. diff --git a/tests/test_categories.py b/tests/test_categories.py index 97a05ee..adb85f3 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,16 +10,22 @@ from confluid.registry import get_registry from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp +from dataflux.ops.tee import Tee from dataflux.sources import DatasetSplit, HuggingFaceSource def test_engine_classes_tagged() -> None: - """``Flux`` / ``JointFlux`` / ``DatasetSplit`` are the generic, task-agnostic *engines* - (containers that compose sources + ops), NOT datasets. They are excluded from FluxStudio's - node palette (you wire Source -> Op instead) and are not offered in navigaitor dataset slots.""" + """The generic, task-agnostic *engines* / composition primitives — NOT GUI-buildable nodes. + + ``Flux`` / ``JointFlux`` / ``DatasetSplit`` compose sources + ops; ``FilterOp`` / ``WrappedOp`` + wrap a raw Python callable. All carry ``category="engine"`` so FluxStudio's positive + op/source/dataset allowlist excludes them (you wire Source → Op instead).""" assert Flux.__confluid_category__ == "engine" assert JointFlux.__confluid_category__ == "engine" assert DatasetSplit.__confluid_category__ == "engine" + assert FilterOp.__confluid_category__ == "engine" + assert WrappedOp.__confluid_category__ == "engine" def test_source_classes_tagged() -> None: @@ -28,9 +34,11 @@ def test_source_classes_tagged() -> None: def test_op_classes_tagged() -> None: - """``FilterOp`` / ``WrappedOp`` are pipeline ops.""" - assert FilterOp.__confluid_category__ == "op" - assert WrappedOp.__confluid_category__ == "op" + """Concrete ``Sample → Sample`` ops carry ``category="op"`` (the FluxStudio op-node allowlist).""" + assert RescaleOp.__confluid_category__ == "op" + assert StandardizeOp.__confluid_category__ == "op" + assert ThresholdOp.__confluid_category__ == "op" + assert Tee.__confluid_category__ == "op" def test_categories_enumerable_via_registry() -> None: @@ -40,6 +48,6 @@ def test_categories_enumerable_via_registry() -> None: not just the class attribute — has to carry the tag. """ registry = get_registry() - assert {"Flux", "JointFlux", "DatasetSplit"} <= registry.list_classes(category="engine") + assert {"Flux", "JointFlux", "DatasetSplit", "FilterOp", "WrappedOp"} <= registry.list_classes(category="engine") assert {"HuggingFaceSource"} <= registry.list_classes(category="source") - assert {"FilterOp", "WrappedOp"} <= registry.list_classes(category="op") + assert {"RescaleOp", "StandardizeOp", "ThresholdOp", "Tee"} <= registry.list_classes(category="op") From 1a2f1a5139a0996aa53bf8c09973003aef62e946 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 31 May 2026 16:23:27 +0200 Subject: [PATCH 004/102] Enhance tests for DataFlux with new target operations and type specifications - Added tests for target movers and encoders in `tests/test_target_ops.py`, covering `MetadataToTargetOp`, `EncodeTargetOp`, and `DecodeTargetOp`. - Improved type specification tests in `tests/test_typespec.py` to validate closed literals for `Framework`, `ImageLayout`, and `DtypeFamily`. - Updated projection tests in `tests/test_projection.py` to utilize new constants for field names. - Enhanced source tests in `tests/test_sources.py` to validate three-way splits and property API for `DatasetSplit`. - Added round-trip tests for array metadata in HDF5 and Zarr storage in `tests/test_storage.py`. - Refactored tests to ensure clarity and maintainability, including checks for metadata handling in various storage formats. --- AGENTS.md | 17 +- README.md | 197 +++++++--- dataflux/__init__.py | 21 +- dataflux/core.py | 7 +- dataflux/discovery.py | 55 ++- dataflux/hf_core.py | 209 ----------- dataflux/ops/__init__.py | 7 +- dataflux/ops/copy.py | 8 +- dataflux/ops/image.py | 281 ++++++++++++++ dataflux/ops/numpy.py | 112 ++++-- dataflux/ops/parallel.py | 2 +- dataflux/ops/stash.py | 4 +- dataflux/ops/swap.py | 2 +- dataflux/ops/target.py | 137 +++++++ dataflux/ops/tee.py | 2 +- dataflux/ops/torch.py | 6 +- dataflux/paired.py | 194 ++++++---- dataflux/projection.py | 24 +- dataflux/sources.py | 434 ++++++++++++++++------ dataflux/storage/base.py | 13 + dataflux/storage/hdf5.py | 40 +- dataflux/storage/intake.py | 152 -------- dataflux/storage/zarr.py | 119 +++++- dataflux/typespec.py | 120 ++++-- examples/dataset_split.yaml | 61 +-- examples/intake_pipeline.py | 103 ----- examples/paired_annotations.py | 52 +-- examples/storage_roundtrip.py | 64 ++++ manifests/datasets/flux.yaml | 18 - manifests/datasets/joint_flux.yaml | 10 - manifests/sources/dataset_split.yaml | 27 -- manifests/sources/huggingface_source.yaml | 34 -- pyproject.toml | 8 +- tests/test_categories.py | 91 ++++- tests/test_coverage_gap.py | 35 +- tests/test_hf_core.py | 38 -- tests/test_image_ops.py | 147 ++++++++ tests/test_intake.py | 156 -------- tests/test_node_docs.py | 4 + tests/test_ops.py | 97 ++++- tests/test_paired.py | 216 +++++------ tests/test_projection.py | 44 ++- tests/test_sources.py | 409 +++++++++++++++++--- tests/test_storage.py | 125 ++++++- tests/test_target_ops.py | 96 +++++ tests/test_typespec.py | 41 +- 46 files changed, 2676 insertions(+), 1363 deletions(-) delete mode 100644 dataflux/hf_core.py create mode 100644 dataflux/ops/image.py create mode 100644 dataflux/ops/target.py delete mode 100644 dataflux/storage/intake.py delete mode 100644 examples/intake_pipeline.py create mode 100644 examples/storage_roundtrip.py delete mode 100644 manifests/datasets/flux.yaml delete mode 100644 manifests/datasets/joint_flux.yaml delete mode 100644 manifests/sources/dataset_split.yaml delete mode 100644 manifests/sources/huggingface_source.yaml delete mode 100644 tests/test_hf_core.py create mode 100644 tests/test_image_ops.py delete mode 100644 tests/test_intake.py create mode 100644 tests/test_target_ops.py diff --git a/AGENTS.md b/AGENTS.md index 35a26b5..32932cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,19 +1,20 @@ # DataFlux Mandates - **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. -- **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. +- **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. -- **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). +- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. Storage classes are bare `@configurable` with **no** discovery `category` — they are YAML `!class:` nodes wired into source/sink slots, not FluxStudio canvas nodes (unlike the `category="source"`/`"op"` classes). **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `dataflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — generic **composition primitives** wired in code, NOT canvas nodes: the dataset engines `Flux` / `JointFlux` / `DatasetSplit` (compose sources + ops; a `Flux` *implements* the `Dataset` interface but is conceptually the engine) AND the higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` (take a *raw Python callable* — nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`). FluxStudio source nodes; navigaitor dataset-slot options. - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, and the waivefront signal/target ops + `Enable`). FluxStudio uses a POSITIVE allowlist `{op, source, dataset}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, and the waivefront signal/target ops + `Enable`). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` / `image` (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). -- **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). +- **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. ## Testing & Validation diff --git a/README.md b/README.md index e8be962..1857dad 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Part of the **Modular Quartet**: `LogFlow`, `Confluid`, `Liquify`, and `DataFlux ### Metadata & Discovery - **Passive Introspection:** Automatically discover available tools and ops for serialized manifests. -- **Discovery Categories:** `@configurable` classes are tagged with a confluid `category` (`Flux`/`JointFlux` → `dataset`, `FilterOp`/`WrappedOp` → `op`) so tools like navigaitor's `list_configurable_classes(category=...)` enumerate them by kind. +- **Discovery Categories:** `@configurable` classes are tagged with a confluid `category` (sources `HuggingFaceSource`/`DatasetSplit` → `source`, engines `Flux`/`JointFlux` → `engine`, concrete `Sample→Sample` ops → `op`; `FilterOp`/`WrappedOp` are deliberately UNcategorised) so tools like navigaitor's `list_configurable_classes(category=...)` enumerate them by kind. - **Serialization Symmetry:** Ensure full-pipeline states are serializable and reconstructible via Confluid. ## 🛠 Quick Start @@ -68,6 +68,13 @@ ArrayType.parse("3 h w", dtype="float32", framework="torch") # jaxtyping-style ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") # an image convenience ``` +`dtype`, `framework`/`frameworks`, and the image `layout` are **closed `Literal`s**, not bare strings — a typo is a type error and a UI / connection-validator enumerates the choices via `typing.get_args(...)`: + +- `Dtype` — concrete names (`"float32"`, `"int64"`, …); `DtypeFamily` — relaxed families (`"floating"`, `"numeric"`, …); `DtypeSpec = Dtype | DtypeFamily` is the `dtype` field type. +- `Framework = Literal["numpy", "torch", "tensorflow"]`, `ImageLayout = Literal["CHW", "HWC"]`. + +Authored dtypes must be canonical names; aliases / casing (`"double"`, `"FLOAT32"`) and exotic platform dtypes (`float128`) are runtime-only conveniences normalized by `canonical_dtype` — the single boundary where arbitrary input crosses into the typed domain. + **Declare an op's contract** with the class attributes `ACCEPTS` / `PRODUCES` (each a `SampleType`; both default to `Any`, so annotating is optional and backward-compatible). No base class — transforms stay plain callables: ```python @@ -88,6 +95,7 @@ adds an opt-in protocol plus lazy helpers: ```python from dataflux import project, iter_targets, num_classes +from dataflux import ProjectionField # Literal["input", "target", "metadata"] # A source MAY implement SupportsProjection (`project(fields)`) to skip building # unrequested fields — e.g. an image dataset reads only the label column for a @@ -99,11 +107,46 @@ labels = list(iter_targets(my_source)) # lazy n = num_classes(my_source) # max(class_id) + 1 — always walks ``` +The field set is a **closed `Literal`**, `ProjectionField`, not a bare `str` — +so a typo is a type error, and a UI / form-spec / MCP schema enumerates the +choices straight from the annotation instead of hard-coding a parallel list: + +```python +from typing import get_args +get_args(ProjectionField) # ('input', 'target', 'metadata') +``` + Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup). `num_classes` is a free function, not a `Flux` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. +## 🖼 Image Conversion (`dataflux.ops.image`) + +The single, modality-agnostic "any value → image" layer — generic so every +project (waivefront spectrograms, any dataset preview, FluxStudio) reuses one +implementation. Domain-specific rendering (overlays, signal plots) stays in the +consuming package. + +```python +from dataflux.ops.image import ConvertToImageOp, value_to_image + +# Op: sample.input (2-D map / CHW tensor / PIL / bool mask) -> PIL image. +op = ConvertToImageOp( + colormap="viridis", # closed `Colormap` Literal -> dropdown in FluxStudio, enum in navigaitor + width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size + flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top +) +sample = op(sample) # also publishes image_width_px / image_height_px to metadata + +# Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): +rgb = value_to_image(some_value, colormap="magma", max_size=512) +``` + +`Colormap` / `COLORMAPS` / `value_to_image` / `sample_to_image` are re-exported +from `waivefront.visualizers` for backward compatibility. Pillow is a runtime +dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). + ## 📦 Storage Integration DataFlux makes it easy to move data between different formats: @@ -119,53 +162,110 @@ Flux.from_source(HDF5Source("input.h5")) \ .to_sink(ZarrGroupSink("output.zarr")) ``` -## ✂️ Train / Val Splitting +### Sinks and their matching sources -`DatasetSplit` carves a subset view out of any indexable source (implementing `__len__` and `__getitem__`). It supports three modes: +Every sink has a source that reads its layout back into `Sample` triplets: -1. **Fraction mode** — pick a reproducible train/val split from a single source: +| Backend | Sink | Source | Round-trips | +|---|---|---|---| +| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | input + target + metadata | +| Zarr group (one group / sample) | `ZarrGroupSink` | `ZarrGroupSource` | input + target + metadata | +| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | input only (uniform shape) | +| Directory (one dir / sample) | `DirectorySink` | — | — | - ```yaml - hf_train: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: train +```python +from dataflux.storage.zarr import ZarrGroupSink, ZarrGroupSource - train_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.1 - seed: 42 +Flux(samples).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) +for sample in ZarrGroupSource("ds.zarr"): # input/target as before, metadata from .zattrs + ... +``` - val_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 - ``` +### Array-valued metadata (e.g. segmentation masks) + +`HDF5Sink` stores scalar/string metadata as HDF5 **attributes**, but HDF5 caps +attribute size — a large array (a segmentation mask, a per-sample weight map) put +in `Sample.metadata` would overflow that limit. So **array-valued metadata +(`np.ndarray` / `torch.Tensor`) is written as its own dataset** under a per-sample +group `{prefix}_meta/`, and `HDF5Source` merges it back into `Sample.metadata` +on read. This is fully backward-compatible: files written before this layout (no +`{prefix}_meta` group) read exactly as before. + +```python +sample = Sample(input=iq, target=label, metadata={"mask": mask_2d, "snr": 12.0}) +Flux([sample]).to_sink(HDF5Sink("ds.h5", overwrite=True)) +loaded = next(iter(HDF5Source("ds.h5"))) +loaded.metadata["mask"] # the full array, byte-exact (not a truncated repr) +loaded.metadata["snr"] # scalar, via attributes as before +``` + +## ✂️ Train / Val / Test Splitting + +`DatasetSplit` partitions any indexable source (implementing `__len__` and `__getitem__`) into reproducible **train / val / test** views. It is a `source` (`category="source"`) — it yields `Sample`s and is wired into a trainer's `source:` slot — and it applies no ops, so it's a source, not an engine. + +**Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: + +```python +from dataflux import DatasetSplit +split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) +split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% +``` + +The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: + +```yaml +hf_train: !class:dataflux.sources.HuggingFaceSource() + path: mnist + split: train + +my_split: !class:dataflux.sources.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + +train_set: !class:dataflux.core.Flux() { source: !ref:my_split.train } +val_set: !class:dataflux.core.Flux() { source: !ref:my_split.val } +test_set: !class:dataflux.core.Flux() { source: !ref:my_split.test } +``` - Same seed + same source length ⇒ deterministic, disjoint, complementary views. +Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). -2. **Range mode** — explicit slice: +**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `dataflux.SplitName`. + +```yaml +val_set: !class:dataflux.sources.DatasetSplit() + source: !ref:hf_train + split: val + val_fraction: 0.1 + seed: 42 +``` + +### Range & concatenation sources + +- **`RangeSource(source, start, end)`** — a contiguous index slice `[start:end)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. ```yaml - first_half: !class:dataflux.sources.DatasetSplit() + first_half: !class:dataflux.sources.RangeSource() source: !ref:hf_train start: 0 end: 5000 ``` -3. **HuggingFace native slicing** (alternative, no `DatasetSplit` needed): +- **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointFlux`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. ```yaml - train_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[:90%]" - val_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[90%:]" + combined: !class:dataflux.sources.ConcatSource() + sources: + - !ref:train_main + - !ref:extra_shard ``` -> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key, so a single `HuggingFaceSource` is loaded once and shared by both splits. Use `!clone:` when you want an independent deep copy instead. +**HuggingFace native slicing** (alternative, no DataFlux split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. + +> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. + +> **Lazy & zero-arg construction** — `HuggingFaceSource` follows the workspace lazy-init convention: the constructor does no work (no network), so `HuggingFaceSource()` is valid and building one is free. The dataset is downloaded only on first access to the read-only `.dataset` property (cached thereafter; reset `_dataset` to reload), and `.resolved_metadata_features` (the `"*"` expansion) is derived lazily from the loaded columns. `path` is therefore optional at construction and validated lazily — accessing `.dataset` with an empty `path` raises a clear `ValueError`. ## 🔁 Reattach an ops-only YAML (`Flux.from_ops_yaml`) @@ -182,43 +282,43 @@ The helper **materializes** the deferred `!class:` markers before attaching (via ## 🔗 Paired Join (Binary ↔ Annotations) -`PairedSource` joins a primary `DataSource` (e.g. raw binary samples) with a secondary mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern. Three join policies cover the scenarios we actually see in ML research: +`AnnotationJoinSource` joins a data `DataSource` (e.g. raw binary samples) with a sidecar mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern — typically re-attaching a LabelStudio export back onto the raw samples for training. Three join policies cover the scenarios we actually see in ML research: | Policy | Iterates | Use case | |---|---|---| -| `left_outer` (default) | Every primary sample; attaches annotation when the key matches | Process everything, use labels where available | -| `inner` | Only primary samples whose key is in the store | Train/evaluate on the labeled subset | -| `right_driven` | Every key in the annotation store; resolves the primary sample via `primary_resolver(key, primary)` | Very sparse labels where full-primary enumeration is costly | +| `left_outer` (default) | Every data sample; attaches annotation when the key matches | Process everything, use labels where available | +| `inner` | Only data samples whose key is in the store | Train/evaluate on the labeled subset | +| `right_driven` | Every key in the annotation store; resolves the data sample via `data_resolver(key, data)` | Very sparse labels where full-data enumeration is costly | ```yaml -primary: !class:waivefront.rfuav.data.source.RFUAVSource() +data: !class:waivefront.rfuav.data.source.RFUAVSource() root: /Volumes/Data/RFUAV window_samples: 1000000 labels: !class:annotaide.store.JSONFileAnnotationStore() path: /Volumes/Data/RFUAV-labels -paired: !class:dataflux.paired.PairedSource() - primary: !ref:primary - secondary: !ref:labels +paired: !class:dataflux.paired.AnnotationJoinSource() + data: !ref:data + annotations: !ref:labels key_fn: "waivefront.rfuav.keys:sample_window_key" policy: left_outer ``` -Annotation records are **flattened into `Sample.metadata`**, so a detection record `{bboxes, labels, scores}` shows up as three independent metadata keys. Two `metadata` keys are always populated: `annotated: bool` and `annotation_key: str`. Optional `prefix` and `store_full_under` parameters shape the layout. +Annotation records are **flattened into `Sample.metadata`**, so a detection record `{bboxes, labels, scores}` shows up as three independent metadata keys. Two `metadata` keys are always populated: `annotated: bool` and `annotation_key: str`. Optional `prefix` and `store_full_under` parameters shape the layout. The parameters are typed, not `Any`: `data` is an `Iterable[Any]` (any source), `annotations` is an `AnnotationStore` (a read-mapping `key → record` — a `dict` or annotaide's `JSONFileAnnotationStore` both qualify), and `policy` is a fixed `Literal["left_outer", "inner", "right_driven"]`. Both the store shape and the policy are validated at construction. ### Coarser-granularity keys (broadcast and slicing) -`key_fn` is free to return a coarser key than the sample granularity. When multiple primary samples map to the same key, they all look up the same record: +`key_fn` is free to return a coarser key than the sample granularity. When multiple data samples map to the same key, they all look up the same record: - **Without `extract_fn`** — the record is broadcast identically into every matching sample's metadata (e.g. a scalar pack-level class label inherited by every window of that pack). - **With `extract_fn`** — the record is projected per sample. The callable is invoked as `extract_fn(record, sample) -> dict | None`; returning `None` marks the sample unannotated (and filters it under `policy="inner"`). Use this when a pack-level annotation carries time-ranged content that must be trimmed to each window's bounds. -Multi-granularity joins (e.g. pack-level + window-level annotations merged together) compose by chaining `PairedSource` instances — the output of one is itself a `DataSource` that the next can consume. +Multi-granularity joins (e.g. pack-level + window-level annotations merged together) compose by chaining `AnnotationJoinSource` instances — the output of one is itself a `DataSource` that the next can consume. ### Callable resolution -`key_fn`, `extract_fn`, and `primary_resolver` all accept either a callable **or** a `"module:function"` string path resolved through `dataflux.discovery.resolve_callable`. The string form is what survives YAML round-trip via Confluid. +`key_fn`, `extract_fn`, and `data_resolver` all accept either a callable **or** a `"module:function"` string path resolved through `dataflux.discovery.resolve_callable`. The string form is what survives YAML round-trip via Confluid. See [`examples/paired_annotations.py`](examples/paired_annotations.py) for a runnable end-to-end walkthrough of all four scenarios. @@ -226,13 +326,18 @@ See [`examples/paired_annotations.py`](examples/paired_annotations.py) for a run DataFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines. -### Intake (Data Discovery & Catalogs) -- **Use Intake for:** Data discovery, remote storage abstraction (S3/GCS), and sharing "canned" datasets via YAML catalogs. -- **Integration:** Wrap an Intake driver in a DataFlux `DataSource` to gain functional `.map()`, `.filter()`, and `.parallel()` capabilities on cataloged data. - ### Hugging Face (Community & Standardized Datasets) - **Use Hugging Face for:** Accessing community datasets and leveraging the `datasets` library for efficient Arrow/Parquet loading. - **Integration:** Use DataFlux to transform `datasets.Dataset` objects into standardized `Sample` triplets, ensuring metadata traceability that often goes missing in simple dictionary-based records. +- **`metadata_features` (which columns ride along on `Sample.metadata`):** `None` / `[]` keep none (the default); an explicit list keeps exactly those columns; and the sentinel **`"*"`** (or `["*"]`) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load. It stays opt-in so existing configs are unchanged. + +```yaml +hf_train: !class:dataflux.sources.HuggingFaceSource() + path: mnist + input_feature: image + target_feature: label + metadata_features: ["*"] # keep every other column as metadata (here: none extra beyond hf_path/hf_split) +``` ### DataFlux (The Functional Engine) - **Use DataFlux for:** The "inner loop" of your experiment. When you need high-performance multiprocess streaming, per-sample metadata preservation, and 100% reproducible pipelines via **Confluid** serialization. diff --git a/dataflux/__init__.py b/dataflux/__init__.py index ca29bc8..5d1818c 100644 --- a/dataflux/__init__.py +++ b/dataflux/__init__.py @@ -4,14 +4,18 @@ from dataflux.core import Flux, JointFlux, WrappedOp from dataflux.ops import RescaleOp, StandardizeOp, ToTensorOp -from dataflux.paired import PairedSource -from dataflux.projection import SupportsProjection, iter_inputs, iter_targets, num_classes, project +from dataflux.paired import AnnotationJoinSource, AnnotationStore +from dataflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project from dataflux.sample import Sample -from dataflux.sources import DatasetSplit, HuggingFaceSource +from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName from dataflux.typespec import ( AnyType, ArrayType, Dim, + Dtype, + DtypeFamily, + DtypeSpec, + Framework, ListType, MappingType, PythonType, @@ -23,20 +27,29 @@ ) __all__ = [ + "AnnotationJoinSource", + "AnnotationStore", "AnyType", "ArrayType", + "ConcatSource", "DatasetSplit", "Dim", + "Dtype", + "DtypeFamily", + "DtypeSpec", "Flux", + "Framework", "HuggingFaceSource", "JointFlux", "ListType", "MappingType", - "PairedSource", + "ProjectionField", "PythonType", + "RangeSource", "RescaleOp", "Sample", "SampleType", + "SplitName", "StandardizeOp", "SupportsProjection", "ToTensorOp", diff --git a/dataflux/core.py b/dataflux/core.py index a05c9b7..f7fb5f8 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -25,6 +25,7 @@ from confluid.fluid import Fluid as _ConfluidFluid from logflow import get_logger +from dataflux.projection import ProjectionField from dataflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, Sample if TYPE_CHECKING: # pragma: no cover - typing only @@ -106,7 +107,7 @@ def _check_ops_materialized(ops: List[Any]) -> None: raise TypeError(_fluid_op_guidance(op, i)) -@configurable(category="engine") +@configurable class FilterOp: """Configurable filter operation. @@ -121,7 +122,7 @@ def __call__(self, s: Sample) -> Optional[Sample]: return s if self.p(s) else None -@configurable(category="engine") +@configurable class WrappedOp: """Configurable transformation wrapper with smart mapping. @@ -490,7 +491,7 @@ def collect(self) -> List[Sample]: """Materialize the full flux into a list.""" return list(self) - def project(self, fields: Collection[str]) -> Iterator[Sample]: + def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: """Yield pipeline-output Samples carrying only ``fields`` (the projection primitive). Implements :class:`dataflux.projection.SupportsProjection`. Flux must run diff --git a/dataflux/discovery.py b/dataflux/discovery.py index da58265..94fa9d0 100644 --- a/dataflux/discovery.py +++ b/dataflux/discovery.py @@ -1,3 +1,22 @@ +"""Passive introspection for DataFlux callables. + +A round-trip bridge between live Python callables (sources, ops, plain +functions) and JSON-serializable schemas, so downstream tools can discover and +wire pipeline pieces without any hand-written tool definitions (the workspace's +"Passive Introspection" mandate). Two halves: + +* **Serialization** (callable <-> string): :func:`get_callable_path` turns a + callable into an importable ``"module:qualname"`` key and + :func:`resolve_callable` imports it back. This is how a pipeline step is + referenced in a Confluid manifest and resurrected later for reproducibility. +* **Discovery** (callable -> JSON schema): :func:`introspect_callable` reflects + a single callable into a schema (signature + docstring + the ``ACCEPTS`` / + ``PRODUCES`` typespec contract), and :func:`scan_module` does the same for + every callable *defined in* a module. FluxStudio reads these to auto-generate + ComfyUI nodes and their property panels; navigaitor builds its MCP form-spec + from the same data. +""" + import importlib import importlib.util import inspect @@ -9,8 +28,12 @@ def get_callable_path(func: Callable) -> str: """ - Convert a function/class into an importable string path. - Avoids '__main__' by resolving the script's filename. + Convert a function/class into an importable string path (``"module:qualname"``). + Avoids '__main__' by resolving the script's filename, so the path stays + re-importable from another process. + + Use: the serialization key that lets a discovered callable be referenced in + a Confluid manifest and rehydrated later via :func:`resolve_callable`. """ if not callable(func): raise TypeError(f"Object {func} is not callable") @@ -35,7 +58,14 @@ def get_callable_path(func: Callable) -> str: def resolve_callable(path: Union[str, Callable]) -> Callable: - """Resolve an importable string path back into a callable.""" + """Resolve an importable string path back into a callable (inverse of + :func:`get_callable_path`). Handles a normal module import, a ``.py`` file + path loaded via ``spec_from_file_location``, and a ``.py``-suffix fallback; + an already-callable argument is returned unchanged. + + Use: rehydrating a serialized pipeline — turning the stored ``"module:func"`` + string back into the live op/source. + """ if callable(path): return path @@ -73,8 +103,14 @@ def resolve_callable(path: Union[str, Callable]) -> Callable: def introspect_callable(func: Callable) -> Dict[str, Any]: """ - Build a JSON-serializable schema for a callable. - Used by FluxStudio to render nodes and property panels. + Build a JSON-serializable schema for a callable by reflecting over its + signature: ``path``, ``name``, ``doc``, per-parameter info (``name`` / + ``type`` / ``default`` / ``required``, skipping ``self`` / ``cls`` / + ``*args`` / ``**kwargs``), plus the declared ``ACCEPTS`` / ``PRODUCES`` + typespec contract when present. + + Use: FluxStudio reads this to render a node and its property-panel widgets, + and it feeds navigaitor's MCP form-spec. """ try: sig = inspect.signature(func) @@ -113,7 +149,14 @@ def _spec_dict(spec: Any) -> Optional[Dict[str, Any]]: def scan_module(path_or_name: Union[str, Path]) -> List[Dict[str, Any]]: """ - Scan a module or script file and return schemas for locally callables. + Scan a module (by import name) or a ``.py`` script (by path) and return an + :func:`introspect_callable` schema for every callable *defined in* that + module — names merely imported into it are filtered out by checking + ``member.__module__ == mod_name``. + + Use: the entry point for whole-module discovery — FluxStudio's ``bridge`` + calls this to auto-generate one node per source/op, fulfilling the + "never require manual tool definitions" mandate. """ is_py = isinstance(path_or_name, Path) or (isinstance(path_or_name, str) and path_or_name.endswith(".py")) diff --git a/dataflux/hf_core.py b/dataflux/hf_core.py deleted file mode 100644 index 471181f..0000000 --- a/dataflux/hf_core.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Hugging Face Datasets based implementation of DataFlux core.""" - -from typing import Any, Callable, Dict, Iterator, List, Union, cast - -import datasets -import numpy as np -import torch -from confluid import configurable -from logflow import get_logger - -from dataflux.sample import Sample - -logger = get_logger(__name__) - - -def _encode_complex(val: Any) -> Any: - """Recursively encode complex types into Arrow-compatible dicts.""" - # Use np.iscomplexobj to catch arrays and scalars (numpy or python) - if np.iscomplexobj(val) or (isinstance(val, torch.Tensor) and val.is_complex()): - kind = "numpy" if not isinstance(val, torch.Tensor) else "torch" - if np.isscalar(val): - scalar = cast(complex, val) - return { - "_complex_": True, - "real": float(scalar.real), - "imag": float(scalar.imag), - "dtype": str(getattr(val, "dtype", "complex128")), - "kind": "scalar", - } - - real = val.real - imag = val.imag - if kind == "torch": - real = real.numpy() - imag = imag.numpy() - dtype = str(val.dtype) - else: - dtype = str(val.dtype) - - return { - "_complex_": True, - "real": real, - "imag": imag, - "dtype": dtype, - "kind": kind, - } - - if isinstance(val, dict): - return {k: _encode_complex(v) for k, v in val.items()} - if isinstance(val, (list, tuple)): - return [_encode_complex(v) for v in val] - return val - - -def _decode_complex(val: Any) -> Any: - """Recursively decode complex types from Arrow-compatible dicts.""" - if isinstance(val, dict) and val.get("_complex_"): - real = val["real"] - imag = val["imag"] - dtype_str = val.get("dtype", "complex128") - kind = val.get("kind", "numpy") - - # When coming back from Arrow/HF, arrays are often lists - if isinstance(real, list): - real = np.array(real) - imag = np.array(imag) - - if kind == "scalar": - return complex(real, imag) - - # Build intermediate numpy complex array - res_np = real + 1j * imag - if kind == "torch": - torch_dtype = torch.complex64 if "complex64" in dtype_str else torch.complex128 - # Convert to appropriate numpy complex first to ensure precision, then to torch - np_dtype = np.complex64 if "complex64" in dtype_str else np.complex128 - return torch.from_numpy(res_np.astype(np_dtype)).to(torch_dtype) - - return res_np.astype(dtype_str) - - if isinstance(val, dict): - return {k: _decode_complex(v) for k, v in val.items()} - if isinstance(val, list): - return [_decode_complex(v) for v in val] - return val - - -def _sample_to_dict(sample: Sample) -> Dict[str, Any]: - """Flatten Sample triplet into a dictionary for HF datasets.""" - d = { - "input": _encode_complex(sample.input), - "target": _encode_complex(sample.target), - } - # Flatten metadata - if sample.metadata: - for k, v in sample.metadata.items(): - # Avoid collisions with reserved keys - if k in ("input", "target"): - k = f"meta_{k}" - d[k] = _encode_complex(v) - return d - - -def _dict_to_sample(d: Dict[str, Any]) -> Sample: - """Reconstruct Sample triplet from an HF dataset row.""" - input_val = _decode_complex(d.get("input")) - target_val = _decode_complex(d.get("target")) - metadata = {k: _decode_complex(v) for k, v in d.items() if k not in ("input", "target")} - # Unflatten meta_ collisions - if "meta_input" in metadata: - metadata["input"] = metadata.pop("meta_input") - if "meta_target" in metadata: - metadata["target"] = metadata.pop("meta_target") - return Sample(input=input_val, target=target_val, metadata=metadata) - - -def wrap_op(op: Callable) -> Callable[[Dict[str, Any]], Dict[str, Any]]: - """Wrap a legacy Sample-based op so it can be used with HF datasets.map().""" - - def wrapper(row: Dict[str, Any]) -> Dict[str, Any]: - sample = _dict_to_sample(row) - result = op(sample) - if result is None: - # For HF filter ops or flat maps - return row # Filtering is separate in HF - return _sample_to_dict(result) - - return wrapper - - -@configurable -class HFFlux: - """ - Flux engine backed by Hugging Face `datasets`. - Provides a high-performance implementation of the DataFlux API. - """ - - def __init__(self, dataset: Union[datasets.Dataset, datasets.IterableDataset]) -> None: - self._dataset = dataset - - @property - def dataset(self) -> Union[datasets.Dataset, datasets.IterableDataset]: - return self._dataset - - @classmethod - def from_source(cls, source: Any) -> "HFFlux": - """Create an HFFlux from any iterable source. - - An in-memory sequence (``list``/``tuple``) is already fully - materialized, so we build a map-style ``datasets.Dataset`` for it — - this enables ``len()`` and integer indexing without writing any cache - files. This does NOT violate the lazy-evaluation mandate: the data is - already in memory, so there is no lazy iterator left to consume eagerly. - - Any other source (generator, streaming DataFlux source) is treated as - lazy and backed by an ``IterableDataset`` to avoid materializing it and - to avoid large cache files / permission issues on external drives - (like /Volumes/Store). - """ - if hasattr(source, "_dataset") and isinstance(source._dataset, (datasets.Dataset, datasets.IterableDataset)): - return cls(source._dataset) - - if isinstance(source, (list, tuple)): - rows = [_sample_to_dict(Sample.from_any(item)) for item in source] - return cls(datasets.Dataset.from_list(rows)) - - def gen() -> Iterator[Dict[str, Any]]: - for item in source: - yield _sample_to_dict(Sample.from_any(item)) - - ds = datasets.IterableDataset.from_generator(gen) - return cls(ds) - - def map(self, func: Callable, **kwargs: Any) -> "HFFlux": - """ - Apply a transformation to the dataset. - If `func` expects a Sample, it should be wrapped. - """ - # Default behavior: assume func handles dicts (HF style) - # We can add a 'wrapper' mode if needed for compatibility with old ops. - return HFFlux(self._dataset.map(func, **kwargs)) - - def filter(self, predicate: Callable, **kwargs: Any) -> "HFFlux": - """Filter the dataset.""" - return HFFlux(self._dataset.filter(predicate, **kwargs)) - - def parallel(self, workers: int = 4) -> "HFFlux": - """Set default number of processes for subsequent operations.""" - # datasets.map uses `num_proc` parameter. - # We can store this or apply it to the next operation. - # For now, this is just for API compatibility. - return self - - def __iter__(self) -> Iterator[Sample]: - for row in self._dataset: - yield _dict_to_sample(row) - - def __len__(self) -> int: - return len(self._dataset) - - def __getitem__(self, index: int) -> Sample: - return _dict_to_sample(self._dataset[index]) - - def collect(self) -> List[Sample]: - return list(self) - - def info(self) -> Any: - """Expose HF dataset info (schema, metadata).""" - return self._dataset.info diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index 12097e9..9e9205a 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -10,15 +10,17 @@ - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - dataflux.ops.swap: SwapInputTargetOp - dataflux.ops.stash: StashInputOp, UnstashInputOp + - dataflux.ops.target: MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp (target field) Flat imports default to torch variants for the data ops; flow / copy / -swap / stash utilities are field-agnostic. +swap / stash / target utilities are field-agnostic. """ from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp from dataflux.ops.parallel import Parallel from dataflux.ops.stash import StashInputOp, UnstashInputOp from dataflux.ops.swap import SwapInputTargetOp +from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp from dataflux.ops.tee import Tee from dataflux.ops.torch import RescaleOp, StandardizeOp, ToTensorOp @@ -27,6 +29,9 @@ "CopyMetadataOp", "CopySampleOp", "CopyTargetOp", + "DecodeTargetOp", + "EncodeTargetOp", + "MetadataToTargetOp", "Parallel", "RescaleOp", "StandardizeOp", diff --git a/dataflux/ops/copy.py b/dataflux/ops/copy.py index 2c97b77..c569b33 100644 --- a/dataflux/ops/copy.py +++ b/dataflux/ops/copy.py @@ -14,7 +14,7 @@ from dataflux.sample import Sample -@configurable(category="op") +@configurable(category="op", group="structure") class CopySampleOp: """Deepcopy of input, target, and metadata.""" @@ -26,7 +26,7 @@ def __call__(self, sample: Sample) -> Sample: ) -@configurable(category="op") +@configurable(category="op", group="structure") class CopyInputOp: """Deepcopy ``sample.input``.""" @@ -34,7 +34,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=copy.deepcopy(sample.input)) -@configurable(category="op") +@configurable(category="op", group="structure") class CopyTargetOp: """Deepcopy ``sample.target``.""" @@ -42,7 +42,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(target=copy.deepcopy(sample.target)) -@configurable(category="op") +@configurable(category="op", group="structure") class CopyMetadataOp: """Deepcopy ``sample.metadata``. diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py new file mode 100644 index 0000000..d10a501 --- /dev/null +++ b/dataflux/ops/image.py @@ -0,0 +1,281 @@ +"""Generic, modality-agnostic image conversion for DataFlux pipelines. + +This is the single home for "turn an arbitrary value into an image": the +:class:`ConvertToImageOp` op plus the library functions +(:func:`value_to_image` / :func:`sample_to_image`) that back it and FluxStudio's +sample preview. It lives in dataflux (not waivefront) because the conversion is +fully generic — a 2-D map, a CHW tensor, a PIL image, a boolean mask all render +the same way regardless of domain — so every project (waivefront's spectrogram +render, any image dataset preview, FluxStudio nodes) reuses ONE implementation. + +Domain-specific rendering stays in the consuming package: waivefront's +``RenderOverlaysOp`` draws signal-region rectangles on top of the PIL image this +op produces, and ``RenderSignalPlotOp`` builds IQ time/freq/constellation panels. +Those need signal semantics; this op does not. + +PIL is a hard dependency here (already used by ``dataflux.typespec``). Matplotlib +is imported lazily inside :func:`_apply_colormap` — only non-``"gray"`` colormaps +need it, so the pure-greyscale path stays matplotlib-free. +""" + +from typing import Any, Literal, Tuple, get_args + +import numpy as np +import torch +from confluid import configurable +from logflow import get_logger +from PIL import Image, ImageDraw + +from dataflux.sample import Sample +from dataflux.typespec import ArrayType as _ArrayType +from dataflux.typespec import PythonType, SampleType, UnionType + +logger = get_logger("dataflux.ops.image") + + +# Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob +# across the workspace (``value_to_image`` / ``sample_to_image`` / ``ConvertToImageOp`` and, via +# re-export, waivefront's renderers) AND for FluxStudio's colormap dropdown (which reads ``COLORMAPS``). +# A closed ``Literal`` (never a bare ``str``) makes the choice self-documenting and machine- +# introspectable: the FluxStudio palette, navigaitor's form-spec, and MCP tool schemas enumerate the +# options straight from the annotation via ``typing.get_args`` instead of hard-coding a parallel list +# that silently drifts. ``"gray"`` is the greyscale path (special-cased in ``_apply_colormap``); every +# other name resolves through ``matplotlib.colormaps[name]``. Per the workspace "closed Literal" +# mandate, derive the runtime tuple FROM the Literal (``get_args``) — never restate the values. +Colormap = Literal[ + "viridis", + "plasma", + "inferno", + "magma", + "cividis", + "gray", + "hot", + "cool", + "jet", + "turbo", + "twilight", + "hsv", +] +COLORMAPS: Tuple[Colormap, ...] = get_args(Colormap) + + +def _apply_colormap(spec_u8: np.ndarray, colormap: Colormap) -> Image.Image: + """Turn a ``(H, W)`` uint8 magnitude map into an RGB PIL image. + + ``colormap="gray"`` reproduces the greyscale-to-RGB path (matplotlib-free). + Any other name is resolved through ``matplotlib.colormaps[name]`` (lazily + imported) so standard cmaps (``"hot"``, ``"viridis"``, ``"magma"``, + ``"plasma"``, ``"inferno"``, ``"turbo"``, …) are supported. + """ + if colormap == "gray": + return Image.fromarray(spec_u8, mode="L").convert("RGB") + import matplotlib + + cmap = matplotlib.colormaps[colormap] + rgba = cmap(spec_u8.astype(np.float32) / 255.0) + rgb = (rgba[..., :3] * 255.0).astype(np.uint8) + return Image.fromarray(rgb, mode="RGB") + + +def _to_uint8(arr: np.ndarray) -> np.ndarray: + """Min-max normalize a numeric array to ``uint8`` in ``[0, 255]``. + + Non-finite entries are treated as the finite minimum. A flat array + (``max == min``) maps to all-zeros to avoid a divide-by-zero. + """ + arr = arr.astype(np.float32) + finite = arr[np.isfinite(arr)] + if finite.size == 0: + return np.zeros(arr.shape, dtype=np.uint8) + lo = float(finite.min()) + hi = float(finite.max()) + if hi <= lo: + return np.zeros(arr.shape, dtype=np.uint8) + norm = (np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) - lo) / (hi - lo) + return (np.clip(norm, 0.0, 1.0) * 255.0).astype(np.uint8) + + +def _text_to_image(text: str, width: int = 512, height: int = 160) -> np.ndarray: + """Render a short string to an ``(H, W, 3)`` uint8 image (non-image fallback).""" + img = Image.new("RGB", (width, height), color=(30, 30, 30)) + draw = ImageDraw.Draw(img) + max_chars = max(1, width // 7) + lines = [text[i : i + max_chars] for i in range(0, min(len(text), max_chars * 8), max_chars)] + draw.multiline_text((6, 6), "\n".join(lines) or "", fill=(220, 220, 220)) + return np.array(img) + + +def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: + """Render an arbitrary value to an ``(H, W, 3)`` uint8 RGB image WITHOUT resizing. + + The core of :func:`value_to_image` factored out so callers that need their + own resize policy (e.g. :class:`ConvertToImageOp`'s exact ``width``/``height``) + don't pay a double resize. Handles PIL images, torch tensors, numpy arrays + (2-D maps → ``colormap``; 3-D → image with channel coercion; bool → 0/255); + anything else falls back to a text rendering of its ``repr``. + """ + data: Any = value + + if hasattr(data, "convert"): # PIL.Image.Image + data = np.array(data.convert("RGB")) + elif isinstance(data, torch.Tensor): + data = data.detach().cpu().numpy() + + if not isinstance(data, np.ndarray): + return _text_to_image(repr(data)) + + arr = np.squeeze(np.asarray(data)) + + if arr.dtype == np.bool_: + arr = arr.astype(np.uint8) * 255 + + if arr.ndim == 2: + return np.array(_apply_colormap(_to_uint8(arr), colormap)) + if arr.ndim == 3: + # Normalize channel position to trailing (HWC). + if arr.shape[0] in (1, 3, 4) and arr.shape[2] not in (1, 3, 4): + arr = np.transpose(arr, (1, 2, 0)) + channels = arr.shape[2] + if channels == 3: + pass + elif channels == 1: + arr = np.repeat(arr, 3, axis=2) + elif channels >= 4: + arr = arr[..., :3] + else: # 2 channels (or other) — replicate the first + arr = np.repeat(arr[..., :1], 3, axis=2) + return arr if arr.dtype == np.uint8 else _to_uint8(arr) + return _text_to_image(f"input ndim={arr.ndim}, shape={arr.shape}") + + +def _bound_longest_side(rgb: np.ndarray, max_size: int) -> np.ndarray: + """Downscale an ``(H, W, 3)`` image so its longest side is ≤ ``max_size`` (aspect preserved).""" + height_px, width_px = rgb.shape[:2] + longest = max(height_px, width_px) + if max_size <= 0 or longest <= max_size: + return rgb.astype(np.uint8) + scale = max_size / longest + resized = Image.fromarray(rgb).resize( + (max(1, int(width_px * scale)), max(1, int(height_px * scale))), + Image.Resampling.BILINEAR, + ) + return np.array(resized).astype(np.uint8) + + +def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: + """Render an arbitrary value (a Sample's ``input`` OR ``target``) to an ``(H, W, 3)`` uint8 RGB image. + + A generic, modality-agnostic preview usable from any DataFlux pipeline (and + by FluxStudio's sample extractor, which renders the selected field). Handles: + + * ``PIL.Image`` — converted to RGB; + * ``torch.Tensor`` — detached to numpy (CHW collapsed to HWC below); + * ``np.ndarray`` — 2-D maps go through ``colormap`` (one of the supported + colormaps — see ``Colormap``; ``"gray"`` for greyscale); 3-D arrays are + treated as images (a leading channel axis is transposed to trailing, + 1/2/4-channel coerced to 3); boolean masks become 0/255; floating arrays + are min-max normalized. + + Anything else (e.g. a bbox list) falls back to a text rendering of its + ``repr`` so the caller still shows *something* rather than erroring. + ``max_size`` bounds the longest side. + + Args: + value: The value to render (image / tensor / ndarray / mask, else a text repr of its ``repr``). + colormap: Colormap applied to 2-D maps — one of the supported names (see ``Colormap``; ``"gray"`` = greyscale). + max_size: Maximum length in pixels of the longest image side; larger renders are downscaled. + """ + return _bound_longest_side(_render_rgb(value, colormap), max_size) + + +def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: + """Render ``sample.input`` to an ``(H, W, 3)`` uint8 RGB image for display. + + Thin wrapper over :func:`value_to_image` (which does the modality-agnostic + rendering) applied to ``sample.input``. Kept as the canonical "preview a + sample" entry point for DataFlux pipelines; use :func:`value_to_image` + directly to render an arbitrary value such as ``sample.target``. + + Args: + sample: The Sample to preview; its ``input`` field is rendered. + colormap: Colormap applied to 2-D maps — one of the supported names (see ``Colormap``; ``"gray"`` = greyscale). + max_size: Maximum length in pixels of the longest image side; larger renders are downscaled. + """ + return value_to_image(sample.input, colormap=colormap, max_size=max_size) + + +@configurable(category="op", group="image") +class ConvertToImageOp: + """Convert ``sample.input`` (array / tensor / 2-D map / PIL image) into a PIL image. + + The generic image-conversion op — normalize → colormap → (flip) → resize. + It is modality-agnostic: a dB spectrogram, a segmentation logit map, a CHW + tensor, or an already-PIL image all become a ``PIL.Image.Image`` on + ``sample.input``. Domain overlays are a SEPARATE concern — chain + ``waivefront.visualizers.RenderOverlaysOp`` after this op to draw + signal-region rectangles; this op never draws annotations. + + Rendering uses :func:`value_to_image`'s core (so 2-D maps are colormapped, + 3-D arrays treated as images, bool masks become 0/255, floats min-max + normalized). Sizing: + + * ``width`` and ``height`` both > 0 → resize to exactly that raster + (e.g. a spectrogram rendered to ``1024x512`` for downstream detectors). + * otherwise → bound the longest side by ``max_size``, preserving aspect. + + ``flip_vertical=True`` mirrors the image top-to-bottom — used when the source + array's row 0 is the *bottom* of the desired image (a spectrogram stores + row 0 = f_min but display wants f_max at the top, so overlay pixel math + lines up). The final ``image_width_px`` / ``image_height_px`` are published + to ``sample.metadata`` so downstream consumers (e.g. a detector + back-projecting pixel boxes to signal regions) can read the raster size. + + Args: + colormap: Colormap applied to 2-D maps — a supported ``Colormap`` name (``"gray"`` = greyscale). + width: Exact output width in pixels; resize to ``(width, height)`` when both width and height are > 0. + height: Exact output height in pixels; resize to ``(width, height)`` when both width and height are > 0. + max_size: When ``width``/``height`` aren't both set, bound the longest side to this many pixels (aspect kept). + flip_vertical: Mirror the image top-to-bottom (e.g. spectrogram row 0 = f_min → display f_max at the top). + """ + + ACCEPTS = SampleType(input=UnionType((PythonType("PIL.Image.Image"), _ArrayType(frameworks={"numpy", "torch"})))) + PRODUCES = SampleType(input=PythonType("PIL.Image.Image")) + + def __init__( + self, + colormap: Colormap = "gray", + width: int = 0, + height: int = 0, + max_size: int = 512, + flip_vertical: bool = False, + ) -> None: + self.colormap: Colormap = colormap + self.width = int(width) + self.height = int(height) + self.max_size = int(max_size) + self.flip_vertical = bool(flip_vertical) + + def __call__(self, sample: Sample) -> Sample: + rgb = _render_rgb(sample.input, self.colormap) + if self.flip_vertical: + rgb = rgb[::-1, :, :] + if self.width > 0 and self.height > 0: + img = Image.fromarray(rgb).resize( + (self.width, self.height), + resample=Image.Resampling.BILINEAR, + ) + else: + img = Image.fromarray(_bound_longest_side(rgb, self.max_size)) + + sample.metadata["image_width_px"] = img.width + sample.metadata["image_height_px"] = img.height + return sample._replace(input=img) + + +__all__ = [ + "Colormap", + "COLORMAPS", + "ConvertToImageOp", + "value_to_image", + "sample_to_image", +] diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index 5559e25..e5421c3 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -1,6 +1,7 @@ +import operator import os import re -from typing import List, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union import numpy as np from confluid import configurable @@ -58,7 +59,7 @@ def _repl(match: "re.Match[str]") -> str: return _EXPR_PATTERN.sub(_repl, value) -@configurable(category="op") +@configurable(category="op", group="numpy") class StandardizeOp: """ Standardizes ndarray values with given mean and standard deviation. @@ -123,7 +124,7 @@ def _require_ndarray(sample: Sample, op_name: str) -> np.ndarray: return arr -@configurable(category="op") +@configurable(category="op", group="numpy") class ClipPercentilesOp: """Clip ``sample.input`` to ``[p_low, p_high]`` percentiles of finite values. @@ -158,7 +159,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=np.clip(arr, lo, hi)) -@configurable(category="op") +@configurable(category="op", group="numpy") class RescaleOp: """Affine rescale ``sample.input`` from ``[in_min, in_max]`` to ``[out_min, out_max]``. @@ -210,7 +211,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=out) -@configurable(category="op") +@configurable(category="op", group="numpy") class ReplaceNonFiniteOp: """Replace ``inf`` / ``-inf`` / ``nan`` entries in ``sample.input``. @@ -249,55 +250,110 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=np.where(non_finite, repl, arr)) -@configurable(category="op") +# ThresholdOp comparison selectors. Closed ``Literal``s (workspace "prefer closed +# Literals over bare strings" mandate) so FluxStudio / navigaitor render the choice +# as a dropdown and the allowed operators stay machine-introspectable via +# ``typing.get_args(...)``. Two distinct types because the lower bound only sensibly +# uses ``>`` / ``>=`` and the upper bound only ``<`` / ``<=``. +LowComparison = Literal[">", ">="] +HighComparison = Literal["<", "<="] + +# Operator dispatch. The dict keys are the single runtime source of truth's +# consumers — ``tests/test_ops.py`` pins ``set(_LOW_COMPARISONS) == get_args(LowComparison)`` +# (and likewise for high) so the map can never drift from the Literal. +_LOW_COMPARISONS: Dict[str, Callable[[Any, float], Any]] = {">": operator.gt, ">=": operator.ge} +_HIGH_COMPARISONS: Dict[str, Callable[[Any, float], Any]] = {"<": operator.lt, "<=": operator.le} + + +@configurable(category="op", group="numpy") class ThresholdOp: - """Threshold ``sample.input`` (ndarray) into a boolean mask: ``input > value``. + """Threshold ``sample.input`` (ndarray) into a boolean mask using one or both bounds. + + Which mask is produced depends on *which* bounds are set (presence-driven), and the + comparison applied for each is selected by ``low_op`` / ``high_op``: + + * only ``low_level`` → ``input low_level`` (values above the floor) + * only ``high_level`` → ``input high_level`` (values below the ceiling) + * both → both conditions AND-ed together (band-pass) + + ``low_op`` is ``">"`` (strict, the default) or ``">="`` (inclusive); ``high_op`` is + ``"<"`` (strict, the default) or ``"<="`` (inclusive). So the defaults yield the OPEN + interval ``low_level < input < high_level``, while ``low_op=">="`` + ``high_op="<="`` + yield the CLOSED interval ``low_level <= input <= high_level``. + + At least one of ``low_level`` / ``high_level`` MUST be provided; passing + neither raises ``ValueError`` at construction. - ``value`` is either a numeric literal or a string expression resolved via + Each bound is either a numeric literal or a string expression resolved via :func:`resolve_expression` against ``sample.metadata`` and ``os.environ``: - * ``5.5`` or ``"5.5"`` — fixed threshold + * ``5.5`` or ``"5.5"`` — fixed bound * ``"{reference_snr_level}"`` — looks up ``metadata["reference_snr_level"]`` * ``"-{reference_snr_level}"`` — negated lookup (the leading ``-`` is carried through ``float(...)`` after substitution) * ``"$REF_SNR"`` / ``"-$REF_SNR"`` — environment-variable lookup - Records the resolved threshold under ``metadata["threshold"]`` for traceability. + Records each resolved bound that was applied under ``metadata["threshold_low"]`` + / ``metadata["threshold_high"]`` for traceability. Args: - value: Threshold as a numeric literal or a string expression resolved against - ``sample.metadata`` / ``os.environ``. + low_level: Lower bound (numeric literal or expression) compared with ``low_op`` when set; + ``None`` disables the lower bound. + high_level: Upper bound (numeric literal or expression) compared with ``high_op`` when set; + ``None`` disables the upper bound. + low_op: Lower-bound comparison — ``">"`` (strict, default) or ``">="`` (inclusive). + high_op: Upper-bound comparison — ``"<"`` (strict, default) or ``"<="`` (inclusive). """ ACCEPTS = SampleType(input=_NDARRAY) PRODUCES = SampleType(input=ArrayType(dtype="bool", frameworks={"numpy"})) - def __init__(self, value: Union[float, int, str] = 0.0) -> None: - self.value = value - - def _resolve(self, sample: Sample) -> float: - if isinstance(self.value, (int, float)): - return float(self.value) - if not isinstance(self.value, str): - raise TypeError(f"ThresholdOp.value must be a number or expression string; got {type(self.value).__name__}") - resolved = resolve_expression(self.value, sample) + def __init__( + self, + low_level: Optional[Union[float, int, str]] = None, + high_level: Optional[Union[float, int, str]] = None, + low_op: LowComparison = ">", + high_op: HighComparison = "<", + ) -> None: + if low_level is None and high_level is None: + raise ValueError("ThresholdOp requires at least one of 'low_level' / 'high_level'") + self.low_level = low_level + self.high_level = high_level + self.low_op = low_op + self.high_op = high_op + + def _resolve(self, bound: Union[float, int, str], sample: Sample) -> float: + if isinstance(bound, (int, float)): + return float(bound) + if not isinstance(bound, str): + raise TypeError(f"ThresholdOp bounds must be a number or expression string; got {type(bound).__name__}") + resolved = resolve_expression(bound, sample) try: return float(resolved) except ValueError as exc: raise ValueError( - f"ThresholdOp: expression {self.value!r} resolved to {resolved!r}, " f"which is not a number" + f"ThresholdOp: expression {bound!r} resolved to {resolved!r}, " f"which is not a number" ) from exc def __call__(self, sample: Sample) -> Sample: arr = sample.input if not isinstance(arr, np.ndarray): raise TypeError(f"ThresholdOp expects an np.ndarray on sample.input, got {type(arr).__name__}") - threshold = self._resolve(sample) - sample.metadata["threshold"] = threshold - return sample._replace(input=arr > threshold) - - -@configurable(category="op") + mask: Optional[np.ndarray] = None + if self.low_level is not None: + low = self._resolve(self.low_level, sample) + sample.metadata["threshold_low"] = low + mask = _LOW_COMPARISONS[self.low_op](arr, low) + if self.high_level is not None: + high = self._resolve(self.high_level, sample) + sample.metadata["threshold_high"] = high + below = _HIGH_COMPARISONS[self.high_op](arr, high) + mask = below if mask is None else (mask & below) + assert mask is not None # guaranteed by the __init__ presence check + return sample._replace(input=mask) + + +@configurable(category="op", group="numpy") class ConnectedComponentsOp: """Label connected ``True`` regions of a boolean mask into bin-bbox tuples. diff --git a/dataflux/ops/parallel.py b/dataflux/ops/parallel.py index 2196d53..283a53d 100644 --- a/dataflux/ops/parallel.py +++ b/dataflux/ops/parallel.py @@ -29,7 +29,7 @@ from dataflux.sample import Sample -@configurable(category="op") +@configurable(category="op", group="compose") class Parallel: """Run an inner op sub-pipeline in a worker pool with bounded prefetch. diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py index 45a2f93..f80b11b 100644 --- a/dataflux/ops/stash.py +++ b/dataflux/ops/stash.py @@ -19,7 +19,7 @@ from dataflux.sample import Sample -@configurable(category="op") +@configurable(category="op", group="structure") class StashInputOp: """Copy ``sample.input`` into ``metadata[key]``; ``sample.input`` unchanged. @@ -40,7 +40,7 @@ def __call__(self, sample: Sample) -> Sample: return sample -@configurable(category="op") +@configurable(category="op", group="structure") class UnstashInputOp: """Set ``sample.input := metadata[key]``. diff --git a/dataflux/ops/swap.py b/dataflux/ops/swap.py index 757d1cd..9542270 100644 --- a/dataflux/ops/swap.py +++ b/dataflux/ops/swap.py @@ -9,7 +9,7 @@ from dataflux.sample import Sample -@configurable(category="op") +@configurable(category="op", group="structure") class SwapInputTargetOp: """Exchange ``sample.input`` ↔ ``sample.target``. Metadata unchanged.""" diff --git a/dataflux/ops/target.py b/dataflux/ops/target.py new file mode 100644 index 0000000..cc2b95a --- /dev/null +++ b/dataflux/ops/target.py @@ -0,0 +1,137 @@ +"""Move and encode the supervised ``target`` field. + +Companions to the input↔metadata movers (:class:`~dataflux.ops.stash.StashInputOp` / +:class:`~dataflux.ops.stash.UnstashInputOp`) and +:class:`~dataflux.ops.swap.SwapInputTargetOp`: + +* :class:`MetadataToTargetOp` moves a value from ``metadata`` onto ``sample.target``. +* :class:`EncodeTargetOp` / :class:`DecodeTargetOp` map ``sample.target`` through an + explicit lookup and back — the declarative analogue of scikit-learn's + ``LabelEncoder``. The label→id mapping is pinned in config, NOT fitted from + whatever labels happen to appear, so train / eval / predict share one identical + ordering. + +These are deliberately small, value-agnostic plumbing ops (no ``ACCEPTS`` / +``PRODUCES`` contract, like ``copy`` / ``swap`` / ``stash``). The encoded value is +written verbatim (e.g. a plain ``int``); wrap it into a framework tensor downstream +(e.g. a collate function) when a loss needs one. +""" + +from typing import Any, Dict, Optional + +from confluid import configurable + +from dataflux.sample import Sample + + +def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: Any, op_name: str) -> Any: + """Return ``mapping[value]``, or ``default`` when missing and ``ignore_unknown``. + + Shared by :class:`EncodeTargetOp` / :class:`DecodeTargetOp`. A plain + module-level function (NOT a base class) so the ops stay independent + callables — DataFlux Functional Purity. + """ + if value in mapping: + return mapping[value] + if ignore_unknown: + return default + sample_keys = list(mapping)[:8] + suffix = "..." if len(mapping) > 8 else "" + raise KeyError( + f"{op_name}: value {value!r} not in mapping (keys: {sample_keys}{suffix}). " + "Pass ignore_unknown=True to substitute `default` instead." + ) + + +@configurable(category="op", group="structure") +class MetadataToTargetOp: + """Set ``sample.target := metadata[key]``; optionally copy it to ``metadata[target_key]``. + + The metadata→target counterpart of :class:`~dataflux.ops.stash.StashInputOp` / + :class:`~dataflux.ops.stash.UnstashInputOp` (which move input↔metadata). Typical + use: a raw label rides in ``metadata`` and must become the supervised ``target`` + before :class:`EncodeTargetOp` overwrites it with a class id. + + Args: + key: Metadata key to read the value from into ``sample.target``. + target_key: When set, the value is also written to ``metadata[target_key]`` + (so the raw label survives a later ``EncodeTargetOp`` and can be decoded + back). ``None`` (default) leaves ``metadata`` untouched. + """ + + def __init__(self, key: str, target_key: Optional[str] = None) -> None: + self.key = str(key) + self.target_key = str(target_key) if target_key is not None else None + + def __call__(self, sample: Sample) -> Sample: + if self.key not in sample.metadata: + raise KeyError( + f"MetadataToTargetOp: sample.metadata has no key {self.key!r}. " + f"Available keys: {sorted(sample.metadata)}" + ) + value = sample.metadata[self.key] + if self.target_key is not None: + sample.metadata[self.target_key] = value + return sample._replace(target=value) + + +@configurable(category="op", group="structure") +class EncodeTargetOp: + """Encode ``sample.target`` through an explicit lookup ``mapping``. + + The declarative analogue of scikit-learn's ``LabelEncoder``: maps a raw target + (typically a string label) to its class id via a config-pinned ``mapping``. + Pinning the mapping — rather than fitting it from whatever labels appear — keeps + train / eval / predict on one identical label→id ordering. The plain mapping value + is written (framework-agnostic); tensorize the target downstream when a loss needs it. + + Args: + mapping: Lookup from raw target → encoded value, e.g. ``{"DJI AVATA2": 2, ...}``. + Must be non-empty. + ignore_unknown: When ``False`` (default), raise on a target missing from + ``mapping``; when ``True``, substitute ``default``. + default: Value written for an unknown target when ``ignore_unknown=True``. + Defaults to ``0``. + """ + + def __init__(self, mapping: Dict[Any, Any], ignore_unknown: bool = False, default: Any = 0) -> None: + if not mapping: + raise ValueError("EncodeTargetOp: mapping must contain at least one entry.") + self.mapping = dict(mapping) + self.ignore_unknown = bool(ignore_unknown) + self.default = default + + def __call__(self, sample: Sample) -> Sample: + encoded = _lookup(sample.target, self.mapping, self.ignore_unknown, self.default, "EncodeTargetOp") + return sample._replace(target=encoded) + + +@configurable(category="op", group="structure") +class DecodeTargetOp: + """Decode ``sample.target`` through a lookup ``mapping`` (inverse of :class:`EncodeTargetOp`). + + Maps an encoded target (e.g. an integer class id) back to its label (e.g. a class + name) — the readback half used in prediction / reporting. + + Args: + mapping: Lookup from encoded value → decoded value, e.g. ``{2: "DJI AVATA2", ...}``. + Must be non-empty. + ignore_unknown: When ``False`` (default), raise on a target missing from + ``mapping``; when ``True``, substitute ``default``. + default: Value written for an unknown target when ``ignore_unknown=True``. + Defaults to ``None``. + """ + + def __init__(self, mapping: Dict[Any, Any], ignore_unknown: bool = False, default: Any = None) -> None: + if not mapping: + raise ValueError("DecodeTargetOp: mapping must contain at least one entry.") + self.mapping = dict(mapping) + self.ignore_unknown = bool(ignore_unknown) + self.default = default + + def __call__(self, sample: Sample) -> Sample: + decoded = _lookup(sample.target, self.mapping, self.ignore_unknown, self.default, "DecodeTargetOp") + return sample._replace(target=decoded) + + +__all__ = ["MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp"] diff --git a/dataflux/ops/tee.py b/dataflux/ops/tee.py index 878565f..b00c6bb 100644 --- a/dataflux/ops/tee.py +++ b/dataflux/ops/tee.py @@ -17,7 +17,7 @@ from dataflux.sample import Sample -@configurable(category="op") +@configurable(category="op", group="compose") class Tee: """Run N op-list branches sequentially on the same sample / metadata. diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index 56907b1..70dcf38 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -11,7 +11,7 @@ _TORCH_FLOAT = ArrayType(dtype="floating", frameworks={"torch"}) -@configurable(category="op") +@configurable(category="op", group="torch") class ToTensorOp: """ Converts input (PIL Image, NumPy array, etc.) to a Torch Tensor. @@ -56,7 +56,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=tensor) -@configurable(category="op") +@configurable(category="op", group="torch") class RescaleOp: """Affine rescale a torch.Tensor from ``[in_min, in_max]`` to ``[out_min, out_max]``. @@ -106,7 +106,7 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=out) -@configurable(category="op") +@configurable(category="op", group="torch") class StandardizeOp: """ Standardizes tensor values with given mean and standard deviation. diff --git a/dataflux/paired.py b/dataflux/paired.py index bd3e30a..7a2d2f8 100644 --- a/dataflux/paired.py +++ b/dataflux/paired.py @@ -1,6 +1,30 @@ -"""Keyed-join source pairing a primary DataSource with a secondary annotation mapping.""" - -from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Union +"""Re-join raw data samples with a sidecar annotation store (the annotation loop). + +The recurring pattern this solves: you have raw data samples (RFUAV I/Q windows, +images, …) coming out of a ``DataSource``, and *separately* a sidecar store of +annotations covering some of them — typically a LabelStudio export that annotaide +writes as a ``sample_id -> record`` JSON mapping. :class:`AnnotationJoinSource` +re-joins the two by a key function so each matched annotation record is attached +to ``Sample.metadata``, ready for training. + + raw data ──annotate (LabelStudio)──▶ annotation store ──AnnotationJoinSource──▶ annotated samples +""" + +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + Literal, + Optional, + Protocol, + Sequence, + Tuple, + Union, + cast, + runtime_checkable, +) from confluid import configurable from logflow import get_logger @@ -10,39 +34,72 @@ logger = get_logger(__name__) -VALID_POLICIES = ("left_outer", "inner", "right_driven") +# Join policy is a closed set. As a Literal it is enforced two ways with no extra +# code: static checkers reject bad values, and Confluid's @configurable validates +# it through pydantic at construction (both the Python and YAML/load paths), so a +# bad policy fails before __init__ runs. It also renders as an enum dropdown in +# the navigaitor form-spec. +Policy = Literal["left_outer", "inner", "right_driven"] + + +@runtime_checkable +class AnnotationStore(Protocol): + """The read contract :class:`AnnotationJoinSource` needs from its annotation store: + membership + lookup + key enumeration (``key -> record``). + + Structural (a ``Protocol``), so it does NOT couple dataflux to annotaide: + annotaide's ``JSONFileAnnotationStore`` satisfies it — and so does a plain + ``dict`` — without any import or inheritance. The write side (``save`` / + ``delete``) lives in annotaide, not here. + + It is ``@runtime_checkable`` on purpose: Confluid's ``@configurable`` layer + isinstance-validates it at construction, so a non-conforming ``annotations`` + is rejected before ``__init__`` runs (the type does the enforcement — no + manual shape guard needed, mirroring how the ``policy`` ``Literal`` is + validated). Every real store (``dict``, ``JSONFileAnnotationStore``) provides + all three methods. + """ + + def __contains__(self, key: str) -> bool: ... + + def __getitem__(self, key: str) -> Dict[str, Any]: ... + + def keys(self) -> Iterable[str]: ... @configurable -class PairedSource: - """Pair a primary DataSource with a secondary annotation mapping via a key function. +class AnnotationJoinSource: + """Join a data source with a sidecar annotation store via a key function. Produces ``Sample`` values where the matched annotation record is flattened into - ``Sample.metadata``. Supports three join policies: + ``Sample.metadata``. Three join policies cover the scenarios we actually see: - - ``left_outer``: iterate primary; attach annotation when the key matches, - otherwise emit the sample unannotated. Preserves primary's ``__len__`` and - ``__getitem__``. (Scenario A.) + - ``left_outer``: iterate ``data``; attach the annotation when the key matches, + otherwise emit the sample unannotated. Preserves the data source's ``__len__`` + and ``__getitem__``. (Every sample, annotated where available.) - ``inner``: same as left_outer, filtered to annotated samples only. - (Scenario B.) - - ``right_driven``: iterate ``secondary.keys()``; resolve each primary sample - via ``primary_resolver(key, primary)``. Use when annotations are sparse. + (The labeled subset.) + - ``right_driven``: iterate ``annotations.keys()``; resolve each data sample + via ``data_resolver(key, data)``. Use when annotations are sparse relative + to the data. Coarser-granularity joins are expressed by returning a coarser key from - ``key_fn`` so multiple primary samples map to the same annotation record. + ``key_fn`` so multiple data samples map to the same annotation record. Use ``extract_fn`` to project the record down to each sample's scope (e.g. trim a pack-level time-ranged annotation to a single window). Returning ``None`` from ``extract_fn`` marks the sample as unannotated. Args: - primary: Any iterable (or ``DataSource``) yielding raw items that + data: The data source — any iterable (or ``DataSource``) yielding raw items that ``Sample.from_any`` can coerce into samples. - secondary: For ``left_outer``/``inner``, a mapping-like object supporting - ``__contains__`` and ``__getitem__``. For ``right_driven``, an object - also supporting ``keys()``. + annotations: The annotation store — a read-mapping (``key -> record``) + satisfying :class:`AnnotationStore` (``__contains__`` + ``__getitem__`` + + ``keys()``); validated at construction. A plain ``dict`` or annotaide's + ``JSONFileAnnotationStore`` qualifies. key_fn: ``"module:function"`` path (or a callable) producing the join key - from a sample. Signature: ``(sample: Sample) -> str``. - policy: One of ``"left_outer"``, ``"inner"``, ``"right_driven"``. + from a sample. Signature: ``(sample: Sample) -> str``. Stored as a path so + the source round-trips through Confluid YAML. + policy: Join policy — one of ``"left_outer"``, ``"inner"``, ``"right_driven"``. extract_fn: Optional ``"module:function"`` path (or callable) called as ``extract_fn(record, sample) -> dict | None`` to project the record per sample. Returning ``None`` marks the sample unannotated. @@ -50,52 +107,43 @@ class PairedSource: flattening into ``Sample.metadata``. store_full_under: If set, also stash the (extracted) record under ``Sample.metadata[store_full_under]``. - primary_resolver: Required for ``right_driven``. ``"module:function"`` - path (or callable) invoked as ``primary_resolver(key, primary)`` - to fetch the primary sample for a given annotation key. + data_resolver: Required for ``right_driven``. ``"module:function"`` path + (or callable) invoked as ``data_resolver(key, data)`` to fetch the data + sample for a given annotation key. """ def __init__( self, - primary: Any, - secondary: Any, + data: Iterable[Any], + annotations: AnnotationStore, key_fn: Union[str, Callable[[Sample], str]], - policy: str = "left_outer", + policy: Policy = "left_outer", extract_fn: Optional[Union[str, Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]]] = None, prefix: str = "", store_full_under: Optional[str] = None, - primary_resolver: Optional[Union[str, Callable[[str, Any], Any]]] = None, + # data arg is Any (not Iterable[Any]): resolvers are written against a + # concrete source type (e.g. RFUAVSource) and contravariance would reject + # those signatures against a broader annotation. + data_resolver: Optional[Union[str, Callable[[str, Any], Any]]] = None, ) -> None: - if policy not in VALID_POLICIES: - raise ValueError(f"Invalid policy {policy!r}; must be one of {VALID_POLICIES}") - - if policy in ("left_outer", "inner"): - if not hasattr(secondary, "__contains__") or not hasattr(secondary, "__getitem__"): - raise TypeError( - f"policy={policy!r} requires secondary to support __contains__ and __getitem__; " - f"got {type(secondary).__name__}" - ) - - if policy == "right_driven": - if primary_resolver is None: - raise ValueError("policy='right_driven' requires primary_resolver") - if not hasattr(secondary, "keys"): - raise TypeError( - f"policy='right_driven' requires secondary to support keys(); " f"got {type(secondary).__name__}" - ) - - self.primary = primary - self.secondary = secondary + # `annotations` shape is enforced by the AnnotationStore Protocol via + # pydantic at construction. Only the policy-conditional requirement that + # right_driven needs a resolver is checked here (a Protocol can't express it). + if policy == "right_driven" and data_resolver is None: + raise ValueError("policy='right_driven' requires data_resolver") + + self.data = data + self.annotations = annotations self.key_fn = get_callable_path(key_fn) if callable(key_fn) else key_fn self.policy = policy self.extract_fn = get_callable_path(extract_fn) if callable(extract_fn) else extract_fn self.prefix = prefix self.store_full_under = store_full_under - self.primary_resolver = get_callable_path(primary_resolver) if callable(primary_resolver) else primary_resolver + self.data_resolver = get_callable_path(data_resolver) if callable(data_resolver) else data_resolver self._key_fn_cache: Optional[Callable[[Sample], str]] = None self._extract_fn_cache: Optional[Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]] = None - self._primary_resolver_cache: Optional[Callable[[str, Any], Any]] = None + self._data_resolver_cache: Optional[Callable[[str, Any], Any]] = None self._inner_length: Optional[int] = None @property @@ -115,12 +163,12 @@ def _resolved_extract_fn( return self._extract_fn_cache @property - def _resolved_primary_resolver(self) -> Callable[[str, Any], Any]: - if self.primary_resolver is None: - raise ValueError("primary_resolver is not set") - if self._primary_resolver_cache is None: - self._primary_resolver_cache = resolve_callable(self.primary_resolver) - return self._primary_resolver_cache + def _resolved_data_resolver(self) -> Callable[[str, Any], Any]: + if self.data_resolver is None: + raise ValueError("data_resolver is not set") + if self._data_resolver_cache is None: + self._data_resolver_cache = resolve_callable(self.data_resolver) + return self._data_resolver_cache def _attach(self, sample: Sample, record: Optional[Dict[str, Any]], key: str) -> Sample: metadata = dict(sample.metadata) @@ -137,9 +185,9 @@ def _attach(self, sample: Sample, record: Optional[Dict[str, Any]], key: str) -> def _lookup(self, sample: Sample) -> Tuple[str, Optional[Dict[str, Any]]]: key = self._resolved_key_fn(sample) - if key not in self.secondary: + if key not in self.annotations: return key, None - record: Optional[Dict[str, Any]] = self.secondary[key] + record: Optional[Dict[str, Any]] = self.annotations[key] extract_fn = self._resolved_extract_fn if extract_fn is not None and record is not None: record = extract_fn(record, sample) @@ -150,7 +198,7 @@ def __iter__(self) -> Iterator[Sample]: yield from self._iter_right_driven() return - for item in self.primary: + for item in self.data: sample = Sample.from_any(item) key, record = self._lookup(sample) if self.policy == "inner" and record is None: @@ -158,11 +206,11 @@ def __iter__(self) -> Iterator[Sample]: yield self._attach(sample, record, key) def _iter_right_driven(self) -> Iterator[Sample]: - resolver = self._resolved_primary_resolver - for key in self.secondary.keys(): - raw = resolver(key, self.primary) + resolver = self._resolved_data_resolver + for key in self.annotations.keys(): + raw = resolver(key, self.data) sample = Sample.from_any(raw) - record: Optional[Dict[str, Any]] = self.secondary[key] + record: Optional[Dict[str, Any]] = self.annotations[key] extract_fn = self._resolved_extract_fn if extract_fn is not None and record is not None: record = extract_fn(record, sample) @@ -172,14 +220,21 @@ def _iter_right_driven(self) -> Iterator[Sample]: yield self._attach(sample, record, key) def __len__(self) -> int: + from collections.abc import Sized + if self.policy == "left_outer": - return len(self.primary) + # left_outer preserves the data source's length; it must be sized. + if not isinstance(self.data, Sized): + raise TypeError( + f"policy='left_outer' requires a sized data source for len(); " f"got {type(self.data).__name__}" + ) + return len(self.data) if self.policy == "right_driven": - return len(list(self.secondary.keys())) + return len(list(self.annotations.keys())) if self._inner_length is None: count = 0 - for item in self.primary: + for item in self.data: sample = Sample.from_any(item) _, record = self._lookup(sample) if record is not None: @@ -190,8 +245,11 @@ def __len__(self) -> int: def __getitem__(self, index: int) -> Sample: if self.policy != "left_outer": raise TypeError(f"__getitem__ is only supported for policy='left_outer'; got {self.policy!r}") - if not hasattr(self.primary, "__getitem__"): - raise TypeError("primary must support __getitem__ for PairedSource.__getitem__") - sample = Sample.from_any(self.primary[index]) + # Duck-typed on __getitem__ (not isinstance Sequence): workspace sources + # like RFUAVSource / HuggingFaceSource expose __getitem__ without + # subclassing collections.abc.Sequence. + if not hasattr(self.data, "__getitem__"): + raise TypeError("data must support __getitem__ for AnnotationJoinSource.__getitem__") + sample = Sample.from_any(cast(Sequence[Any], self.data)[index]) key, record = self._lookup(sample) return self._attach(sample, record, key) diff --git a/dataflux/projection.py b/dataflux/projection.py index 0110677..4faaacf 100644 --- a/dataflux/projection.py +++ b/dataflux/projection.py @@ -23,14 +23,23 @@ make every ``Flux`` look classification-capable to duck-typed consumers. """ -from typing import Any, Collection, Iterator, Protocol, runtime_checkable +from typing import Any, Collection, Iterator, Literal, Protocol, Tuple, get_args, runtime_checkable from dataflux.sample import Sample -INPUT = "input" -TARGET = "target" -METADATA = "metadata" -_FIELDS = (INPUT, TARGET, METADATA) +#: The projectable :class:`~dataflux.sample.Sample` fields, as a *closed* +#: ``Literal`` rather than a bare ``str``. Typing the field set this way lets +#: UIs, form-spec builders, and MCP tool schemas enumerate the allowed values +#: straight from the annotation (``typing.get_args(ProjectionField)``) and lets +#: a type checker reject a typo at the call site — the Literal-over-strings +#: discipline the workspace mandate calls for, applied because the set is fixed +#: and short. +ProjectionField = Literal["input", "target", "metadata"] + +INPUT: ProjectionField = "input" +TARGET: ProjectionField = "target" +METADATA: ProjectionField = "metadata" +_FIELDS: Tuple[ProjectionField, ...] = get_args(ProjectionField) @runtime_checkable @@ -44,10 +53,10 @@ class SupportsProjection(Protocol): (``{}`` for ``metadata``). """ - def project(self, fields: Collection[str]) -> Iterator[Sample]: ... + def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: ... -def project(source: Any, fields: Collection[str]) -> Iterator[Sample]: +def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample]: """Yield :class:`Sample` records from ``source`` carrying only ``fields``. Uses the source's own ``project`` when it implements @@ -136,6 +145,7 @@ def num_classes(source: Any) -> int: __all__ = [ + "ProjectionField", "SupportsProjection", "project", "iter_inputs", diff --git a/dataflux/sources.py b/dataflux/sources.py index 6c0f04a..a34fb5c 100644 --- a/dataflux/sources.py +++ b/dataflux/sources.py @@ -1,5 +1,6 @@ +import bisect import random -from typing import Any, Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Literal, Optional, get_args from confluid import configurable from logflow import get_logger @@ -8,6 +9,45 @@ logger = get_logger(__name__) +# Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer +# closed Literals over bare strings — self-documenting + machine-introspectable by UIs / +# navigaitor form-spec / MCP schemas via ``typing.get_args``). The runtime-validation tuple +# is derived from the Literal so there is ONE source of truth — never restate the values. +SplitName = Literal["train", "val", "test"] +_SPLIT_NAMES = get_args(SplitName) + +# Sentinel for ``HuggingFaceSource.metadata_features`` meaning "every dataset column except the +# input/target features" — the full-traceability option, kept OPT-IN (``None`` / ``[]`` still = no +# extra metadata) so existing configs are unaffected. Resolved against the loaded dataset's +# ``column_names`` at construction. Accepted bare (``"*"``) or as the one-element list (``["*"]``); +# FluxStudio's metadata picker offers it as a selectable "*" entry. +METADATA_ALL_FEATURES = "*" + + +def _resolve_metadata_features( + requested: Optional[Any], + column_names: Optional[List[str]], + input_feature: str, + target_feature: str, +) -> List[str]: + """Resolve a ``metadata_features`` spec into a concrete, order-preserving column list. + + ``None`` / ``[]`` -> ``[]`` (no extra metadata — the backward-compatible default). The sentinel + ``"*"`` (bare or inside a list) -> every column in ``column_names`` except ``input_feature`` / + ``target_feature`` (full traceability). An explicit list of names is used verbatim. ``"*"`` may + be combined with extra names (union, order-preserving: the "rest" first, then the extras). + """ + if not requested: + return [] + if isinstance(requested, str): + requested = [requested] + if METADATA_ALL_FEATURES not in requested: + return list(requested) + excluded = {input_feature, target_feature} + rest = [c for c in (column_names or []) if c not in excluded] + extras = [r for r in requested if r != METADATA_ALL_FEATURES and r not in excluded and r not in rest] + return rest + extras + @configurable(category="source") class HuggingFaceSource: @@ -15,19 +55,25 @@ class HuggingFaceSource: DataFlux Source for Hugging Face Datasets. Configurable mapping of dataset features to DataFlux Sample triplets. + Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md + "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and + does NO functional work — ``HuggingFaceSource()`` is valid, and the dataset is downloaded + only on first access to :attr:`dataset` (cached thereafter; reset ``_dataset`` to reload). + ``path`` is therefore optional at construction and validated lazily when the data is needed. + Args: path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. split: HF split name (``train`` / ``validation`` / ``test`` / etc.). input_feature: Dataset feature column to map onto ``Sample.input``. target_feature: Dataset feature column to map onto ``Sample.target``. - metadata_features: Feature columns to preserve on ``Sample.metadata`` (``None`` = none). + metadata_features: Columns onto ``Sample.metadata``; ``None``=none, ``"*"``=all but input/target, else a list. count: Optional cap on the number of samples yielded (useful for fast smoke runs). name: Optional HF subset/config name (e.g. for multi-config datasets). """ def __init__( self, - path: str, + path: str = "", split: str = "train", input_feature: str = "image", target_feature: str = "label", @@ -36,23 +82,61 @@ def __init__( name: Optional[str] = None, **kwargs: Any, ) -> None: - from datasets import load_dataset - + # Lazy constructor: store config only — never load here. Real work (the network/disk + # download) is deferred to the ``dataset`` property so the object is cheap to build and + # configurable post-construction. self.path = path self.split = split self.input_feature = input_feature self.target_feature = target_feature - self.metadata_features = metadata_features or [] + # Stored as the RAW spec (``None`` / ``"*"`` / list) — resolved against the loaded dataset's + # columns lazily by the ``resolved_metadata_features`` property, not eagerly here. + self.metadata_features = metadata_features self.count = count - - logger.info(f"HuggingFaceSource: Loading {path} ({split})...") - self._dataset = load_dataset(path, name=name, split=split, **kwargs) + self.name = name + # Extra kwargs forwarded verbatim to ``datasets.load_dataset`` at load time (e.g. ``token``, + # ``trust_remote_code``). Captured now, applied lazily in the ``dataset`` property. + self._load_kwargs = dict(kwargs) + # Lazy cache for the materialized dataset (see the ``dataset`` property). + self._dataset: Any = None + + @property + def dataset(self) -> Any: + """The HF dataset, loaded on first access and cached. Resetting ``_dataset`` to None reloads. + + Raises ``ValueError`` if ``path`` was never set — the zero-arg constructor allows building an + unconfigured source, but materializing one without a dataset id cannot succeed. + """ + if self._dataset is None: + if not self.path: + raise ValueError( + "HuggingFaceSource.path is empty — set it (constructor arg, YAML, or configure()) " + "before iterating or indexing the source." + ) + from datasets import load_dataset + + logger.info(f"HuggingFaceSource: Loading {self.path} ({self.split})...") + self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self._load_kwargs) + return self._dataset + + @property + def resolved_metadata_features(self) -> List[str]: + """``metadata_features`` resolved against the live dataset's columns (expands the ``"*"`` sentinel). + + Lazy because the ``"*"`` expansion needs the loaded dataset's ``column_names``; ``None`` / ``[]`` + stays "no extra metadata" (backward-compatible). + """ + return _resolve_metadata_features( + self.metadata_features, getattr(self.dataset, "column_names", None), self.input_feature, self.target_feature + ) def __iter__(self) -> Iterator[Sample]: counter = 0 - limit = self.count or len(self._dataset) + dataset = self.dataset + metadata_features = self.resolved_metadata_features + limit = self.count or len(dataset) - for item in self._dataset: + for item in dataset: if counter >= limit: break @@ -63,7 +147,7 @@ def __iter__(self) -> Iterator[Sample]: target_val = item.get(self.target_feature) # 3. Build Metadata - metadata = {f: item.get(f) for f in self.metadata_features} + metadata = {f: item.get(f) for f in metadata_features} metadata["hf_path"] = self.path metadata["hf_split"] = self.split @@ -71,8 +155,8 @@ def __iter__(self) -> Iterator[Sample]: counter += 1 def __getitem__(self, index: int) -> Sample: - item = self._dataset[index] - metadata = {f: item.get(f) for f in self.metadata_features} + item = self.dataset[index] + metadata = {f: item.get(f) for f in self.resolved_metadata_features} metadata["hf_path"] = self.path metadata["hf_split"] = self.split return Sample( @@ -87,147 +171,211 @@ def __len__(self) -> int: # would report 0 for the common "0 == unlimited" case, making the source # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) # even though iteration yields every sample. - return self.count or len(self._dataset) + return self.count or len(self.dataset) -@configurable(category="engine") +@configurable(category="source") class DatasetSplit: """ - Selects a subset view of an indexable source (e.g. ``HuggingFaceSource``). + Splits an indexable source into reproducible ``train`` / ``val`` / ``test`` views. - Supports three mutually exclusive modes: + A ``source`` (it yields ``Sample``s and is wired into a trainer's ``source:`` slot), + not an engine — it applies no ops, it just exposes a reproducible partition of another + source. (For a contiguous index slice use :class:`RangeSource`; to concatenate several + sources use :class:`ConcatSource`.) - 1. **Fraction mode** — pick a reproducible train/val split from a single source. - Pass ``split`` (``"train"`` or ``"val"``), ``val_fraction``, and ``seed``. - Two ``DatasetSplit`` instances sharing a source (by Python identity or - by ``!ref:`` in YAML) and the *same* ``seed`` + ``val_fraction`` yield - disjoint, complementary views. With Confluid ``!ref:``, the underlying - source is loaded exactly once and shared by identity between the splits. + **Property API (preferred).** Configure ONE ``DatasetSplit`` with ``seed`` and the + held-out fraction(s) (``val_fraction`` and/or ``test_fraction``) and read the three + cached view sources off it:: - 2. **Range mode** — plain index slice ``[start:end)`` over the source. - Pass ``start`` and/or ``end``. + split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) + split.train # ≈80% — the remainder + split.val # ≈10% + split.test # ≈10% - 3. **Passthrough** — no split args yields a full view (rarely useful, - mostly for symmetry in YAML templates). - - The wrapped source must implement ``__len__`` and ``__getitem__``. - Lazy evaluation is preserved: only index arithmetic happens at - construction time; samples are produced on demand. + The views are disjoint and complementary, computed once (cached) over a single + deterministic shuffle, so the underlying source is consumed once. In Confluid YAML the + views are reachable by **attribute reference** — ``!ref:my_split.train`` / ``.val`` / + ``.test`` — and because two ``!ref:`` to the same key flow the *same* instance, the + partition and the source load are shared across all three references:: - Args: - source: The underlying indexable source. - split: ``"train"`` or ``"val"`` (fraction mode only). - val_fraction: Fraction of samples assigned to the ``"val"`` view - (fraction mode). Must be in ``(0, 1)``. - seed: Seed for the deterministic shuffle (fraction mode). - Required when ``val_fraction`` is set so that sibling splits - stay consistent. - start: Inclusive start index (range mode). - end: Exclusive end index (range mode). - - Example (fraction mode, YAML):: - - hf_train: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: train - - train_set: !class:dataflux.sources.DatasetSplit() + my_split: !class:dataflux.sources.DatasetSplit() source: !ref:hf_train - split: train val_fraction: 0.1 + test_fraction: 0.1 seed: 42 - val_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 + train_set: !class:dataflux.core.Flux() + source: !ref:my_split.train + val_set: !class:dataflux.core.Flux() + source: !ref:my_split.val - Alternative (no shared load, HuggingFace native slicing):: + **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one + view (``split=None`` ⇒ ``train``), so it is directly usable as a single ``source:``. - # Loads the HF dataset twice — kept for clarity if sharing is not desired. - train_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[:90%]" - val_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[90%:]" + Omit ``test_fraction`` for a plain two-way train/val split; omit both fractions for a + degenerate split where ``train`` is the whole source and ``val`` / ``test`` are empty. + + The wrapped source must implement ``__len__`` and ``__getitem__``. Lazy: only index + arithmetic happens up front; samples are produced on demand. + + Args: + source: The underlying indexable source. + split: View this iterates as a source — ``train`` / ``val`` / ``test`` (``None`` ⇒ ``train``). + val_fraction: Fraction of samples assigned to the ``val`` view. Must be in ``(0, 1)``. + test_fraction: Fraction of samples assigned to the ``test`` view. Must be in ``(0, 1)``. + seed: Seed for the deterministic shuffle. Required when any fraction is set. """ def __init__( self, source: Any, - split: Optional[str] = None, + split: Optional[SplitName] = None, val_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, seed: Optional[int] = None, - start: Optional[int] = None, - end: Optional[int] = None, ) -> None: if not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): raise TypeError( "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" ) - - fraction_args = val_fraction is not None or split is not None - range_args = start is not None or end is not None - if fraction_args and range_args: + if split is not None and split not in _SPLIT_NAMES: + raise ValueError(f"split must be one of {_SPLIT_NAMES}; got {split!r}") + if (val_fraction is not None or test_fraction is not None) and seed is None: + raise ValueError("DatasetSplit requires `seed` when a fraction is set, so the partition is reproducible.") + if val_fraction is not None and not (0.0 < val_fraction < 1.0): + raise ValueError(f"val_fraction must be in (0, 1); got {val_fraction}") + if test_fraction is not None and not (0.0 < test_fraction < 1.0): + raise ValueError(f"test_fraction must be in (0, 1); got {test_fraction}") + if (val_fraction or 0.0) + (test_fraction or 0.0) >= 1.0: raise ValueError( - "DatasetSplit accepts either fraction-mode args (split, val_fraction, seed) " - "or range-mode args (start, end), not both." + "val_fraction + test_fraction must be < 1 (to leave a non-empty train split); " + f"got val_fraction={val_fraction}, test_fraction={test_fraction}" ) self.source = source self.split = split self.val_fraction = val_fraction + self.test_fraction = test_fraction self.seed = seed - self.start = start - self.end = end + # Cache of materialized split views. Underscore-prefixed so confluid's + # vars(obj)-based discovery / dump ignores it (the `train`/`val`/`test` + # @property descriptors live on the class, not in vars(obj), so they never + # surface as configurable attributes either). + self._views: Dict[str, "_SplitView"] = {} + + def _partition(self) -> Dict[str, List[int]]: + """Deterministically partition the source indices into ``train`` / ``val`` / ``test``. + + One shuffle seeded by ``seed`` (skipped when no fraction is set, so the degenerate + "all train" case keeps source order); layout is ``[train | val | test]``. ``max(1, …)`` + guarantees a held-out split gets at least one sample on tiny sources. + """ + n = len(self.source) + val_fraction = self.val_fraction or 0.0 + test_fraction = self.test_fraction or 0.0 + shuffled = list(range(n)) + if val_fraction or test_fraction: + random.Random(self.seed).shuffle(shuffled) + val_count = max(1, int(round(n * val_fraction))) if val_fraction else 0 + test_count = max(1, int(round(n * test_fraction))) if test_fraction else 0 + train_count = max(0, n - val_count - test_count) + return { + "train": shuffled[:train_count], + "val": shuffled[train_count : train_count + val_count], + "test": shuffled[train_count + val_count :], + } + + def _view(self, split: SplitName) -> "_SplitView": + if split not in self._views: + self._views[split] = _SplitView(self.source, self._partition()[split]) + return self._views[split] + + @property + def train(self) -> "_SplitView": + """Cached training-split view (the remainder after ``val`` / ``test`` are held out).""" + return self._view("train") + + @property + def val(self) -> "_SplitView": + """Cached validation-split view (≈ ``val_fraction`` of the source).""" + return self._view("val") + + @property + def test(self) -> "_SplitView": + """Cached test-split view (≈ ``test_fraction`` of the source).""" + return self._view("test") - self._indices: List[int] = self._compute_indices() - logger.debug( - "DatasetSplit: mode=%s size=%d source_size=%d", - "fraction" if fraction_args else ("range" if range_args else "passthrough"), - len(self._indices), - len(source), - ) + def __iter__(self) -> Iterator[Sample]: + return iter(self._view(self.split or "train")) - def _compute_indices(self) -> List[int]: - n = len(self.source) + def __getitem__(self, index: int) -> Sample: + return self._view(self.split or "train")[index] + + def __len__(self) -> int: + return len(self._view(self.split or "train")) + + +class _SplitView: + """An indexable view of ``source`` restricted (and reordered) to ``indices``. + + Internal to :class:`DatasetSplit` — produced by its ``train`` / ``val`` / ``test`` + properties (and reachable in Confluid YAML via ``!ref:my_split.train``). Deliberately + NOT a ``@configurable``: it is never constructed directly in a config, only read off a + live ``DatasetSplit`` instance, so it carries no discovery surface of its own. + """ + + def __init__(self, source: Any, indices: List[int]) -> None: + self.source = source + self.indices = indices + + def __iter__(self) -> Iterator[Sample]: + for idx in self.indices: + yield Sample.from_any(self.source[idx]) + + def __getitem__(self, index: int) -> Sample: + return Sample.from_any(self.source[self.indices[index]]) + + def __len__(self) -> int: + return len(self.indices) + + +@configurable(category="source") +class RangeSource: + """A contiguous index slice ``[start:end)`` over an indexable source. + + The plain-slice counterpart to :class:`DatasetSplit` (which shuffles + partitions) — + extracted from DatasetSplit's old "range mode". Negative ``start`` / ``end`` count from + the end; both are clamped to ``[0, len(source)]``. Lazy: only index arithmetic happens + up front; samples are produced on demand. - # Fraction mode - if self.val_fraction is not None or self.split is not None: - if self.val_fraction is None: - raise ValueError("DatasetSplit fraction mode requires `val_fraction`.") - if self.seed is None: - raise ValueError("DatasetSplit fraction mode requires `seed` so that sibling splits stay consistent.") - if not (0.0 < self.val_fraction < 1.0): - raise ValueError(f"val_fraction must be in (0, 1); got {self.val_fraction}") - if self.split not in ("train", "val"): - raise ValueError(f"split must be 'train' or 'val' in fraction mode; got {self.split!r}") - - rng = random.Random(self.seed) - shuffled = list(range(n)) - rng.shuffle(shuffled) - val_count = max(1, int(round(n * self.val_fraction))) - train_count = n - val_count - if self.split == "train": - return shuffled[:train_count] - return shuffled[train_count:] - - # Range mode - if self.start is not None or self.end is not None: - start = 0 if self.start is None else self.start - end = n if self.end is None else self.end - if start < 0: - start = max(0, n + start) - if end < 0: - end = max(0, n + end) - start = max(0, min(start, n)) - end = max(start, min(end, n)) - return list(range(start, end)) - - # Passthrough - return list(range(n)) + The wrapped source must implement ``__len__`` and ``__getitem__``. + + Args: + source: The underlying indexable source. + start: Inclusive start index (``None`` ⇒ 0; a negative value counts from the end). + end: Exclusive end index (``None`` ⇒ len(source); a negative value counts from the end). + """ + + def __init__(self, source: Any, start: Optional[int] = None, end: Optional[int] = None) -> None: + if not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + self.source = source + self.start = start + self.end = end + n = len(source) + s = 0 if start is None else start + e = n if end is None else end + if s < 0: + s = max(0, n + s) + if e < 0: + e = max(0, n + e) + s = max(0, min(s, n)) + e = max(s, min(e, n)) + self._indices: List[int] = list(range(s, e)) + logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) def __iter__(self) -> Iterator[Sample]: for idx in self._indices: @@ -238,3 +386,53 @@ def __getitem__(self, index: int) -> Sample: def __len__(self) -> int: return len(self._indices) + + +@configurable(category="source") +class ConcatSource: + """Concatenates multiple indexable sources into one longer indexable source. + + The indexable counterpart to :class:`dataflux.core.JointFlux` (which is iteration-only): + ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning + sub-source, so a ``ConcatSource`` can itself be wrapped by :class:`DatasetSplit` / + :class:`RangeSource`. (Distinct from :class:`dataflux.paired.AnnotationJoinSource`, which + *column-joins* annotations onto samples — this one *concatenates* sequences end to end.) + + Each sub-source must implement ``__len__`` and ``__getitem__``. + + Args: + sources: The indexable sources to concatenate, walked in order. + """ + + def __init__(self, sources: List[Any]) -> None: + for i, src in enumerate(sources): + if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): + raise TypeError( + "ConcatSource requires sources supporting __len__ and __getitem__; " + f"source[{i}] is {type(src).__name__}" + ) + self.sources = list(sources) + # Cumulative END offsets, for an O(log k) global-index → (sub-source, local index) map. + self._offsets: List[int] = [] + total = 0 + for src in self.sources: + total += len(src) + self._offsets.append(total) + + def __len__(self) -> int: + return self._offsets[-1] if self._offsets else 0 + + def __getitem__(self, index: int) -> Sample: + n = len(self) + if index < 0: + index += n + if not 0 <= index < n: + raise IndexError(index) + j = bisect.bisect_right(self._offsets, index) + start = self._offsets[j - 1] if j > 0 else 0 + return Sample.from_any(self.sources[j][index - start]) + + def __iter__(self) -> Iterator[Sample]: + for src in self.sources: + for item in src: + yield Sample.from_any(item) diff --git a/dataflux/storage/base.py b/dataflux/storage/base.py index 609ef49..c60b5e6 100644 --- a/dataflux/storage/base.py +++ b/dataflux/storage/base.py @@ -1,8 +1,21 @@ from typing import Any, Iterator, Protocol, runtime_checkable +import torch + from dataflux.sample import Sample +def to_numpy(data: Any) -> Any: + """Convert a torch tensor to a numpy array for array-storage backends (HDF5 / Zarr). + + Detaches and moves to CPU first so tensors carrying grad or living on a GPU + convert cleanly. Non-tensor values pass through unchanged. + """ + if isinstance(data, torch.Tensor): + return data.detach().cpu().numpy() + return data + + @runtime_checkable class DataSource(Protocol): """Minimum contract for a DataFlux data source.""" diff --git a/dataflux/storage/hdf5.py b/dataflux/storage/hdf5.py index 53c5031..fa89273 100644 --- a/dataflux/storage/hdf5.py +++ b/dataflux/storage/hdf5.py @@ -1,24 +1,18 @@ from pathlib import Path -from typing import Any, Iterator, Optional, Union +from typing import Iterator, Optional, Union import h5py +import numpy as np import torch from confluid import configurable from logflow import get_logger from dataflux.sample import Sample -from dataflux.storage.base import DataSink, DataSource, Storage +from dataflux.storage.base import DataSink, DataSource, Storage, to_numpy logger = get_logger("dataflux.storage.hdf5") -def to_numpy(data: Any) -> Any: - """Utility to convert torch tensors to numpy arrays for HDF5 storage.""" - if isinstance(data, torch.Tensor): - return data.detach().cpu().numpy() - return data - - @configurable class HDF5Source(Storage, DataSource): """Clean, high-performance HDF5 data source.""" @@ -55,6 +49,12 @@ def __iter__(self) -> Iterator[Sample]: data = self._file[f"{pref}_data"][()] target = self._file[f"{pref}_target"][()] if f"{pref}_target" in self._file else None metadata = dict(self._file[f"{pref}_data"].attrs) + # Merge array-valued metadata written as datasets under the per-sample meta group + # (see HDF5Sink.write). Absent on files written before this layout — old files read unchanged. + meta_grp = self._file.get(f"{pref}_meta") + if isinstance(meta_grp, h5py.Group): + for key, dset in meta_grp.items(): + metadata[key] = dset[()] # Source returns Tensors to match schema yield Sample(input=torch.from_numpy(data), target=target, metadata=metadata) @@ -112,12 +112,24 @@ def write(self, sample: Sample) -> None: ds = self._file.create_dataset(f"{prefix}_data", data=input_data, **kwargs) - # 2. Write Attributes (Metadata) + # 2. Write Metadata. Scalars/strings go on the data dataset's HDF5 attributes (compact, + # round-trips for the common case). Array-valued metadata (e.g. a segmentation mask) CANNOT + # be stored as an attribute — HDF5 caps attribute size ("object header message is too large") + # and the str() fallback would silently truncate the array — so it is written as its own + # dataset under a per-sample group ``{prefix}_meta/`` (the "/" makes h5py auto-create the + # group; arbitrary metadata keys are safe as dataset names). HDF5Source merges both back. for k, v in sample.metadata.items(): - try: - ds.attrs[k] = v - except Exception: - ds.attrs[k] = str(v) + if isinstance(v, (np.ndarray, torch.Tensor)): + arr = to_numpy(v) + m_kwargs = {} + if self.compression and getattr(arr, "ndim", 0) > 0: + m_kwargs["compression"] = self.compression + self._file.create_dataset(f"{prefix}_meta/{k}", data=arr, **m_kwargs) + else: + try: + ds.attrs[k] = v + except Exception: + ds.attrs[k] = str(v) # 3. Write Target if target_data is not None: diff --git a/dataflux/storage/intake.py b/dataflux/storage/intake.py deleted file mode 100644 index 9e96edf..0000000 --- a/dataflux/storage/intake.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Generic adapter that exposes any intake DataSource as a DataFlux DataSource. - -This bridge lets DataFlux pipelines consume any data exposed via the -`intake `_ catalog system. It is deliberately -container-agnostic: ``xarray.DataArray`` / ``xarray.Dataset``, ``numpy.ndarray``, -``pandas.DataFrame``, and arbitrary Python objects all map cleanly onto the -DataFlux ``Sample(input, target, metadata)`` triplet. - -The ``intake`` import is lazy so DataFlux's hard dependency surface is unchanged -— users only pay the cost when they actually construct an ``IntakeSource``. -""" - -from pathlib import Path -from typing import Any, Iterator, Optional, Union - -import confluid -import numpy as np -import torch -from logflow import get_logger - -from dataflux.sample import Sample -from dataflux.storage.base import DataSource, Storage - -logger = get_logger(__name__) - -_INTAKE_INSTALL_HINT = "IntakeSource requires the 'intake' package. " "Install it with: pip install 'intake>=2.0'" - - -@confluid.configurable -class IntakeSource(Storage, DataSource): - """Wrap an intake DataSource so it satisfies the DataFlux ``DataSource`` protocol. - - Each intake partition becomes one ``Sample``. Mapping rules: - - - ``xarray.DataArray`` / ``xarray.Dataset``: ``Sample.input`` is the - underlying numpy array (wrapped as a torch tensor when possible), - ``Sample.metadata`` is ``dict(da.attrs)``, and if ``target_attr`` is - set, ``Sample.target`` is read from ``attrs[target_attr]``. - - ``numpy.ndarray``: ``Sample.input`` is the array as a torch tensor; - no target, empty metadata. - - Everything else: routed through :py:meth:`dataflux.sample.Sample.from_any`. - - Construct with either a (catalog_path, source_name) pair (preferred for - Confluid serialization symmetry) or a pre-instantiated intake source via - ``source`` (handy for tests and ad-hoc use). - - Args: - catalog_path: Path to an intake catalog YAML. - source_name: Key inside the catalog identifying the source. - source: Pre-instantiated intake DataSource. Mutually exclusive with - the (catalog_path, source_name) pair. - target_attr: Name of an attribute key whose value should be lifted to - ``Sample.target`` for xarray containers. ``None`` means - no target. - """ - - def __init__( - self, - catalog_path: Optional[Union[str, Path]] = None, - source_name: Optional[str] = None, - source: Any = None, - target_attr: Optional[str] = None, - ) -> None: - if source is None and (catalog_path is None or source_name is None): - raise ValueError( - "IntakeSource requires either a (catalog_path, source_name) pair " - "or a pre-instantiated `source` object." - ) - if source is not None and (catalog_path is not None or source_name is not None): - raise ValueError("IntakeSource: pass either (catalog_path, source_name) or `source`, not both.") - - try: - import intake # noqa: F401 - except ImportError as e: # pragma: no cover - exercised via test_missing_intake_dep - raise ImportError(_INTAKE_INSTALL_HINT) from e - - self.catalog_path: Optional[str] = str(catalog_path) if catalog_path is not None else None - self.source_name = source_name - self.target_attr = target_attr - self._source = source - self._catalog: Any = None - - def _resolve_source(self) -> Any: - if self._source is not None: - return self._source - import intake - - logger.info(f"IntakeSource: opening catalog {self.catalog_path!r}, source {self.source_name!r}") - self._catalog = intake.open_catalog(self.catalog_path) - self._source = self._catalog[self.source_name] - return self._source - - def open(self) -> "IntakeSource": - self._resolve_source() - return self - - def close(self) -> None: - src = self._source - if src is not None and hasattr(src, "_close"): - try: - src._close() - except Exception as e: # pragma: no cover - defensive - logger.warning(f"IntakeSource: source close raised {e!r}") - # Only drop catalog-derived sources; user-supplied ones stay alive. - if self._catalog is not None: - self._source = None - self._catalog = None - - def __iter__(self) -> Iterator[Sample]: - src = self._resolve_source() - npart = self._npartitions(src) - for i in range(npart): - yield self._to_sample(src.read_partition(i)) - - def __len__(self) -> int: - return self._npartitions(self._resolve_source()) - - @staticmethod - def _npartitions(src: Any) -> int: - # intake 2.x lazily populates the schema; npartitions reads 0 until discover() runs. - if hasattr(src, "discover"): - try: - src.discover() - except Exception as e: # pragma: no cover - defensive - logger.debug(f"IntakeSource: discover() raised {e!r}") - n = getattr(src, "npartitions", None) - if n is None or n == 0: - schema = src._get_schema() - n = getattr(schema, "npartitions", 1) or 1 - return int(n) - - def _to_sample(self, value: Any) -> Sample: - # xarray: DataArray and Dataset both expose .values + .attrs. - if hasattr(value, "values") and hasattr(value, "attrs"): - attrs = dict(value.attrs) - data = value.values - tensor = self._array_to_tensor(data) - target = attrs.get(self.target_attr) if self.target_attr else None - return Sample(input=tensor, target=target, metadata=attrs) - if isinstance(value, np.ndarray): - return Sample(input=self._array_to_tensor(value), target=None, metadata={}) - return Sample.from_any(value) - - @staticmethod - def _array_to_tensor(array: np.ndarray) -> Any: - # Complex dtypes are preserved; non-complex go through torch.from_numpy directly. - try: - return torch.from_numpy(array) - except TypeError: - # Older torch versions can't ingest some dtypes (e.g. complex on CPU < 1.8). - # Fall back to returning the numpy array; downstream ops can handle it. - return array diff --git a/dataflux/storage/zarr.py b/dataflux/storage/zarr.py index 0a4ace3..68c64c3 100644 --- a/dataflux/storage/zarr.py +++ b/dataflux/storage/zarr.py @@ -1,11 +1,13 @@ from pathlib import Path -from typing import List, Optional, Union +from typing import Iterator, List, Optional, Union, cast import confluid +import numpy as np +import torch import zarr from dataflux.sample import Sample -from dataflux.storage.base import DataSink, Storage +from dataflux.storage.base import DataSink, DataSource, Storage, to_numpy @confluid.configurable @@ -37,26 +39,13 @@ def write(self, sample: Sample) -> None: name = f"sample_{self._counter:06d}" grp = self._root.require_group(name) - # 1. Save data and target (Explicit shape/dtype for Zarr v3) - # Use require_dataset or delete if exists - if "data" in grp: - del grp["data"] - grp.create_dataset( - "data", - data=sample.input, - shape=sample.input.shape, - dtype=sample.input.dtype, - ) + # 1. Save data and target. create_array needs a numpy array (it can't read a + # torch tensor's dtype); to_numpy detaches/moves to CPU. overwrite=True replaces + # an existing node, so no manual delete is needed on re-write. + grp.create_array("data", data=to_numpy(sample.input), overwrite=True) if sample.target is not None: - if "target" in grp: - del grp["target"] - grp.create_dataset( - "target", - data=sample.target, - shape=sample.target.shape, - dtype=sample.target.dtype, - ) + grp.create_array("target", data=to_numpy(sample.target), overwrite=True) # 2. Save metadata as Zarr attributes (.zattrs) if sample.metadata: @@ -68,6 +57,57 @@ def flush(self) -> None: pass # pragma: no cover +@confluid.configurable +class ZarrGroupSource(Storage, DataSource): + """Read samples written by :class:`ZarrGroupSink` (one Zarr group per sample). + + Mirrors the group sink's layout: each ``sample_NNNNNN`` subgroup carries a + ``data`` array, an optional ``target`` array, and the sample metadata as + group attributes (``.zattrs``). Groups are iterated in sorted name order so + the read order matches the write order. + + Args: + path: Path to the Zarr group written by ZarrGroupSink. + sample_key: Name of the per-sample array holding ``Sample.input``. + target_key: Name of the per-sample array holding ``Sample.target`` (absent when the sample had no target). + """ + + def __init__( + self, + path: Union[str, Path], + sample_key: str = "data", + target_key: str = "target", + ) -> None: + self.path = str(path) + self.sample_key = sample_key + self.target_key = target_key + self._root: Optional[zarr.Group] = None + + def open(self) -> "ZarrGroupSource": + if self._root is None: + self._root = zarr.open_group(self.path, mode="r") + return self + + def close(self) -> None: + self._root = None + + def __iter__(self) -> Iterator[Sample]: + self.open() + if self._root is None: + return + for name in sorted(self._root.group_keys()): + grp = cast(zarr.Group, self._root[name]) + data = cast(zarr.Array, grp[self.sample_key])[:] + target = cast(zarr.Array, grp[self.target_key])[:] if self.target_key in grp else None + yield Sample(input=torch.from_numpy(np.asarray(data)), target=target, metadata=dict(grp.attrs)) + + def __len__(self) -> int: + self.open() + if self._root is None: + return 0 + return len(list(self._root.group_keys())) + + @confluid.configurable class ZarrBatchSink(Storage, DataSink): """ @@ -118,3 +158,42 @@ def write(self, sample: Sample) -> None: def flush(self) -> None: pass # pragma: no cover + + +@confluid.configurable +class ZarrBatchSource(Storage, DataSource): + """Read samples written by :class:`ZarrBatchSink` (one stacked array). + + The batch sink appends every sample's input along axis 0 of a single + ``data`` array and stores no per-sample target or metadata, so this source + yields input-only :class:`~dataflux.sample.Sample` objects — one per row of + the leading axis. + + Args: + path: Path to the Zarr store written by ZarrBatchSink (the directory holding the ``data`` array). + """ + + def __init__(self, path: Union[str, Path]) -> None: + self.path = str(path) + self._data_arr: Optional[zarr.Array] = None + + def open(self) -> "ZarrBatchSource": + if self._data_arr is None: + self._data_arr = zarr.open_array(store=f"{self.path}/data", mode="r") + return self + + def close(self) -> None: + self._data_arr = None + + def __iter__(self) -> Iterator[Sample]: + self.open() + if self._data_arr is None: + return + for i in range(self._data_arr.shape[0]): + yield Sample(input=torch.from_numpy(np.asarray(self._data_arr[i]))) + + def __len__(self) -> int: + self.open() + if self._data_arr is None: + return 0 + return int(self._data_arr.shape[0]) diff --git a/dataflux/typespec.py b/dataflux/typespec.py index e23ee42..9c65de5 100644 --- a/dataflux/typespec.py +++ b/dataflux/typespec.py @@ -41,7 +41,21 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, AbstractSet, Any, Callable, Dict, FrozenSet, List, Optional, Tuple, TypeVar, Union +from typing import ( + TYPE_CHECKING, + AbstractSet, + Any, + Callable, + Dict, + FrozenSet, + List, + Literal, + Optional, + Tuple, + TypeVar, + Union, + cast, +) import numpy as np @@ -52,6 +66,16 @@ # alias is bound at runtime (annotations are strings under ``from __future__ import annotations``). TypeSpec = Union["AnyType", "ArrayType", "PythonType", "UnionType", "MappingType", "ListType"] +# Closed enumerations for the small, fixed string sets the type system uses — declared as ``Literal`` +# rather than bare ``str`` so authors get a typo-checked value and UIs / the FluxStudio connection- +# validator enumerate the choices straight from the annotation (``typing.get_args(...)``); the +# workspace "prefer closed ``Literal``s over bare strings" mandate. Both are *deliberately closed* — +# extend the Literal when adding real support (e.g. a ``"jax"`` framework), don't widen to ``str``. +# (The dtype enumerations — ``Dtype`` / ``DtypeFamily`` / ``DtypeSpec`` — are defined next to +# ``_DTYPE_FAMILIES`` below, since they share that block's set membership as their source of truth.) +Framework = Literal["numpy", "torch", "tensorflow"] +ImageLayout = Literal["CHW", "HWC"] + C = TypeVar("C") __all__ = [ @@ -64,6 +88,11 @@ "ListType", "SampleType", "TypeSpec", + "Framework", + "ImageLayout", + "Dtype", + "DtypeFamily", + "DtypeSpec", "typed", "accepts", "compatible", @@ -101,29 +130,66 @@ "numeric": frozenset(_FLOATING | _INTEGER | _UNSIGNED), } +#: A concrete dtype name — a closed ``Literal`` (not bare ``str``) so an authored ``ACCEPTS`` / +#: ``PRODUCES`` dtype is typo-checked and UIs / the FluxStudio connection-validator enumerate the +#: choices via ``typing.get_args(Dtype)``. These ARE the union of the family members above (pinned +#: equal in ``tests/test_typespec.py`` so the two can't drift). Authoring uses canonical lowercase +#: names; aliases / casing (``"double"``, ``"FLOAT32"``) and genuinely exotic, platform-dependent +#: dtypes (``float128``, ``complex256``) are *runtime-only* — they reach the field via +#: :func:`canonical_dtype`, the single boundary that normalizes arbitrary input into this domain, and +#: an unmodeled one keeps its own name and simply matches no family. +Dtype = Literal[ + "bool", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "float16", + "bfloat16", + "float32", + "float64", + "complex64", + "complex128", +] +#: A relaxed dtype *family* constraint (matches any concrete member). The names are the keys of +#: ``_DTYPE_FAMILIES`` (pinned equal in tests); kept as a ``Literal`` for the same author/UI reasons. +DtypeFamily = Literal["floating", "integer", "unsigned", "bool", "complex", "numeric"] +#: What :attr:`ArrayType.dtype` accepts: a concrete :data:`Dtype` or a relaxed :data:`DtypeFamily`. +DtypeSpec = Union[Dtype, DtypeFamily] + -def canonical_dtype(x: Any) -> str: +def canonical_dtype(x: Any) -> DtypeSpec: """Normalize a dtype (str, numpy dtype/scalar-type, or torch dtype) to a canonical lowercase name. Family names (``"floating"``, ``"integer"``, ``"numeric"`` …) pass through unchanged so they can - be used as relaxed dtype constraints on an :class:`ArrayType`. + be used as relaxed dtype constraints on an :class:`ArrayType`. This is the single boundary where + arbitrary input (aliases, casing, framework dtype objects, exotic dtypes) crosses into the typed + :data:`DtypeSpec` domain — hence the closing ``cast``: a genuinely unmodeled dtype keeps its own + name (and simply matches no family), which is correct even though it lies outside the Literal. """ if isinstance(x, str): s = x.lower() - return _DTYPE_ALIASES.get(s, s) - if isinstance(x, np.dtype): - return str(x.name) - if isinstance(x, type) and issubclass(x, np.generic): - return str(np.dtype(x).name) - # torch dtype (lazy — torch is a hard dep but we avoid importing it at module load) - try: - import torch - - if isinstance(x, torch.dtype): - return str(x).replace("torch.", "") - except ImportError: # pragma: no cover - torch is a hard dep - pass - return str(x).lower() + name = _DTYPE_ALIASES.get(s, s) + elif isinstance(x, np.dtype): + name = str(x.name) + elif isinstance(x, type) and issubclass(x, np.generic): + name = str(np.dtype(x).name) + else: + # Default for any non-numpy value; refined to the bare name for a torch dtype (lazy import — + # torch is a hard dep but we avoid importing it at module load). + name = str(x).lower() + try: + import torch + + if isinstance(x, torch.dtype): + name = str(x).replace("torch.", "") + except ImportError: # pragma: no cover - torch is a hard dep + pass + return cast(DtypeSpec, name) def _dtype_accepts(consumer: str, producer: str) -> bool: @@ -214,16 +280,18 @@ class ArrayType: * ``ndim`` — required rank (derived from ``shape`` when that is given). * ``shape`` — per-axis :class:`Dim` tuple. - * ``dtype`` — canonical name (``"float32"``) or family (``"floating"``/``"numeric"`` …). - * ``frameworks`` — allowed framework set (``{"numpy", "torch"}`` …). + * ``dtype`` — a :data:`DtypeSpec`: a concrete :data:`Dtype` (``"float32"``) or a relaxed + :data:`DtypeFamily` (``"floating"``/``"numeric"`` …). Stored canonicalized via + :func:`canonical_dtype`, which also accepts aliases / casing / framework dtype objects at runtime. + * ``frameworks`` — allowed framework set, each a :data:`Framework` (``{"numpy", "torch"}`` …). * ``semantic`` — free-form tag (e.g. ``"image"``); display/inference hint, never matched. """ ndim: Optional[int] = None shape: Optional[Tuple[Dim, ...]] = None - dtype: Optional[str] = None + dtype: Optional[DtypeSpec] = None # Accept any set on construction (ergonomic ``frameworks={"torch"}``); ``__post_init__`` freezes it. - frameworks: Optional[AbstractSet[str]] = None + frameworks: Optional[AbstractSet[Framework]] = None semantic: Optional[str] = None def __post_init__(self) -> None: @@ -242,10 +310,10 @@ def __post_init__(self) -> None: @classmethod def image( cls, - layout: str = "CHW", + layout: ImageLayout = "CHW", channels: Union[int, Tuple[int, ...]] = 3, - dtype: Optional[str] = None, - framework: Optional[str] = None, + dtype: Optional[DtypeSpec] = None, + framework: Optional[Framework] = None, ) -> "ArrayType": """Rank-3 image convenience. ``channels`` as a tuple is treated as the inclusive range ``[min, max]`` (a pragmatic approximation — pass an exact :class:`Dim` via the constructor @@ -269,8 +337,8 @@ def image( def parse( cls, spec: str, - dtype: Optional[str] = None, - framework: Optional[str] = None, + dtype: Optional[DtypeSpec] = None, + framework: Optional[Framework] = None, semantic: Optional[str] = None, ) -> "ArrayType": """Build from a jaxtyping-style shape string: space-separated axes where a bare int is an diff --git a/examples/dataset_split.yaml b/examples/dataset_split.yaml index c1ae87b..1b284e7 100644 --- a/examples/dataset_split.yaml +++ b/examples/dataset_split.yaml @@ -1,9 +1,8 @@ -# DataFlux DatasetSplit Example +# DataFlux DatasetSplit / RangeSource / ConcatSource Example # -# Shows how to use DatasetSplit to carve a single HuggingFace source -# into disjoint train/val views sharing the same seed. -# -# Reload with `confluid.load(...)` and iterate the Fluxes. +# DatasetSplit is a `category="source"` that partitions an indexable source into +# reproducible train/val/test views. `split` is the closed Literal["train","val","test"] +# (dataflux.SplitName). Reload with `confluid.load(...)` and iterate the Fluxes. hf_train: !class:dataflux.sources.HuggingFaceSource() path: mnist @@ -11,27 +10,45 @@ hf_train: !class:dataflux.sources.HuggingFaceSource() input_feature: image target_feature: label +# === Property API (preferred): ONE DatasetSplit exposes .train / .val / .test === +# Configure the split once (80/10/10 here) and reference its cached views by +# attribute. All three `!ref:my_split.` resolve to the SAME materialized +# DatasetSplit, so `hf_train` is loaded exactly once and the partition is shared. +my_split: !class:dataflux.sources.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + +train_set: !class:dataflux.core.Flux() + source: !ref:my_split.train + +val_set: !class:dataflux.core.Flux() + source: !ref:my_split.val + +test_set: !class:dataflux.core.Flux() + source: !ref:my_split.test + +# === RangeSource: a contiguous [start:end) slice over any indexable source === +first_1000: !class:dataflux.core.Flux() + source: !class:dataflux.sources.RangeSource() + source: !ref:hf_train + start: 0 + end: 1000 + +# === ConcatSource: join several indexable sources into one (then optionally split) === hf_test: !class:dataflux.sources.HuggingFaceSource() path: mnist split: test input_feature: image target_feature: label -# 90/10 train/val split, deterministic via seed=42 -train_set: !class:dataflux.core.Flux() - source: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.1 - seed: 42 - -val_set: !class:dataflux.core.Flux() - source: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 +# Concatenate train + test into one source; ConcatSource is indexable, so it can +# itself feed a DatasetSplit (e.g. to re-partition the combined pool). +combined: !class:dataflux.sources.ConcatSource() + sources: + - !ref:hf_train + - !ref:hf_test -# Test set stays untouched until final evaluation. -test_set: !class:dataflux.core.Flux() - source: !ref:hf_test +combined_pool: !class:dataflux.core.Flux() + source: !ref:combined diff --git a/examples/intake_pipeline.py b/examples/intake_pipeline.py deleted file mode 100644 index a90baec..0000000 --- a/examples/intake_pipeline.py +++ /dev/null @@ -1,103 +0,0 @@ -"""IntakeSource adapter demo. - -Shows how the generic ``dataflux.storage.intake.IntakeSource`` adapter consumes -any intake DataSource — using a tiny in-process driver here so the example needs -no external data or services. - -Demonstrates: -1. Wrapping an ``intake.source.base.DataSource`` that yields ``xarray.DataArray`` - partitions. -2. ``target_attr`` lifting the per-partition label from xarray attrs into - ``Sample.target``. -3. Round-tripping a Flux pipeline through ``HDF5Sink`` → ``HDF5Source`` to show - that the metadata flowing in via xarray attrs survives storage. -""" - -import tempfile -from pathlib import Path -from typing import Any - -import intake -import numpy as np -import xarray as xr - -from dataflux.core import Flux -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.intake import IntakeSource - - -class _DemoXArraySource(intake.source.base.DataSource): - """Tiny in-process intake DataSource yielding 4 xarray.DataArray partitions.""" - - container = "xarray" - name = "_demo_xarray" - version = "0.1" - partition_access = True - - def __init__(self, n: int = 4, label: str = "demo", metadata: Any = None) -> None: - super().__init__(metadata=metadata or {}) - self._n = n - self._label = label - - def _get_schema(self) -> Any: - return intake.source.base.Schema( - datashape=None, - dtype=str(np.dtype(np.float32)), - shape=(8,), - npartitions=self._n, - extra_metadata={}, - ) - - def _get_partition(self, i: int) -> xr.DataArray: - return xr.DataArray( - np.arange(8, dtype=np.float32) + (i * 100), - dims=["x"], - attrs={"label": self._label, "partition_index": i, "samplerate": 1000.0}, - ) - - def _close(self) -> None: - pass - - -def main() -> None: - with tempfile.TemporaryDirectory(prefix="dataflux-intake-demo-") as tmp: - # 1. Build an intake source and wrap it. - intake_src = _DemoXArraySource(n=4, label="demo") - df_src = IntakeSource(source=intake_src, target_attr="label") - print(f"IntakeSource length: {len(df_src)} partitions") - - # 2. Iterate and inspect the first sample. - first = next(iter(df_src)) - print( - "First sample:", - "input.shape=", - tuple(first.input.shape), - "input.dtype=", - first.input.dtype, - "target=", - first.target, - "metadata.keys=", - sorted(first.metadata.keys()), - ) - - # 3. Pipe through Flux → HDF5Sink and read back. - h5_path = Path(tmp) / "intake_roundtrip.h5" - Flux.from_source(df_src).to_sink(HDF5Sink(h5_path, overwrite=True)) - - loaded = list(HDF5Source(h5_path)) - print(f"Wrote {len(loaded)} samples to {h5_path}") - print( - "Roundtripped sample 2:", - "shape=", - tuple(loaded[2].input.shape), - "samplerate(meta)=", - loaded[2].metadata.get("samplerate"), - "label(meta)=", - loaded[2].metadata.get("label"), - ) - - print("OK — intake_pipeline.py finished.") - - -if __name__ == "__main__": - main() diff --git a/examples/paired_annotations.py b/examples/paired_annotations.py index 37f104c..0d051a4 100644 --- a/examples/paired_annotations.py +++ b/examples/paired_annotations.py @@ -1,7 +1,7 @@ -"""PairedSource walkthrough: binary-first, annotation-first, broadcast, slicing. +"""AnnotationJoinSource walkthrough: binary-first, annotation-first, broadcast, slicing. Runs standalone with no external data. Demonstrates the four scenarios the -``PairedSource`` primitive is designed for, using a tiny in-memory primary +``AnnotationJoinSource`` primitive is designed for, using a tiny in-memory data source and a dict-shaped annotation store. """ @@ -9,7 +9,7 @@ import confluid # type: ignore[import-not-found] -from dataflux.paired import PairedSource +from dataflux.paired import AnnotationJoinSource from dataflux.sample import Sample @@ -44,7 +44,7 @@ def __iter__(self) -> Iterator[Sample]: @confluid.configurable class DictStore: - """Mapping-shaped secondary for demos.""" + """Mapping-shaped annotations for demos.""" def __init__(self, records: Optional[Dict[str, Dict[str, Any]]] = None) -> None: self.records = records or {} @@ -69,8 +69,8 @@ def pack_key(sample: Sample) -> str: return str(sample.metadata["pack_id"]) -def resolve_by_window_key(key: str, primary: WindowedSource) -> Sample: - for item in primary: +def resolve_by_window_key(key: str, data: WindowedSource) -> Sample: + for item in data: if window_key(item) == key: return item raise KeyError(key) @@ -96,10 +96,10 @@ def slice_intervals(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str def scenario_a_binary_first() -> None: """Scenario A: iterate all samples; attach annotation when available.""" print("\n=== Scenario A: binary-first, annotations optional ===") - primary = WindowedSource(n_windows=4) + data = WindowedSource(n_windows=4) store = DictStore({"demo:pack1:win00000100": {"label": "dji_mavic", "score": 0.92}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=window_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key) for s in paired: flag = "ANNOTATED" if s.metadata["annotated"] else " -" @@ -110,7 +110,7 @@ def scenario_a_binary_first() -> None: def scenario_b_annotation_first() -> None: """Scenario B: only emit samples that have an annotation.""" print("\n=== Scenario B: annotation-first, curated labeled subset ===") - primary = WindowedSource(n_windows=6) + data = WindowedSource(n_windows=6) store = DictStore( { "demo:pack1:win00000000": {"label": "wifi"}, @@ -118,20 +118,20 @@ def scenario_b_annotation_first() -> None: } ) - paired = PairedSource(primary=primary, secondary=store, key_fn=window_key, policy="inner") + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key, policy="inner") for s in paired: print(f" key={s.metadata['annotation_key']:<30} label={s.metadata['label']!r}") - print(f" -> {len(list(paired))} samples (out of {len(primary)} in primary)") + print(f" -> {len(list(paired))} samples (out of {len(data)} in data)") def scenario_c1_broadcast() -> None: """Scenario C1: one annotation per pack, broadcast to every window.""" print("\n=== Scenario C1: pack-level broadcast ===") - primary = WindowedSource(n_windows=4) + data = WindowedSource(n_windows=4) store = DictStore({"demo:pack1": {"drone": "DJI Mavic 3 Pro", "operator": "alice"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=pack_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=pack_key) for s in paired: print( @@ -143,7 +143,7 @@ def scenario_c1_broadcast() -> None: def scenario_c2_slicing() -> None: """Scenario C2: pack-level time-ranged annotation, sliced per window.""" print("\n=== Scenario C2: pack-level time intervals, sliced per window ===") - primary = WindowedSource(n_windows=6, samples_per_window=100) + data = WindowedSource(n_windows=6, samples_per_window=100) # Samplerate is 1 MHz and windows are 100 samples = 100 us each, so: # win0 = [0, 100us], win1 = [100us, 200us], ..., win5 = [500us, 600us]. # An interval at [150us, 470us] overlaps windows 1, 2, 3, 4. @@ -156,9 +156,9 @@ def scenario_c2_slicing() -> None: } ) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=pack_key, extract_fn=slice_intervals, ) @@ -176,9 +176,9 @@ def scenario_c2_slicing() -> None: def scenario_d_right_driven() -> None: - """Scenario D: iterate the annotation store, resolve primary on demand.""" - print("\n=== Scenario D: right-driven (sparse labels, large primary) ===") - primary = WindowedSource(n_windows=1000) # pretend this is huge + """Scenario D: iterate the annotation store, resolve data on demand.""" + print("\n=== Scenario D: right-driven (sparse labels, large data) ===") + data = WindowedSource(n_windows=1000) # pretend this is huge store = DictStore( { "demo:pack1:win00000000": {"label": "wifi"}, @@ -187,12 +187,12 @@ def scenario_d_right_driven() -> None: } ) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=window_key, policy="right_driven", - primary_resolver=resolve_by_window_key, + data_resolver=resolve_by_window_key, ) for s in paired: @@ -202,9 +202,9 @@ def scenario_d_right_driven() -> None: def scenario_e_confluid_roundtrip() -> None: """Show that the pipeline survives YAML serialization via Confluid.""" print("\n=== Scenario E: Confluid YAML round-trip ===") - primary = WindowedSource(n_windows=3) + data = WindowedSource(n_windows=3) store = DictStore({"demo:pack1:win00000000": {"label": "wifi"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=window_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key) yaml_state = confluid.dump(paired) print(yaml_state) diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py new file mode 100644 index 0000000..64cc8fc --- /dev/null +++ b/examples/storage_roundtrip.py @@ -0,0 +1,64 @@ +"""Storage round-trips: HDF5 array-valued metadata + the Zarr sources. + +Demonstrates two storage features end-to-end on synthetic data (no external +deps or services): + +1. ``HDF5Sink`` / ``HDF5Source`` round-trips a ``Sample`` whose ``metadata`` + carries a 2-D array (a stand-in for a segmentation mask). Array metadata is + stored as a dataset under a per-sample ``{prefix}_meta/`` group — it would + otherwise overflow HDF5's attribute-size limit and be silently truncated. +2. ``ZarrGroupSink`` / ``ZarrGroupSource`` round-trips input + target + metadata, + and ``ZarrBatchSink`` / ``ZarrBatchSource`` round-trips a stacked uniform array. +""" + +import tempfile +from pathlib import Path + +import numpy as np + +from dataflux.core import Flux +from dataflux.sample import Sample +from dataflux.storage.hdf5 import HDF5Sink, HDF5Source +from dataflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="dataflux-storage-demo-") as tmp: + root = Path(tmp) + + # 1. HDF5 with an array in metadata (the segmentation-mask case). + print("--- HDF5: Sample with a 2-D mask in metadata ---") + mask = np.random.randint(0, 2, size=(64, 64), dtype=np.uint8) + h5 = root / "ds.h5" + Flux([Sample(input=np.random.randn(10), target=np.array([1]), metadata={"mask": mask, "snr": 12.0})]).to_sink( + HDF5Sink(h5, overwrite=True) + ) + loaded = next(iter(HDF5Source(h5))) + print(f" mask round-trips exact : {np.array_equal(loaded.metadata['mask'], mask)}") + print(f" scalar metadata kept : snr={loaded.metadata['snr']}") + + # 2. Zarr group source — full input/target/metadata round-trip. + print("\n--- Zarr group: ZarrGroupSink -> ZarrGroupSource ---") + zg = root / "group.zarr" + samples = [ + Sample(input=np.arange(5, dtype="float32"), target=np.array([1]), metadata={"id": "a"}), + Sample(input=np.arange(3, dtype="float32"), metadata={"id": "b"}), + ] + Flux(samples).to_sink(ZarrGroupSink(zg, overwrite=True)) + for s in ZarrGroupSource(zg): + tgt = None if s.target is None else s.target.tolist() + print(f" input={s.input.tolist()} target={tgt} id={s.metadata['id']!r}") + + # 3. Zarr batch source — stacked uniform array, input only. + print("\n--- Zarr batch: ZarrBatchSink -> ZarrBatchSource ---") + zb = root / "batch.zarr" + Flux([Sample(input=np.full((4,), i, dtype=np.float32)) for i in range(3)]).to_sink( + ZarrBatchSink(zb, shape=[4], overwrite=True) + ) + print(f" rows read back: {[int(s.input[0]) for s in ZarrBatchSource(zb)]}") + + print("\nStorage round-trips verified!") + + +if __name__ == "__main__": + main() diff --git a/manifests/datasets/flux.yaml b/manifests/datasets/flux.yaml deleted file mode 100644 index 5b31ab5..0000000 --- a/manifests/datasets/flux.yaml +++ /dev/null @@ -1,18 +0,0 @@ -description: "DataFlux Flux — primary stream engine wrapping any iterable or indexed dataset with a functional op chain." -use_case: "The fundamental DataFlux container. Wrap a Source (HuggingFaceSource / RegionsJsonSource / a list of Samples) and apply zero or more Ops lazily on __getitem__. Composable via JointFlux for source-mixing or DatasetSplit for train/val partitioning. Supports map-style (__getitem__) when the underlying source has __len__." -category: "Generic" -flux_type: "Dataset" -class_path: "dataflux.core.Flux" -params: - source: - type: "Iterable[Sample] | torch.utils.data.Dataset" - required: true - description: "Underlying data source — anything iterable yielding Samples, or a torch Dataset." - ops: - type: "List[dataflux.core.Op] | null" - default: null - description: "Ordered list of ops applied lazily on __getitem__ / iteration. Empty / null = pass-through." - chunk_size: - type: "int | null" - default: 0 - description: "Optional chunking hint for streaming sources. 0 = no chunking." diff --git a/manifests/datasets/joint_flux.yaml b/manifests/datasets/joint_flux.yaml deleted file mode 100644 index 5dfe49a..0000000 --- a/manifests/datasets/joint_flux.yaml +++ /dev/null @@ -1,10 +0,0 @@ -description: "Aggregate multiple Flux streams into a single joint sequential stream." -use_case: "Use to mix sources during training (e.g. fold a small slice of the test set into train). Each sub-flux keeps its own op chain — JointFlux only sequences iteration. Pair with DatasetSplit when you want a reproducible disjoint train/val partition with shared random seed." -category: "Generic" -flux_type: "Dataset" -class_path: "dataflux.core.JointFlux" -params: - fluxes: - type: "List[dataflux.core.Flux]" - required: true - description: "Sub-fluxes whose iteration is concatenated end-to-end." diff --git a/manifests/sources/dataset_split.yaml b/manifests/sources/dataset_split.yaml deleted file mode 100644 index d83ca92..0000000 --- a/manifests/sources/dataset_split.yaml +++ /dev/null @@ -1,27 +0,0 @@ -description: "Select a reproducible subset view of an indexable source." -use_case: "Three modes: (1) fraction-mode train/val split with shared seed (two DatasetSplits with same seed + val_fraction yield disjoint complementary views — wire via !ref to share the underlying source); (2) explicit indices for a fixed subset; (3) range selection. Heavily used in the rfuav templates to fold-test-into-train." -category: "Generic" -flux_type: "DataSource" -class_path: "dataflux.sources.DatasetSplit" -params: - source: - type: "Indexable source" - required: true - description: "Underlying source with __len__ and __getitem__ (e.g. a Flux wrapping a RegionsJsonSource)." - split: - type: "str" - default: "train" - choices: ["train", "val"] - description: "Which side of the partition this view returns. With shared seed across two DatasetSplits, 'train' and 'val' are guaranteed disjoint and complementary." - val_fraction: - type: "float" - default: 0.2 - description: "Fraction of the source going to the 'val' branch (the rest goes to 'train')." - seed: - type: "int" - default: 42 - description: "Random seed for the shuffle. Must match across paired splits to get a clean partition." - indices: - type: "List[int] | null" - default: null - description: "Explicit indices mode (mutually exclusive with split/val_fraction)." diff --git a/manifests/sources/huggingface_source.yaml b/manifests/sources/huggingface_source.yaml deleted file mode 100644 index 65d0bd2..0000000 --- a/manifests/sources/huggingface_source.yaml +++ /dev/null @@ -1,34 +0,0 @@ -description: "Hugging Face Datasets adapter — maps HF dataset features to DataFlux Sample triplets." -use_case: "Standard adapter for any HF Hub dataset (e.g. kitofrank/RFUAV). Configure which feature column maps to Sample.input, which to Sample.target, and which (if any) to Sample.metadata." -category: "HuggingFace" -flux_type: "DataSource" -class_path: "dataflux.sources.HuggingFaceSource" -params: - path: - type: "str" - required: true - description: "HF dataset path — repo identifier (kitofrank/RFUAV) or local imagefolder path." - split: - type: "str" - default: "train" - description: "HF split name (train / validation / test / etc.)." - input_feature: - type: "str" - default: "image" - description: "Feature column to map to Sample.input." - target_feature: - type: "str" - default: "label" - description: "Feature column to map to Sample.target." - metadata_features: - type: "List[str] | null" - default: null - description: "Feature columns preserved on Sample.metadata." - count: - type: "int | null" - default: null - description: "Optional cap on the number of samples (useful for fast smoke runs)." - name: - type: "str | null" - default: null - description: "Optional HF subset name (e.g. for multi-config datasets)." diff --git a/pyproject.toml b/pyproject.toml index cc002a7..a852a9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "confluid>=0.1.0", "log-flow>=0.1.0", "numpy", + "Pillow", "h5py", "zarr", "torch", @@ -28,9 +29,6 @@ dev = [ "mypy>=1.0.0,<2.0.0", "pytest>=7.0.0,<9.0.0", "pytest-cov>=4.0.0,<7.0.0", - "Pillow", - "intake>=2.0", - "xarray", ] notebook = [ "matplotlib", @@ -60,6 +58,8 @@ dataflux-ops-numpy = "dataflux.ops.numpy" dataflux-ops-torch = "dataflux.ops.torch" dataflux-ops-copy = "dataflux.ops.copy" dataflux-ops-swap = "dataflux.ops.swap" +dataflux-ops-target = "dataflux.ops.target" +dataflux-ops-image = "dataflux.ops.image" [tool.setuptools.packages.find] where = ["."] @@ -85,7 +85,5 @@ module = [ "h5py.*", "PIL.*", "zarr.*", - "intake.*", - "xarray.*", ] ignore_missing_imports = true diff --git a/tests/test_categories.py b/tests/test_categories.py index adb85f3..7df34c5 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,27 +10,49 @@ from confluid.registry import get_registry from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from dataflux.ops.copy import CopyInputOp +from dataflux.ops.image import ConvertToImageOp from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp +from dataflux.ops.parallel import Parallel +from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp from dataflux.ops.tee import Tee -from dataflux.sources import DatasetSplit, HuggingFaceSource +from dataflux.ops.torch import ToTensorOp +from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource def test_engine_classes_tagged() -> None: - """The generic, task-agnostic *engines* / composition primitives — NOT GUI-buildable nodes. + """The generic, task-agnostic *engines* — composition primitives that compose sources + ops. - ``Flux`` / ``JointFlux`` / ``DatasetSplit`` compose sources + ops; ``FilterOp`` / ``WrappedOp`` - wrap a raw Python callable. All carry ``category="engine"`` so FluxStudio's positive - op/source/dataset allowlist excludes them (you wire Source → Op instead).""" + ``Flux`` / ``JointFlux`` carry ``category="engine"``. They (and ``DatasetSplit``, now a + ``source``) are canvas-composable in FluxStudio: its allowlist now includes ``engine`` and the + source-typed constructor params (``source`` / ``fluxes`` / ``ops``) render as wired sockets.""" assert Flux.__confluid_category__ == "engine" assert JointFlux.__confluid_category__ == "engine" - assert DatasetSplit.__confluid_category__ == "engine" - assert FilterOp.__confluid_category__ == "engine" - assert WrappedOp.__confluid_category__ == "engine" + + +def test_raw_callable_wrappers_uncategorised() -> None: + """``FilterOp`` / ``WrappedOp`` wrap a *raw Python callable*, so they are neither an ``op`` + (nothing to wire) nor an ``engine`` — they carry NO category (bare ``@configurable``) and are + excluded from FluxStudio by being uncategorised, like a module-level helper function. So even + once ``engine`` is added to the allowlist these wrappers stay out (correct — they're not buildable).""" + assert getattr(FilterOp, "__confluid_category__", None) is None + assert getattr(WrappedOp, "__confluid_category__", None) is None + # Still registered/configurable, just untagged. + assert FilterOp.__confluid_configurable__ is True + assert WrappedOp.__confluid_configurable__ is True def test_source_classes_tagged() -> None: - """``HuggingFaceSource`` is a concrete data *source* (it loads a dataset).""" + """``HuggingFaceSource`` is a concrete data *source* (it loads a dataset). + + ``DatasetSplit`` / ``RangeSource`` / ``ConcatSource`` are also ``source``s: they yield + ``Sample``s and are wired into a trainer's ``source:`` slot, each exposing a derived *view* + of other source(s) (split / contiguous slice / concatenation) — they apply no ops, so they + are sources, not engines.""" assert HuggingFaceSource.__confluid_category__ == "source" + assert DatasetSplit.__confluid_category__ == "source" + assert RangeSource.__confluid_category__ == "source" + assert ConcatSource.__confluid_category__ == "source" def test_op_classes_tagged() -> None: @@ -39,6 +61,27 @@ def test_op_classes_tagged() -> None: assert StandardizeOp.__confluid_category__ == "op" assert ThresholdOp.__confluid_category__ == "op" assert Tee.__confluid_category__ == "op" + assert MetadataToTargetOp.__confluid_category__ == "op" + assert EncodeTargetOp.__confluid_category__ == "op" + assert DecodeTargetOp.__confluid_category__ == "op" + + +def test_op_group_tags() -> None: + """Ops carry a path-like ``group`` (FluxStudio palette nesting: Taidal/DataFlux/Op/). + + Presentation-only — orthogonal to the category that gates discovery. A renamed/dropped group + re-files the node in the palette but never hides it; pinned so the taxonomy is a regression gate.""" + assert RescaleOp.__confluid_group__ == "numpy" + assert StandardizeOp.__confluid_group__ == "numpy" + assert ThresholdOp.__confluid_group__ == "numpy" + assert ToTensorOp.__confluid_group__ == "torch" + assert CopyInputOp.__confluid_group__ == "structure" + assert MetadataToTargetOp.__confluid_group__ == "structure" + assert EncodeTargetOp.__confluid_group__ == "structure" + assert DecodeTargetOp.__confluid_group__ == "structure" + assert Tee.__confluid_group__ == "compose" + assert Parallel.__confluid_group__ == "compose" + assert ConvertToImageOp.__confluid_group__ == "image" def test_categories_enumerable_via_registry() -> None: @@ -48,6 +91,30 @@ def test_categories_enumerable_via_registry() -> None: not just the class attribute — has to carry the tag. """ registry = get_registry() - assert {"Flux", "JointFlux", "DatasetSplit", "FilterOp", "WrappedOp"} <= registry.list_classes(category="engine") - assert {"HuggingFaceSource"} <= registry.list_classes(category="source") - assert {"RescaleOp", "StandardizeOp", "ThresholdOp", "Tee"} <= registry.list_classes(category="op") + assert {"Flux", "JointFlux"} <= registry.list_classes(category="engine") + # DatasetSplit is a source now, not an engine. + assert "DatasetSplit" not in registry.list_classes(category="engine") + # FilterOp / WrappedOp are uncategorised, so they appear in NO category index. + assert not ({"FilterOp", "WrappedOp"} & registry.list_classes(category="engine")) + assert {"HuggingFaceSource", "DatasetSplit", "RangeSource", "ConcatSource"} <= registry.list_classes( + category="source" + ) + assert { + "RescaleOp", + "StandardizeOp", + "ThresholdOp", + "Tee", + "MetadataToTargetOp", + "EncodeTargetOp", + "DecodeTargetOp", + } <= registry.list_classes(category="op") + + +def test_groups_enumerable_via_registry() -> None: + """The registry's group index must surface the tagged ops (``list_classes(group=...)``).""" + registry = get_registry() + assert {"RescaleOp", "StandardizeOp", "ThresholdOp"} <= registry.list_classes(group="numpy") + assert {"Tee", "Parallel"} <= registry.list_classes(group="compose") + assert {"MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp"} <= registry.list_classes(group="structure") + # group × category intersect, like task × role. + assert "Tee" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_coverage_gap.py b/tests/test_coverage_gap.py index 2d0f568..9797105 100644 --- a/tests/test_coverage_gap.py +++ b/tests/test_coverage_gap.py @@ -10,7 +10,7 @@ from dataflux.storage.base import Storage from dataflux.storage.directory import DirectorySink from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink +from dataflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource def test_storage_base_close() -> None: @@ -154,3 +154,36 @@ def test_zarr_batch_chunks(tmp_path: Path) -> None: p = tmp_path / "chunks.zarr" sink = ZarrBatchSink(p, shape=[10], chunks=[1, 10], overwrite=True) sink.write(Sample(input=np.zeros(10))) + + +def test_zarr_group_source_none_root() -> None: + # hits the unopened-root guard branches in ZarrGroupSource + source = ZarrGroupSource("nonexistent.zarr") + source.open = lambda: source # type: ignore + assert list(source) == [] + assert len(source) == 0 + + +def test_zarr_batch_source_none_array() -> None: + # hits the unopened-array guard branches in ZarrBatchSource + source = ZarrBatchSource("nonexistent.zarr") + source.open = lambda: source # type: ignore + assert list(source) == [] + assert len(source) == 0 + + +def test_zarr_sources_close(tmp_path: Path) -> None: + # hits ZarrGroupSource.close / ZarrBatchSource.close + g = tmp_path / "g.zarr" + ZarrGroupSink(g, overwrite=True).write(Sample(input=np.zeros(3))) + gsrc = ZarrGroupSource(g) + gsrc.open() + gsrc.close() + assert gsrc._root is None + + b = tmp_path / "b.zarr" + ZarrBatchSink(b, shape=[3], overwrite=True).write(Sample(input=np.zeros(3))) + bsrc = ZarrBatchSource(b) + bsrc.open() + bsrc.close() + assert bsrc._data_arr is None diff --git a/tests/test_hf_core.py b/tests/test_hf_core.py deleted file mode 100644 index 339d5c7..0000000 --- a/tests/test_hf_core.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Any, Dict - -from dataflux.hf_core import HFFlux -from dataflux.sample import Sample - - -def test_hfflux_from_list() -> None: - raw_data = [ - Sample(input="data1", target=0, metadata={"id": "s1"}), - Sample(input="data2", target=1, metadata={"id": "s2"}), - ] - flux = HFFlux.from_source(raw_data) - assert len(flux) == 2 - assert flux[0].input == "data1" - assert flux[0].metadata["id"] == "s1" - - -def test_hfflux_map() -> None: - raw_data = [Sample(input=1), Sample(input=2)] - flux = HFFlux.from_source(raw_data) - - # HF style map expects a dict and returns a dict - def double(example: Dict[str, Any]) -> Dict[str, Any]: - example["input"] = example["input"] * 2 - return example - - mapped = flux.map(double) - assert mapped[0].input == 2 - assert mapped[1].input == 4 - - -def test_hfflux_filter() -> None: - raw_data = [Sample(input=1), Sample(input=2), Sample(input=3)] - flux = HFFlux.from_source(raw_data) - - filtered = flux.filter(lambda x: x["input"] > 1) - assert len(filtered) == 2 - assert filtered[0].input == 2 diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py new file mode 100644 index 0000000..81a463d --- /dev/null +++ b/tests/test_image_ops.py @@ -0,0 +1,147 @@ +"""Tests for :mod:`dataflux.ops.image` — generic value→image conversion. + +``ConvertToImageOp`` is the generic image-conversion op (normalize → colormap → +optional flip → resize), and ``value_to_image`` / ``sample_to_image`` back it +(and FluxStudio's preview). The signal-specific overlay drawing lives in +waivefront (``RenderOverlaysOp``) and is tested there. +""" + +from typing import get_args + +import numpy as np +import pytest +import torch +from PIL import Image + +from dataflux.ops.image import COLORMAPS, Colormap, ConvertToImageOp, _apply_colormap, sample_to_image, value_to_image +from dataflux.sample import Sample + + +def _sample(value: object) -> Sample: + return Sample(input=value, target=None, metadata={}) + + +# --------------------------------------------------------------------------- +# ConvertToImageOp +# --------------------------------------------------------------------------- + + +def test_convert_2d_map_to_exact_size_pil_and_publishes_dims() -> None: + arr = np.linspace(0.0, 1.0, 64 * 32, dtype=np.float32).reshape(64, 32) + out = ConvertToImageOp(colormap="gray", width=128, height=256)(_sample(arr)) + assert isinstance(out.input, Image.Image) + assert out.input.size == (128, 256) + assert out.metadata["image_width_px"] == 128 + assert out.metadata["image_height_px"] == 256 + + +def test_convert_max_size_path_bounds_longest_side() -> None: + out = ConvertToImageOp(max_size=256)(_sample(np.zeros((1000, 400), dtype=np.float32))) + assert max(out.input.size) == 256 + # Dims are published from the actual rendered raster. + assert out.metadata["image_width_px"] == out.input.width + assert out.metadata["image_height_px"] == out.input.height + + +def test_convert_flip_vertical_mirrors_top_to_bottom() -> None: + m = np.zeros((10, 4), dtype=np.float32) + m[0, :] = 1.0 # row 0 bright + noflip = np.asarray(ConvertToImageOp(colormap="gray", flip_vertical=False)(_sample(m)).input.convert("L")) + flip = np.asarray(ConvertToImageOp(colormap="gray", flip_vertical=True)(_sample(m)).input.convert("L")) + assert noflip[0].mean() > noflip[-1].mean(), "no-flip: row 0 stays at the top" + assert flip[-1].mean() > flip[0].mean(), "flip: row 0 moves to the bottom" + + +def test_convert_colormap_gray_is_monochrome_color_is_not() -> None: + arr = np.linspace(0.0, 1.0, 100, dtype=np.float32).reshape(10, 10) + gray = np.asarray(ConvertToImageOp(colormap="gray", width=16, height=16)(_sample(arr)).input) + color = np.asarray(ConvertToImageOp(colormap="viridis", width=16, height=16)(_sample(arr)).input) + assert np.array_equal(gray[..., 0], gray[..., 1]) and np.array_equal(gray[..., 1], gray[..., 2]) + assert not np.array_equal(color[..., 0], color[..., 1]) + + +def test_convert_accepts_torch_chw_tensor() -> None: + out = ConvertToImageOp(width=64, height=48)(_sample(torch.rand(3, 100, 200))) + assert isinstance(out.input, Image.Image) + assert out.input.size == (64, 48) + + +def test_convert_accepts_pil_passthrough() -> None: + out = ConvertToImageOp(width=20, height=20)(_sample(Image.new("RGB", (8, 8), color=(10, 20, 30)))) + assert isinstance(out.input, Image.Image) + assert out.input.size == (20, 20) + + +# --------------------------------------------------------------------------- +# value_to_image / sample_to_image — generic, modality-agnostic preview +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "make_input", + [ + lambda: Image.new("L", (12, 10)), # PIL greyscale + lambda: Image.new("RGB", (12, 10)), # PIL RGB + lambda: np.random.rand(10, 12).astype(np.float32), # 2-D float -> colormap + lambda: (np.random.rand(10, 12, 3) * 255).astype(np.uint8), # 3-D HWC uint8 + lambda: np.random.rand(3, 10, 12).astype(np.float32), # 3-D CHW -> transposed + lambda: np.random.rand(10, 12) > 0.5, # boolean mask + lambda: np.random.rand(10, 12, 1).astype(np.float32), # singleton channel + lambda: np.random.rand(10, 12, 4).astype(np.float32), # RGBA -> drop alpha + lambda: torch.rand(3, 10, 12), # torch CHW tensor + ], +) +def test_sample_to_image_returns_hwc_uint8_rgb(make_input) -> None: # type: ignore[no-untyped-def] + img = sample_to_image(Sample(input=make_input())) + assert img.dtype == np.uint8 + assert img.ndim == 3 and img.shape[2] == 3 + + +def test_sample_to_image_falls_back_to_text_for_non_array() -> None: + img = sample_to_image(Sample(input=[(0, 1, 2, 3), (4, 5, 6, 7)])) + assert img.dtype == np.uint8 and img.ndim == 3 and img.shape[2] == 3 + + +def test_sample_to_image_bounds_longest_side() -> None: + img = sample_to_image(Sample(input=np.zeros((2000, 500), dtype=np.float32)), max_size=256) + assert max(img.shape[:2]) <= 256 + + +def test_sample_to_image_gray_is_monochrome_color_is_not() -> None: + arr = np.linspace(0.0, 1.0, 100).reshape(10, 10).astype(np.float32) + gray = sample_to_image(Sample(input=arr), colormap="gray") + color = sample_to_image(Sample(input=arr), colormap="viridis") + assert np.array_equal(gray[..., 0], gray[..., 1]) + assert not np.array_equal(color[..., 0], color[..., 1]) + + +def test_sample_to_image_flat_array_is_all_zero() -> None: + img = sample_to_image(Sample(input=np.full((8, 8), 5.0, dtype=np.float32)), colormap="gray") + assert int(img.max()) == 0 + + +def test_value_to_image_renders_an_arbitrary_value() -> None: + img = value_to_image(np.eye(12, dtype=bool), colormap="gray") + assert img.dtype == np.uint8 and img.ndim == 3 and img.shape[2] == 3 + + +def test_sample_to_image_delegates_to_value_to_image() -> None: + arr = np.linspace(0.0, 1.0, 64).reshape(8, 8).astype(np.float32) + assert np.array_equal(sample_to_image(Sample(input=arr)), value_to_image(arr)) + + +# --------------------------------------------------------------------------- +# Colormap closed-Literal contract +# --------------------------------------------------------------------------- + + +def test_colormaps_tuple_is_the_literal_set() -> None: + assert COLORMAPS == get_args(Colormap) + assert "viridis" in COLORMAPS and "gray" in COLORMAPS + + +def test_every_colormap_in_the_literal_set_renders() -> None: + spec_u8 = np.linspace(0, 255, 64, dtype=np.uint8).reshape(8, 8) + for cmap in COLORMAPS: + img = _apply_colormap(spec_u8, cmap) + assert img.mode == "RGB" and img.size == (8, 8) diff --git a/tests/test_intake.py b/tests/test_intake.py deleted file mode 100644 index 2e65aa7..0000000 --- a/tests/test_intake.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Tests for the generic intake → DataFlux adapter.""" - -from pathlib import Path -from typing import Any - -import confluid -import intake -import intake.source.base -import numpy as np -import pytest -import torch -import xarray as xr - -from dataflux.sample import Sample -from dataflux.storage.intake import IntakeSource - - -class _XArrayPartitionedSource(intake.source.base.DataSource): - """Minimal in-process intake DataSource for testing. - - Yields ``n_partitions`` tiny 1-D xarray.DataArray objects with attrs. - """ - - container = "xarray" - name = "test_xarray_partitioned" - version = "0.1" - partition_access = True - - def __init__(self, n_partitions: int = 3, label: str = "alpha", metadata: Any = None) -> None: - super().__init__(metadata=metadata or {}) - self._n = n_partitions - self._label = label - - def _get_schema(self) -> Any: - return intake.source.base.Schema( - datashape=None, - dtype=str(np.dtype(np.float32)), - shape=(8,), - npartitions=self._n, - extra_metadata={}, - ) - - def _get_partition(self, i: int) -> xr.DataArray: - arr = np.arange(8, dtype=np.float32) + (i * 100) - return xr.DataArray( - arr, - dims=["x"], - attrs={"label": self._label, "partition_index": i, "samplerate": 100.0}, - ) - - def _close(self) -> None: - pass - - -def test_intake_source_iterates_xarray_partitions_as_samples() -> None: - src = _XArrayPartitionedSource(n_partitions=4) - df_src = IntakeSource(source=src) - samples = list(df_src) - assert len(samples) == 4 - assert all(isinstance(s, Sample) for s in samples) - assert all(isinstance(s.input, torch.Tensor) for s in samples) - assert all(s.input.shape == (8,) for s in samples) - # Metadata propagates from xarray attrs. - assert samples[0].metadata["label"] == "alpha" - assert samples[0].metadata["samplerate"] == 100.0 - assert samples[2].metadata["partition_index"] == 2 - - -def test_intake_source_uses_target_attr() -> None: - src = _XArrayPartitionedSource(n_partitions=2, label="bravo") - df_src = IntakeSource(source=src, target_attr="label") - samples = list(df_src) - assert all(s.target == "bravo" for s in samples) - - -def test_intake_source_target_none_when_attr_missing() -> None: - src = _XArrayPartitionedSource(n_partitions=1) - df_src = IntakeSource(source=src, target_attr="not_a_real_attr") - s = next(iter(df_src)) - assert s.target is None - # The metadata still includes the actual attrs; it's only target lookup that misses. - assert "label" in s.metadata - - -def test_intake_source_len_matches_npartitions() -> None: - src = _XArrayPartitionedSource(n_partitions=7) - df_src = IntakeSource(source=src) - assert len(df_src) == 7 - - -def test_intake_source_handles_numpy_partition() -> None: - class NDArraySource(intake.source.base.DataSource): - container = "ndarray" - name = "test_ndarray" - version = "0.1" - partition_access = True - - def _get_schema(self) -> Any: - return intake.source.base.Schema( - datashape=None, - dtype=str(np.dtype(np.float64)), - shape=(3,), - npartitions=2, - extra_metadata={}, - ) - - def _get_partition(self, i: int) -> np.ndarray: - return np.array([i, i + 1, i + 2], dtype=np.float64) - - def _close(self) -> None: - pass - - df_src = IntakeSource(source=NDArraySource()) - samples = list(df_src) - assert len(samples) == 2 - assert isinstance(samples[0].input, torch.Tensor) - assert samples[0].input.tolist() == [0.0, 1.0, 2.0] - assert samples[0].metadata == {} - - -def test_intake_source_requires_either_catalog_or_source() -> None: - with pytest.raises(ValueError, match="catalog_path"): - IntakeSource() - - -def test_intake_source_rejects_both_catalog_and_source() -> None: - src = _XArrayPartitionedSource() - with pytest.raises(ValueError, match="not both"): - IntakeSource(catalog_path="a.yml", source_name="x", source=src) - - -def test_intake_source_partial_pair_raises() -> None: - with pytest.raises(ValueError): - IntakeSource(catalog_path="a.yml") - with pytest.raises(ValueError): - IntakeSource(source_name="x") - - -def test_intake_source_configurable_yaml_roundtrip() -> None: - df_src = IntakeSource(catalog_path="cat.yml", source_name="entry", target_attr="label") - state = confluid.dump(df_src) - restored = confluid.load(state) - assert isinstance(restored, IntakeSource) - assert restored.catalog_path == "cat.yml" - assert restored.source_name == "entry" - assert restored.target_attr == "label" - - -def test_intake_source_reset_after_close(tmp_path: Path) -> None: - src = _XArrayPartitionedSource(n_partitions=2) - df_src = IntakeSource(source=src) - samples_a = list(df_src) - df_src.close() - # User-supplied source is preserved; iterating again still works. - samples_b = list(df_src) - assert len(samples_a) == len(samples_b) == 2 diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index bc37326..b494f28 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -14,6 +14,7 @@ from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp from dataflux.ops.numpy import ConnectedComponentsOp, StandardizeOp, ThresholdOp +from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp from dataflux.ops.tee import Tee from dataflux.ops.torch import StandardizeOp as TorchStandardizeOp from dataflux.ops.torch import ToTensorOp @@ -31,6 +32,9 @@ ConnectedComponentsOp, ToTensorOp, TorchStandardizeOp, + MetadataToTargetOp, + EncodeTargetOp, + DecodeTargetOp, ] diff --git a/tests/test_ops.py b/tests/test_ops.py index a13ed3b..15fcea1 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -612,56 +612,125 @@ def test_missing_env_var_raises(self) -> None: class TestThresholdOp: - def test_numeric_value(self) -> None: + def test_numeric_low_level(self) -> None: arr = np.array([0.0, 1.0, 2.0, 3.0]) - out = np_ops.ThresholdOp(value=1.5)(Sample(input=arr)) + out = np_ops.ThresholdOp(low_level=1.5)(Sample(input=arr, metadata={})) np.testing.assert_array_equal(out.input, [False, False, True, True]) - assert out.metadata["threshold"] == 1.5 + assert out.metadata["threshold_low"] == 1.5 + assert "threshold_high" not in out.metadata + + def test_numeric_high_level(self) -> None: + arr = np.array([0.0, 1.0, 2.0, 3.0]) + out = np_ops.ThresholdOp(high_level=1.5)(Sample(input=arr, metadata={})) + np.testing.assert_array_equal(out.input, [True, True, False, False]) + assert out.metadata["threshold_high"] == 1.5 + assert "threshold_low" not in out.metadata + + def test_band_low_and_high(self) -> None: + arr = np.array([0.0, 1.0, 2.0, 3.0, 4.0]) + out = np_ops.ThresholdOp(low_level=1.0, high_level=3.0)(Sample(input=arr, metadata={})) + # strictly between 1.0 and 3.0 (default open interval: > and <) + np.testing.assert_array_equal(out.input, [False, False, True, False, False]) + assert out.metadata["threshold_low"] == 1.0 + assert out.metadata["threshold_high"] == 3.0 + + def test_low_level_inclusive(self) -> None: + arr = np.array([0.0, 1.0, 2.0]) + # ">" excludes the boundary; ">=" includes it. + strict = np_ops.ThresholdOp(low_level=1.0)(Sample(input=arr, metadata={})) + np.testing.assert_array_equal(strict.input, [False, False, True]) + inclusive = np_ops.ThresholdOp(low_level=1.0, low_op=">=")(Sample(input=arr, metadata={})) + np.testing.assert_array_equal(inclusive.input, [False, True, True]) + + def test_high_level_inclusive(self) -> None: + arr = np.array([1.0, 2.0, 3.0]) + # "<" excludes the boundary; "<=" includes it. + strict = np_ops.ThresholdOp(high_level=2.0)(Sample(input=arr, metadata={})) + np.testing.assert_array_equal(strict.input, [True, False, False]) + inclusive = np_ops.ThresholdOp(high_level=2.0, high_op="<=")(Sample(input=arr, metadata={})) + np.testing.assert_array_equal(inclusive.input, [True, True, False]) + + def test_closed_band(self) -> None: + arr = np.array([0.0, 1.0, 2.0, 3.0, 4.0]) + out = np_ops.ThresholdOp(low_level=1.0, high_level=3.0, low_op=">=", high_op="<=")( + Sample(input=arr, metadata={}) + ) + # closed interval [1.0, 3.0]: both boundaries kept + np.testing.assert_array_equal(out.input, [False, True, True, True, False]) + + def test_invalid_operator_rejected(self) -> None: + # ``low_op`` is a closed ``Literal[">", ">="]`` — confluid's pydantic + # validation rejects anything else before the body runs. + from pydantic import ValidationError + + with pytest.raises((ValueError, ValidationError)): + np_ops.ThresholdOp(low_level=1.0, low_op=">>") # type: ignore[arg-type] def test_string_numeric(self) -> None: arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(value="1.5")(Sample(input=arr)) + out = np_ops.ThresholdOp(low_level="1.5")(Sample(input=arr)) np.testing.assert_array_equal(out.input, [False, False, True]) def test_metadata_lookup(self) -> None: arr = np.array([-50.0, -30.0, -10.0]) sample = Sample(input=arr, target=None, metadata={"reference_snr_level": -25.0}) - out = np_ops.ThresholdOp(value="{reference_snr_level}")(sample) + out = np_ops.ThresholdOp(low_level="{reference_snr_level}")(sample) np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.metadata["threshold"] == -25.0 + assert out.metadata["threshold_low"] == -25.0 def test_metadata_lookup_with_negation(self) -> None: arr = np.array([-50.0, -30.0, -10.0]) sample = Sample(input=arr, target=None, metadata={"reference_snr_level": 30.0}) - out = np_ops.ThresholdOp(value="-{reference_snr_level}")(sample) + out = np_ops.ThresholdOp(low_level="-{reference_snr_level}")(sample) np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.metadata["threshold"] == -30.0 + assert out.metadata["threshold_low"] == -30.0 def test_env_lookup(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DATAFLUX_TEST_THRESHOLD", "1.0") arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(value="$DATAFLUX_TEST_THRESHOLD")(Sample(input=arr)) + out = np_ops.ThresholdOp(low_level="$DATAFLUX_TEST_THRESHOLD")(Sample(input=arr)) np.testing.assert_array_equal(out.input, [False, False, True]) + def test_high_level_expression(self) -> None: + arr = np.array([-50.0, -30.0, -10.0]) + sample = Sample(input=arr, target=None, metadata={"ceiling": -20.0}) + out = np_ops.ThresholdOp(high_level="{ceiling}")(sample) + np.testing.assert_array_equal(out.input, [True, True, False]) + assert out.metadata["threshold_high"] == -20.0 + + def test_raises_when_no_bounds(self) -> None: + with pytest.raises(ValueError, match="at least one of 'low_level' / 'high_level'"): + np_ops.ThresholdOp() + def test_raises_on_non_ndarray(self) -> None: with pytest.raises(TypeError, match="ThresholdOp expects an np.ndarray"): - np_ops.ThresholdOp(value=0.0)(Sample(input=[1.0, 2.0])) + np_ops.ThresholdOp(low_level=0.0)(Sample(input=[1.0, 2.0])) def test_raises_on_non_numeric_resolution(self) -> None: sample = Sample(input=np.array([0.0]), target=None, metadata={"drone": "yz"}) with pytest.raises(ValueError, match="not a number"): - np_ops.ThresholdOp(value="{drone}")(sample) + np_ops.ThresholdOp(low_level="{drone}")(sample) def test_raises_on_bad_value_type(self) -> None: # Confluid's ``@configurable`` validates kwargs against the - # auto-generated pydantic schema before the body runs. ``value`` is - # typed as ``float | int | str``, so a list is rejected at the + # auto-generated pydantic schema before the body runs. ``low_level`` is + # typed as ``float | int | str | None``, so a list is rejected at the # validation layer first; the body's hand-rolled ``TypeError`` # remains as a safety net. from pydantic import ValidationError with pytest.raises((TypeError, ValidationError)): - np_ops.ThresholdOp(value=[1, 2])(Sample(input=np.array([0.0]))) # type: ignore[arg-type] + np_ops.ThresholdOp(low_level=[1, 2])(Sample(input=np.array([0.0]))) # type: ignore[arg-type] + + +def test_threshold_comparison_maps_match_literals() -> None: + # The operator-dispatch dicts must stay in lockstep with their closed + # Literals (one source of truth) — a new operator added to the Literal but + # not the map (or vice versa) is a bug this pins. + from typing import get_args + + assert set(np_ops._LOW_COMPARISONS) == set(get_args(np_ops.LowComparison)) + assert set(np_ops._HIGH_COMPARISONS) == set(get_args(np_ops.HighComparison)) # --------------------------------------------------------------------------- diff --git a/tests/test_paired.py b/tests/test_paired.py index ee8a994..14eeedc 100644 --- a/tests/test_paired.py +++ b/tests/test_paired.py @@ -1,4 +1,4 @@ -"""Tests for dataflux.paired.PairedSource.""" +"""Tests for dataflux.paired.AnnotationJoinSource.""" from typing import Any, Dict, Iterator, Optional @@ -6,7 +6,7 @@ import pytest from dataflux.discovery import get_callable_path -from dataflux.paired import PairedSource +from dataflux.paired import AnnotationJoinSource from dataflux.sample import Sample # --------------------------------------------------------------------------- @@ -16,7 +16,7 @@ @confluid.configurable class PairedIndexedSource: - """Indexable primary source producing Samples keyed by integer id.""" + """Indexable data source producing Samples keyed by integer id.""" def __init__(self, size: int = 4) -> None: self.size = size @@ -34,7 +34,7 @@ def __iter__(self) -> Iterator[Sample]: @confluid.configurable class DictStore: - """Minimal mapping-shaped secondary for tests.""" + """Minimal mapping-shaped annotations for tests.""" def __init__(self, records: Optional[Dict[str, Dict[str, Any]]] = None) -> None: self.records: Dict[str, Dict[str, Any]] = records or {} @@ -70,10 +70,10 @@ def odd_only_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[st return None -def resolve_by_id(key: str, primary: PairedIndexedSource) -> Sample: - """Reverse-lookup: key 's' -> primary[N].""" +def resolve_by_id(key: str, data: PairedIndexedSource) -> Sample: + """Reverse-lookup: key 's' -> data[N].""" idx = int(key[1:]) - return primary[idx] + return data[idx] # --------------------------------------------------------------------------- @@ -81,10 +81,10 @@ def resolve_by_id(key: str, primary: PairedIndexedSource) -> Sample: # --------------------------------------------------------------------------- -def test_left_outer_emits_all_primary() -> None: - primary = PairedIndexedSource(size=4) +def test_left_outer_emits_all_data() -> None: + data = PairedIndexedSource(size=4) store = DictStore({"s0": {"label": "a"}, "s2": {"label": "c"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) samples = list(paired) @@ -94,9 +94,9 @@ def test_left_outer_emits_all_primary() -> None: def test_left_outer_flattens_record_into_metadata() -> None: - primary = PairedIndexedSource(size=2) + data = PairedIndexedSource(size=2) store = DictStore({"s0": {"label": "dog", "confidence": 0.9}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) samples = list(paired) @@ -108,19 +108,19 @@ def test_left_outer_flattens_record_into_metadata() -> None: def test_left_outer_preserves_original_metadata() -> None: - primary = PairedIndexedSource(size=1) + data = PairedIndexedSource(size=1) store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) sample = list(paired)[0] - assert sample.metadata["id"] == "s0" # primary metadata survived + assert sample.metadata["id"] == "s0" # data metadata survived assert sample.metadata["label"] == "x" def test_left_outer_prefix() -> None: - primary = PairedIndexedSource(size=1) + data = PairedIndexedSource(size=1) store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, prefix="ann_") + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, prefix="ann_") sample = list(paired)[0] assert sample.metadata["ann_label"] == "x" @@ -128,11 +128,11 @@ def test_left_outer_prefix() -> None: def test_left_outer_store_full_under() -> None: - primary = PairedIndexedSource(size=1) + data = PairedIndexedSource(size=1) store = DictStore({"s0": {"label": "x", "score": 0.5}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, store_full_under="raw_annotation", ) @@ -143,18 +143,18 @@ def test_left_outer_store_full_under() -> None: assert sample.metadata["label"] == "x" -def test_left_outer_len_delegates_to_primary() -> None: - primary = PairedIndexedSource(size=7) +def test_left_outer_len_delegates_to_data() -> None: + data = PairedIndexedSource(size=7) store = DictStore({"s0": {"label": "a"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) assert len(paired) == 7 def test_left_outer_getitem_matched_and_unmatched() -> None: - primary = PairedIndexedSource(size=3) + data = PairedIndexedSource(size=3) store = DictStore({"s1": {"label": "y"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) matched = paired[1] assert matched.metadata["annotated"] is True @@ -171,9 +171,9 @@ def test_left_outer_getitem_matched_and_unmatched() -> None: def test_inner_emits_only_matched() -> None: - primary = PairedIndexedSource(size=4) + data = PairedIndexedSource(size=4) store = DictStore({"s0": {"label": "a"}, "s3": {"label": "d"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, policy="inner") + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, policy="inner") samples = list(paired) @@ -183,9 +183,9 @@ def test_inner_emits_only_matched() -> None: def test_inner_len_is_cached_scan() -> None: - primary = PairedIndexedSource(size=10) + data = PairedIndexedSource(size=10) store = DictStore({f"s{i}": {"label": "x"} for i in (0, 2, 4, 6)}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, policy="inner") + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, policy="inner") assert len(paired) == 4 # Second call hits cache; should still be correct. @@ -193,9 +193,9 @@ def test_inner_len_is_cached_scan() -> None: def test_inner_rejects_getitem() -> None: - primary = PairedIndexedSource(size=2) + data = PairedIndexedSource(size=2) store = DictStore({"s0": {"label": "a"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, policy="inner") + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, policy="inner") with pytest.raises(TypeError, match="left_outer"): _ = paired[0] @@ -207,11 +207,11 @@ def test_inner_rejects_getitem() -> None: def test_extract_fn_transforms_record() -> None: - primary = PairedIndexedSource(size=2) + data = PairedIndexedSource(size=2) store = DictStore({"s0": {"label": "a"}, "s1": {"label": "b"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, extract_fn=identity_extract, ) @@ -223,11 +223,11 @@ def test_extract_fn_transforms_record() -> None: def test_extract_fn_returning_none_marks_unannotated() -> None: - primary = PairedIndexedSource(size=3) + data = PairedIndexedSource(size=3) store = DictStore({"s0": {"label": "x"}, "s1": {"label": "y"}, "s2": {"label": "z"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, extract_fn=odd_only_extract, ) @@ -239,11 +239,11 @@ def test_extract_fn_returning_none_marks_unannotated() -> None: def test_extract_fn_with_inner_policy_filters() -> None: - primary = PairedIndexedSource(size=4) + data = PairedIndexedSource(size=4) store = DictStore({f"s{i}": {"label": "x"} for i in range(4)}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, extract_fn=odd_only_extract, policy="inner", @@ -255,11 +255,11 @@ def test_extract_fn_with_inner_policy_filters() -> None: def test_extract_fn_none_suppresses_flattening() -> None: - primary = PairedIndexedSource(size=1) + data = PairedIndexedSource(size=1) store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, extract_fn=none_extract, ) @@ -280,9 +280,9 @@ def pack_key(sample: Sample) -> str: def test_coarser_key_broadcasts_to_all_matching_samples() -> None: - primary = PairedIndexedSource(size=3) + data = PairedIndexedSource(size=3) store = DictStore({"pack": {"drone": "dji_mavic"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=pack_key) + paired = AnnotationJoinSource(data=data, annotations=store, key_fn=pack_key) samples = list(paired) @@ -296,14 +296,14 @@ def test_coarser_key_broadcasts_to_all_matching_samples() -> None: def test_right_driven_iterates_annotation_keys() -> None: - primary = PairedIndexedSource(size=10) + data = PairedIndexedSource(size=10) store = DictStore({"s0": {"label": "a"}, "s3": {"label": "d"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, policy="right_driven", - primary_resolver=resolve_by_id, + data_resolver=resolve_by_id, ) samples = list(paired) @@ -313,29 +313,29 @@ def test_right_driven_iterates_annotation_keys() -> None: assert [s.input for s in samples] == [0, 30] -def test_right_driven_len_is_secondary_len() -> None: - primary = PairedIndexedSource(size=100) +def test_right_driven_len_is_annotations_len() -> None: + data = PairedIndexedSource(size=100) store = DictStore({"s1": {"label": "a"}, "s5": {"label": "b"}, "s9": {"label": "c"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, policy="right_driven", - primary_resolver=resolve_by_id, + data_resolver=resolve_by_id, ) assert len(paired) == 3 def test_right_driven_skips_when_extract_fn_returns_none() -> None: - primary = PairedIndexedSource(size=4) + data = PairedIndexedSource(size=4) store = DictStore({f"s{i}": {"label": "x"} for i in range(4)}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, policy="right_driven", - primary_resolver=resolve_by_id, + data_resolver=resolve_by_id, extract_fn=odd_only_extract, ) @@ -350,26 +350,28 @@ def test_right_driven_skips_when_extract_fn_returns_none() -> None: def test_invalid_policy_raises() -> None: - with pytest.raises(ValueError, match="Invalid policy"): - PairedSource( - primary=PairedIndexedSource(), - secondary=DictStore(), + # policy is a Literal: Confluid's @configurable validates it via pydantic at + # construction (pydantic's ValidationError is a ValueError subclass). + with pytest.raises(ValueError, match="policy"): + AnnotationJoinSource( + data=PairedIndexedSource(), + annotations=DictStore(), key_fn=sample_id_key, - policy="outer_join", + policy="outer_join", # type: ignore[arg-type] ) -def test_right_driven_requires_primary_resolver() -> None: - with pytest.raises(ValueError, match="primary_resolver"): - PairedSource( - primary=PairedIndexedSource(), - secondary=DictStore(), +def test_right_driven_requires_data_resolver() -> None: + with pytest.raises(ValueError, match="data_resolver"): + AnnotationJoinSource( + data=PairedIndexedSource(), + annotations=DictStore(), key_fn=sample_id_key, policy="right_driven", ) -def test_right_driven_requires_secondary_keys_method() -> None: +def test_right_driven_requires_annotations_keys_method() -> None: class NoKeys: def __contains__(self, k: str) -> bool: # pragma: no cover - defensive return False @@ -377,13 +379,16 @@ def __contains__(self, k: str) -> bool: # pragma: no cover - defensive def __getitem__(self, k: str) -> Any: # pragma: no cover - defensive raise KeyError(k) - with pytest.raises(TypeError, match="keys"): - PairedSource( - primary=PairedIndexedSource(), - secondary=NoKeys(), + # A store missing keys() doesn't satisfy the AnnotationStore Protocol; + # Confluid validates the param via pydantic at construction (ValidationError + # is a ValueError subclass). + with pytest.raises(ValueError, match="AnnotationStore"): + AnnotationJoinSource( + data=PairedIndexedSource(), + annotations=NoKeys(), # type: ignore[arg-type] # intentionally missing keys() key_fn=sample_id_key, policy="right_driven", - primary_resolver=resolve_by_id, + data_resolver=resolve_by_id, ) @@ -391,10 +396,11 @@ def test_left_outer_requires_mapping_interface() -> None: class NoContains: pass - with pytest.raises(TypeError, match="__contains__"): - PairedSource( - primary=PairedIndexedSource(), - secondary=NoContains(), + # Not mapping-shaped → fails the AnnotationStore Protocol at construction. + with pytest.raises(ValueError, match="AnnotationStore"): + AnnotationJoinSource( + data=PairedIndexedSource(), + annotations=NoContains(), # type: ignore[arg-type] # intentionally not mapping-shaped key_fn=sample_id_key, ) @@ -405,11 +411,11 @@ class NoContains: def test_key_fn_accepts_string_path() -> None: - primary = PairedIndexedSource(size=1) + data = PairedIndexedSource(size=1) store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=get_callable_path(sample_id_key), ) @@ -418,11 +424,11 @@ def test_key_fn_accepts_string_path() -> None: def test_extract_fn_accepts_string_path() -> None: - primary = PairedIndexedSource(size=1) + data = PairedIndexedSource(size=1) store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, extract_fn=get_callable_path(identity_extract), ) @@ -432,9 +438,9 @@ def test_extract_fn_accepts_string_path() -> None: def test_callable_is_stored_as_string() -> None: - paired = PairedSource( - primary=PairedIndexedSource(), - secondary=DictStore(), + paired = AnnotationJoinSource( + data=PairedIndexedSource(), + annotations=DictStore(), key_fn=sample_id_key, ) @@ -448,13 +454,13 @@ def test_callable_is_stored_as_string() -> None: def test_chained_paired_sources_compose() -> None: - """Pack-level + window-level annotations merged via two PairedSources.""" - primary = PairedIndexedSource(size=3) + """Pack-level + window-level annotations merged via two AnnotationJoinSources.""" + data = PairedIndexedSource(size=3) pack_store = DictStore({"pack": {"drone": "mavic"}}) window_store = DictStore({"s1": {"event": "takeoff"}}) - pack_paired = PairedSource(primary=primary, secondary=pack_store, key_fn=pack_key) - full_paired = PairedSource(primary=pack_paired, secondary=window_store, key_fn=sample_id_key) + pack_paired = AnnotationJoinSource(data=data, annotations=pack_store, key_fn=pack_key) + full_paired = AnnotationJoinSource(data=pack_paired, annotations=window_store, key_fn=sample_id_key) samples = list(full_paired) @@ -470,11 +476,11 @@ def test_chained_paired_sources_compose() -> None: def test_confluid_roundtrip_preserves_behavior() -> None: - primary = PairedIndexedSource(size=3) + data = PairedIndexedSource(size=3) store = DictStore({"s0": {"label": "a"}, "s2": {"label": "c"}}) - paired = PairedSource( - primary=primary, - secondary=store, + paired = AnnotationJoinSource( + data=data, + annotations=store, key_fn=sample_id_key, policy="left_outer", prefix="ann_", diff --git a/tests/test_projection.py b/tests/test_projection.py index 67af618..82807bf 100644 --- a/tests/test_projection.py +++ b/tests/test_projection.py @@ -1,16 +1,44 @@ """Tests for the field-projection primitive and the num_classes helper.""" import itertools -from typing import Collection, Iterator, List +from typing import Collection, Iterator, List, get_args import numpy as np import pytest import torch from dataflux.core import Flux -from dataflux.projection import SupportsProjection, _to_int, iter_inputs, iter_targets, num_classes, project +from dataflux.projection import ( + _FIELDS, + INPUT, + TARGET, + ProjectionField, + SupportsProjection, + _to_int, + iter_inputs, + iter_targets, + num_classes, + project, +) from dataflux.sample import Sample +# --------------------------------------------------------------------------- # +# ProjectionField is a closed Literal a UI / form-spec can enumerate +# --------------------------------------------------------------------------- # + + +def test_projection_field_literal_enumerates_the_field_set() -> None: + # The whole point of the Literal (vs a bare ``str``): callers — UIs, MCP + # schemas, form-spec builders — read the allowed values from the annotation. + assert get_args(ProjectionField) == ("input", "target", "metadata") + + +def test_fields_constant_is_derived_from_the_literal() -> None: + # Single source of truth: the runtime-validation tuple comes FROM the Literal, + # so the two can never drift. + assert _FIELDS == get_args(ProjectionField) + + # --------------------------------------------------------------------------- # # Fallback path (sources that do NOT implement SupportsProjection) # --------------------------------------------------------------------------- # @@ -24,21 +52,23 @@ def _plain_source() -> List[Sample]: def test_project_fallback_nulls_unrequested_fields() -> None: - out = list(project(_plain_source(), ("target",))) + out = list(project(_plain_source(), (TARGET,))) assert [s.target for s in out] == [0, 2] assert all(s.input is None for s in out) assert all(s.metadata == {} for s in out) def test_project_fallback_input_only() -> None: - out = list(project(_plain_source(), ("input",))) + out = list(project(_plain_source(), (INPUT,))) assert all(s.target is None for s in out) assert np.array_equal(out[0].input, np.array([1, 2])) def test_project_rejects_unknown_field() -> None: + # An off-type value reaches the runtime guard (the Literal is a static hint, + # not a runtime gate). mypy rightly objects — ignore it; that's the point. with pytest.raises(ValueError, match="Unknown projection field"): - list(project(_plain_source(), ("bogus",))) + list(project(_plain_source(), ("bogus",))) # type: ignore[arg-type] def test_iter_helpers() -> None: @@ -75,7 +105,7 @@ def __iter__(self) -> Iterator[Sample]: for i, t in enumerate(self.targets): yield Sample(input=self._build_input(i), target=t, metadata={}) - def project(self, fields: Collection[str]) -> Iterator[Sample]: + def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: want = frozenset(fields) for i, t in enumerate(self.targets): yield Sample( @@ -176,7 +206,7 @@ def infinite() -> Iterator[Sample]: def test_flux_project_runs_pipeline_then_drops_fields() -> None: flux = Flux([Sample(input=np.array([1]), target=7, metadata={"k": "v"})]) assert isinstance(flux, SupportsProjection) - out = list(flux.project(("target",))) + out = list(flux.project((TARGET,))) assert out[0].target == 7 assert out[0].input is None # routed through the module-level project() too diff --git a/tests/test_sources.py b/tests/test_sources.py index 0a17937..264d7bf 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1,5 +1,6 @@ -"""Tests for DataFlux sources, in particular DatasetSplit.""" +"""Tests for DataFlux sources: DatasetSplit (+ cached split views), RangeSource, ConcatSource.""" +import inspect from typing import Any, Iterator, List import confluid # type: ignore[import-not-found] @@ -7,12 +8,12 @@ from dataflux.core import Flux from dataflux.sample import Sample -from dataflux.sources import DatasetSplit +from dataflux.sources import ConcatSource, DatasetSplit, RangeSource @confluid.configurable class IndexedSource: - """A configurable indexable source for DatasetSplit tests. + """A configurable indexable source for the tests. Stores a list of integers; each `__getitem__` returns ``Sample(input=i)``. """ @@ -32,12 +33,12 @@ def __iter__(self) -> Iterator[Sample]: # --------------------------------------------------------------------------- -# Fraction mode +# DatasetSplit — select-one API (split=) # --------------------------------------------------------------------------- def test_fraction_mode_partitions_cleanly() -> None: - """train + val cover the full source with no overlap.""" + """split='train' + split='val' cover the full source with no overlap.""" source = IndexedSource(size=100) train = DatasetSplit(source=source, split="train", val_fraction=0.1, seed=42) val = DatasetSplit(source=source, split="val", val_fraction=0.1, seed=42) @@ -72,14 +73,9 @@ def test_fraction_mode_requires_seed() -> None: DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.1) -def test_fraction_mode_requires_val_fraction() -> None: - with pytest.raises(ValueError, match="val_fraction"): - DatasetSplit(source=IndexedSource(size=10), split="train", seed=0) - - def test_fraction_mode_rejects_invalid_split() -> None: with pytest.raises(ValueError, match="split"): - DatasetSplit(source=IndexedSource(size=10), split="test", val_fraction=0.1, seed=0) + DatasetSplit(source=IndexedSource(size=10), split="holdout", val_fraction=0.1, seed=0) # type: ignore[arg-type] def test_fraction_mode_rejects_out_of_range_fraction() -> None: @@ -89,42 +85,96 @@ def test_fraction_mode_rejects_out_of_range_fraction() -> None: # --------------------------------------------------------------------------- -# Range mode +# DatasetSplit — three-way (select-one) # --------------------------------------------------------------------------- -def test_range_mode_slices_source() -> None: - source = IndexedSource(size=20) - view = DatasetSplit(source=source, start=5, end=15) - values = [s.input for s in view] - assert values == list(range(5, 15)) - assert len(view) == 10 +def test_three_way_split_partitions_cleanly() -> None: + """split='train'/'val'/'test' cover the full source with no overlap.""" + source = IndexedSource(size=100) + kw = dict(source=source, val_fraction=0.2, test_fraction=0.1, seed=42) + train = DatasetSplit(split="train", **kw) # type: ignore[arg-type] + val = DatasetSplit(split="val", **kw) # type: ignore[arg-type] + test = DatasetSplit(split="test", **kw) # type: ignore[arg-type] + train_idx = {s.input for s in train} + val_idx = {s.input for s in val} + test_idx = {s.input for s in test} -def test_range_mode_open_ended() -> None: - source = IndexedSource(size=20) - head = DatasetSplit(source=source, end=10) - tail = DatasetSplit(source=source, start=10) - assert [s.input for s in head] == list(range(10)) - assert [s.input for s in tail] == list(range(10, 20)) + assert len(val) == 20 + assert len(test) == 10 + assert len(train) == 70 + assert train_idx.isdisjoint(val_idx) + assert train_idx.isdisjoint(test_idx) + assert val_idx.isdisjoint(test_idx) + assert train_idx | val_idx | test_idx == set(range(100)) -def test_range_mode_clamps_out_of_bounds() -> None: - source = IndexedSource(size=5) - view = DatasetSplit(source=source, start=-10, end=100) - assert len(view) == 5 +def test_test_fraction_out_of_range_rejected() -> None: + with pytest.raises(ValueError, match="test_fraction"): + DatasetSplit(source=IndexedSource(size=10), split="test", test_fraction=1.5, seed=0) + + +def test_val_plus_test_fraction_must_be_under_one() -> None: + with pytest.raises(ValueError, match="must be < 1"): + DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.6, test_fraction=0.5, seed=0) + + +# --------------------------------------------------------------------------- +# DatasetSplit — cached property API (.train / .val / .test) +# --------------------------------------------------------------------------- + + +def test_property_api_partitions_cleanly() -> None: + """A single DatasetSplit exposes the three disjoint, complementary views.""" + source = IndexedSource(size=100) + split = DatasetSplit(source=source, val_fraction=0.2, test_fraction=0.1, seed=42) + + train_idx = {s.input for s in split.train} + val_idx = {s.input for s in split.val} + test_idx = {s.input for s in split.test} + + assert len(split.train) == 70 and len(split.val) == 20 and len(split.test) == 10 + assert train_idx.isdisjoint(val_idx) + assert train_idx.isdisjoint(test_idx) + assert val_idx.isdisjoint(test_idx) + assert train_idx | val_idx | test_idx == set(range(100)) + + +def test_property_views_are_cached() -> None: + split = DatasetSplit(source=IndexedSource(size=20), val_fraction=0.25, seed=1) + assert split.train is split.train # memoized — same object each access + assert split.val is split.val + assert split.test is split.test -def test_passthrough_returns_full_source() -> None: +def test_property_and_select_one_agree() -> None: + """``split='val'`` (select-one) yields the same indices as the ``.val`` property.""" + source = IndexedSource(size=40) + selected = DatasetSplit(source=source, split="val", val_fraction=0.25, seed=3) + split = DatasetSplit(source=source, val_fraction=0.25, seed=3) + assert [s.input for s in selected] == [s.input for s in split.val] + + +def test_no_fractions_train_is_full_val_test_empty() -> None: + """No fractions (and no seed needed) → train is the whole source (unshuffled), val/test empty.""" source = IndexedSource(size=8) - view = DatasetSplit(source=source) - assert [s.input for s in view] == list(range(8)) + split = DatasetSplit(source=source) + assert [s.input for s in split.train] == list(range(8)) + assert len(split.val) == 0 + assert len(split.test) == 0 + + +def test_default_iteration_is_train() -> None: + """Iterating a DatasetSplit with no ``split`` delegates to the ``train`` view.""" + split = DatasetSplit(source=IndexedSource(size=10), val_fraction=0.2, seed=1) + assert [s.input for s in split] == [s.input for s in split.train] -def test_mixed_modes_rejected() -> None: - source = IndexedSource(size=10) - with pytest.raises(ValueError, match="not both"): - DatasetSplit(source=source, split="train", val_fraction=0.1, seed=0, start=0, end=5) +def test_datasetsplit_dropped_range_params() -> None: + """Range mode moved to RangeSource — DatasetSplit no longer accepts start/end.""" + params = set(inspect.signature(DatasetSplit).parameters) + assert {"source", "split", "val_fraction", "test_fraction", "seed"} == params def test_invalid_source_type() -> None: @@ -137,37 +187,102 @@ class _Plain: # --------------------------------------------------------------------------- -# Indexing +# RangeSource (the extracted contiguous-slice mode) # --------------------------------------------------------------------------- -def test_getitem_resolves_through_underlying_source() -> None: - source = IndexedSource(size=30) - view = DatasetSplit(source=source, start=10, end=20) - # First element of the view is index 10 of the source +def test_range_source_slices() -> None: + source = IndexedSource(size=20) + view = RangeSource(source=source, start=5, end=15) + assert [s.input for s in view] == list(range(5, 15)) + assert len(view) == 10 + + +def test_range_source_open_ended() -> None: + source = IndexedSource(size=20) + head = RangeSource(source=source, end=10) + tail = RangeSource(source=source, start=10) + assert [s.input for s in head] == list(range(10)) + assert [s.input for s in tail] == list(range(10, 20)) + + +def test_range_source_clamps_out_of_bounds() -> None: + view = RangeSource(source=IndexedSource(size=5), start=-10, end=100) + assert len(view) == 5 + + +def test_range_source_getitem_resolves_through_underlying_source() -> None: + view = RangeSource(source=IndexedSource(size=30), start=10, end=20) assert view[0].input == 10 assert view[-1].input == 19 # Python list indexing supports negatives -# --------------------------------------------------------------------------- -# Works inside a Flux pipeline -# --------------------------------------------------------------------------- +def test_range_source_invalid_source_type() -> None: + with pytest.raises(TypeError, match="__len__"): + + class _Plain: + pass + + RangeSource(source=_Plain()) def _scale(value: int, factor: int = 1) -> int: return value * factor -def test_dataset_split_inside_flux() -> None: - source = IndexedSource(size=20) - view = DatasetSplit(source=source, start=0, end=5) +def test_range_source_inside_flux() -> None: + view = RangeSource(source=IndexedSource(size=20), start=0, end=5) flux = Flux(source=view).map(_scale, factor=10) results: List[Sample] = flux.collect() assert [s.input for s in results] == [0, 10, 20, 30, 40] # --------------------------------------------------------------------------- -# Confluid serialization round-trip +# ConcatSource (indexable join of multiple sources) +# --------------------------------------------------------------------------- + + +def test_concat_source_len_and_iter() -> None: + cat = ConcatSource(sources=[IndexedSource(size=3), IndexedSource(size=2)]) + assert len(cat) == 5 + # Each sub-source yields its own 0..n-1 inputs, walked in order. + assert [s.input for s in cat] == [0, 1, 2, 0, 1] + + +def test_concat_source_getitem_maps_to_subsource() -> None: + cat = ConcatSource(sources=[IndexedSource(size=3), IndexedSource(size=2)]) + assert [cat[i].input for i in range(5)] == [0, 1, 2, 0, 1] # 3 from src0, 2 from src1 + assert cat[-1].input == 1 # last item of src1 + + +def test_concat_source_out_of_bounds() -> None: + cat = ConcatSource(sources=[IndexedSource(size=3)]) + with pytest.raises(IndexError): + cat[3] + + +def test_concat_source_empty() -> None: + cat = ConcatSource(sources=[]) + assert len(cat) == 0 + with pytest.raises(IndexError): + cat[0] + + +def test_concat_source_rejects_non_indexable() -> None: + with pytest.raises(TypeError, match="__len__"): + ConcatSource(sources=[object()]) + + +def test_concat_source_is_splittable() -> None: + """A ConcatSource is indexable, so DatasetSplit can partition the joined sources.""" + cat = ConcatSource(sources=[IndexedSource(size=30), IndexedSource(size=20)]) + split = DatasetSplit(source=cat, val_fraction=0.2, seed=1) + assert len(split.train) == 40 and len(split.val) == 10 + assert len(split.train) + len(split.val) == 50 + + +# --------------------------------------------------------------------------- +# Confluid serialization round-trips # --------------------------------------------------------------------------- @@ -185,10 +300,76 @@ def test_serialization_roundtrip_preserves_split() -> None: assert [s.input for s in restored] == [s.input for s in split] -def test_ref_based_splits_share_source_and_partition_cleanly() -> None: - """Two DatasetSplits referencing the same YAML source via ``!ref:`` share - the underlying source by identity (single load) and produce disjoint, - complementary views.""" +def test_serialization_roundtrip_preserves_three_way_split() -> None: + source = IndexedSource(size=60) + split = DatasetSplit(source=source, split="test", val_fraction=0.2, test_fraction=0.1, seed=5) + yaml_state = confluid.dump(split) + assert "!class:DatasetSplit" in yaml_state + assert "test_fraction: 0.1" in yaml_state + + restored: Any = confluid.load(yaml_state) + assert len(restored) == len(split) + assert [s.input for s in restored] == [s.input for s in split] + + +def test_property_split_roundtrip_via_views() -> None: + """A property-style DatasetSplit (no ``split``) round-trips; views recompute identically.""" + source = IndexedSource(size=40) + split = DatasetSplit(source=source, val_fraction=0.25, seed=123) + yaml_state = confluid.dump(split) + assert "!class:DatasetSplit" in yaml_state + restored: Any = confluid.load(yaml_state) + assert [s.input for s in restored.val] == [s.input for s in split.val] + assert [s.input for s in restored.train] == [s.input for s in split.train] + + +def test_property_refs_share_one_instance_and_partition_cleanly() -> None: + """ONE DatasetSplit referenced via ``!ref:my_split.train`` / ``.val`` — the new pattern. + + Both attribute-refs resolve from the SAME flowed DatasetSplit (Confluid's dotted-ref now reuses + the materialized instance — see confluid ``test_dotted_attribute_ref_reuses_single_instance``), + so the cached views share the single underlying source — it is the document's ``hf`` instance, + loaded exactly once — and form a disjoint, complementary partition. + """ + yaml_state = """ +hf: !class:IndexedSource() + size: 50 + +my_split: !class:DatasetSplit() + source: !ref:hf + val_fraction: 0.2 + seed: 9 + +train_set: !class:dataflux.core.Flux() + source: !ref:my_split.train + +val_set: !class:dataflux.core.Flux() + source: !ref:my_split.val +""" + state: Any = confluid.load(yaml_state) + train_flux = state["train_set"] + val_flux = state["val_set"] + + # Both Flux sources are views off the SAME DatasetSplit, wrapping the SINGLE ``hf`` instance + # (one load), and the shared split's cached property IS the view the Flux received. + assert train_flux.source.source is val_flux.source.source + assert train_flux.source.source is state["hf"] + assert state["my_split"].train is train_flux.source + + train_idx = {s.input for s in train_flux} + val_idx = {s.input for s in val_flux} + assert len(val_idx) == 10 + assert train_idx.isdisjoint(val_idx) + assert train_idx | val_idx == set(range(50)) + + +def test_select_one_refs_share_source_and_partition_cleanly() -> None: + """Select-one pattern: two DatasetSplits over a shared ``!ref:source`` — a single load. + + Both ``!ref:hf_train`` resolve (by Confluid's instance memo) to the SAME source instance, so + the source is materialized exactly once and the two views partition it cleanly. This is the + load-once-guaranteed pattern when a single shared instance matters (e.g. ``HuggingFaceSource``). + """ yaml_state = """ hf_train: !class:IndexedSource() size: 50 @@ -209,8 +390,6 @@ def test_ref_based_splits_share_source_and_partition_cleanly() -> None: train = state["train_set"] val = state["val_set"] - # Confluid !ref: resolves to the same live object — critical for - # expensive sources like HuggingFaceSource so the dataset loads once. assert train.source is val.source assert train.source is state["hf_train"] @@ -220,18 +399,54 @@ def test_ref_based_splits_share_source_and_partition_cleanly() -> None: assert train_idx | val_idx == set(range(50)) +def test_concat_source_roundtrip() -> None: + cat = ConcatSource(sources=[IndexedSource(size=3), IndexedSource(size=4)]) + yaml_state = confluid.dump(cat) + assert "!class:ConcatSource" in yaml_state + restored: Any = confluid.load(yaml_state) + assert len(restored) == 7 + assert [s.input for s in restored] == [s.input for s in cat] + + +# --------------------------------------------------------------------------- +# HuggingFaceSource lazy / zero-arg construction (no network in __init__) +# --------------------------------------------------------------------------- + + +def test_hf_source_zero_arg_construction_does_no_work() -> None: + # Per the lazy / zero-arg convention: building the source must not touch the network and + # must succeed with no constructor arguments. Nothing is materialized until first use. + from dataflux.sources import HuggingFaceSource + + src = HuggingFaceSource() + assert src._dataset is None # nothing loaded at construction time + # Even a fully-configured source stays unmaterialized until the dataset is accessed. + configured = HuggingFaceSource(path="some/dataset", split="test", count=7) + assert configured._dataset is None + assert configured.path == "some/dataset" and configured.split == "test" and configured.count == 7 + + +def test_hf_source_dataset_without_path_raises() -> None: + # The zero-arg constructor allows an unconfigured source, but materializing one without a + # dataset id cannot succeed — the error surfaces lazily, at the `dataset` property, not in __init__. + from dataflux.sources import HuggingFaceSource + + src = HuggingFaceSource() + with pytest.raises(ValueError, match="path is empty"): + _ = src.dataset + + # --------------------------------------------------------------------------- -# HuggingFaceSource.__len__ / count semantics (no network — __init__ bypassed) +# HuggingFaceSource.__len__ / count semantics (lazy `_dataset` pre-seeded, no network) # --------------------------------------------------------------------------- def _hf_source_with_count(count: Any, dataset_len: int = 13) -> Any: - """Build a HuggingFaceSource without the network (skip __init__'s load_dataset).""" + """Build a HuggingFaceSource and pre-seed its lazy cache so `dataset` never hits the network.""" from dataflux.sources import HuggingFaceSource - src: Any = HuggingFaceSource.__new__(HuggingFaceSource) - src.count = count - src._dataset = list(range(dataset_len)) + src: Any = HuggingFaceSource(count=count) + src._dataset = list(range(dataset_len)) # short-circuits the lazy load in the `dataset` property return src @@ -248,3 +463,81 @@ def test_hf_source_len_count_none_means_all() -> None: def test_hf_source_len_positive_count_caps() -> None: assert len(_hf_source_with_count(5)) == 5 + + +# --------------------------------------------------------------------------- +# HuggingFaceSource.metadata_features resolution ("*" sentinel = the rest) +# --------------------------------------------------------------------------- + + +def test_resolve_metadata_features_none_and_empty_mean_none() -> None: + from dataflux.sources import _resolve_metadata_features + + cols = ["image", "label", "id", "source_file"] + assert _resolve_metadata_features(None, cols, "image", "label") == [] + assert _resolve_metadata_features([], cols, "image", "label") == [] + + +def test_resolve_metadata_features_explicit_list_verbatim() -> None: + from dataflux.sources import _resolve_metadata_features + + cols = ["image", "label", "id", "source_file"] + assert _resolve_metadata_features(["id"], cols, "image", "label") == ["id"] + # used verbatim — names need not exist in column_names (caller's choice) + assert _resolve_metadata_features(["id", "extra"], cols, "image", "label") == ["id", "extra"] + + +def test_resolve_metadata_features_star_is_the_rest() -> None: + from dataflux.sources import _resolve_metadata_features + + cols = ["image", "label", "id", "source_file"] + # the rest = every column except input/target, order preserved + assert _resolve_metadata_features(["*"], cols, "image", "label") == ["id", "source_file"] + # bare string form accepted (YAML users may write `metadata_features: "*"`) + assert _resolve_metadata_features("*", cols, "image", "label") == ["id", "source_file"] + + +def test_resolve_metadata_features_star_plus_extras_union() -> None: + from dataflux.sources import _resolve_metadata_features + + cols = ["image", "label", "id"] + # "*" plus a name already in the rest -> no duplicate; an out-of-columns extra is appended + assert _resolve_metadata_features(["*", "id", "note"], cols, "image", "label") == ["id", "note"] + + +def test_resolve_metadata_features_star_without_columns_degrades() -> None: + from dataflux.sources import _resolve_metadata_features + + # No column_names available (e.g. a non-Dataset backing) -> "*" yields just the extras. + assert _resolve_metadata_features(["*"], None, "image", "label") == [] + assert _resolve_metadata_features(["*", "note"], None, "image", "label") == ["note"] + + +class _StubHFDataset(list): + """A list of row-dicts that also exposes ``column_names`` like a real ``datasets.Dataset``. + + Lets the lazy ``resolved_metadata_features`` property expand the ``"*"`` sentinel against the + backing columns without touching the network — iterable + indexable + ``len``-able for free. + """ + + def __init__(self, rows: List[Any], column_names: List[str]) -> None: + super().__init__(rows) + self.column_names = column_names + + +def test_hf_source_iter_metadata_features_star_expands_on_real_dataset() -> None: + # End-to-end through __iter__: a dataset with extra columns + metadata_features="*" carries + # every non-input/target column onto Sample.metadata (plus the synthetic hf_path/hf_split). + # The "*" expansion is now lazy (resolved_metadata_features reads dataset.column_names). + from dataflux.sources import HuggingFaceSource + + rows = [{"image": i, "label": i % 2, "id": f"r{i}", "src": "a"} for i in range(3)] + src = HuggingFaceSource(path="fake/ds", split="train", metadata_features=["*"]) + src._dataset = _StubHFDataset(rows, ["image", "label", "id", "src"]) # pre-seed: no network + + samples = list(src) + assert [s.input for s in samples] == [0, 1, 2] + md = samples[0].metadata + assert md["id"] == "r0" and md["src"] == "a" + assert "image" not in md and "label" not in md # input/target excluded from metadata + assert md["hf_path"] == "fake/ds" and md["hf_split"] == "train" diff --git a/tests/test_storage.py b/tests/test_storage.py index 7bb9c45..75bf741 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import cast +import confluid import numpy as np import torch @@ -8,7 +9,7 @@ from dataflux.sample import Sample from dataflux.storage.directory import DirectorySink from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink +from dataflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource def test_hdf5_storage(tmp_path: Path) -> None: @@ -33,6 +34,57 @@ def test_hdf5_storage(tmp_path: Path) -> None: source.close() +def test_hdf5_array_metadata_roundtrip(tmp_path: Path) -> None: + """Array-valued metadata (e.g. a segmentation mask) survives the HDF5 round-trip. + + Regression test: writing such a value as an HDF5 *attribute* overflows the attribute + size limit and the old str() fallback silently truncated it. It must now be stored as a + dataset under the per-sample ``{prefix}_meta/`` group and read back byte-exact, while + scalar metadata continues to round-trip via attributes. + """ + h5_path = tmp_path / "meta.h5" + mask = np.random.randint(0, 2, size=(128, 128), dtype=np.uint8) + samples = [ + Sample( + input=torch.randn(10), + target=torch.tensor([1]), + metadata={"id": "a", "samplerate": 100.0, "mask": mask}, + ), + Sample(input=torch.randn(10), metadata={"id": "b"}), + ] + + sink = HDF5Sink(h5_path, overwrite=True) + Flux(samples).to_sink(sink) + sink.close() + + source = HDF5Source(h5_path) + loaded = list(source) + source.close() + + assert len(loaded) == 2 + # Scalar metadata round-trips via attributes. + assert loaded[0].metadata["id"] == "a" + assert loaded[0].metadata["samplerate"] == 100.0 + assert loaded[1].metadata["id"] == "b" + # Array metadata round-trips exactly (no truncation). + assert np.array_equal(loaded[0].metadata["mask"], mask) + # The sample without array metadata has no spurious mask key. + assert "mask" not in loaded[1].metadata + + +def test_hdf5_array_metadata_no_compression(tmp_path: Path) -> None: + """Array metadata is stored as a dataset even when compression is disabled.""" + h5_path = tmp_path / "meta_nc.h5" + mask = np.arange(16, dtype=np.uint8).reshape(4, 4) + + sink = HDF5Sink(h5_path, compression=None, overwrite=True) + Flux([Sample(input=np.array([1.0]), metadata={"mask": mask})]).to_sink(sink) + sink.close() + + loaded = list(HDF5Source(h5_path)) + assert np.array_equal(loaded[0].metadata["mask"], mask) + + def test_zarr_group_storage(tmp_path: Path) -> None: zarr_path = tmp_path / "test.zarr" samples = [ @@ -138,3 +190,74 @@ def test_zarr_group_with_target(tmp_path: Path) -> None: z = zarr.open_group(str(zarr_path), mode="r") grp = cast(zarr.Group, z["sample_000000"]) assert "target" in grp + + +def test_zarr_group_source_roundtrip(tmp_path: Path) -> None: + zarr_path = tmp_path / "group_rt.zarr" + samples = [ + Sample(input=np.arange(5, dtype="float32"), target=np.array([1]), metadata={"id": "a"}), + Sample(input=np.arange(3, dtype="float32"), metadata={"id": "b"}), + ] + + Flux(samples).to_sink(ZarrGroupSink(zarr_path, overwrite=True)) + + source = ZarrGroupSource(zarr_path) + loaded = list(source) + assert len(loaded) == 2 + assert len(source) == 2 + # Input is returned as a tensor (matches HDF5Source); order matches write order. + assert torch.equal(loaded[0].input, torch.arange(5, dtype=torch.float32)) + assert torch.equal(loaded[1].input, torch.arange(3, dtype=torch.float32)) + # Target round-trips; absent target stays None. + assert np.array_equal(loaded[0].target, np.array([1])) + assert loaded[1].target is None + # Metadata round-trips via group attributes. + assert loaded[0].metadata["id"] == "a" + assert loaded[1].metadata["id"] == "b" + source.close() + + +def test_zarr_group_sink_handles_torch_tensors(tmp_path: Path) -> None: + """ZarrGroupSink writes torch-tensor input/target (e.g. streamed from HDF5Source). + + Regression: zarr's ``create_array`` can't read a torch tensor's dtype, so the sink + must convert via ``to_numpy`` first — otherwise a torch-tensor sample raises + ``TypeError: Cannot interpret 'torch.float32' as a data type``. + """ + zarr_path = tmp_path / "torch.zarr" + samples = [Sample(input=torch.arange(5, dtype=torch.float32), target=torch.tensor([1]))] + Flux(samples).to_sink(ZarrGroupSink(zarr_path, overwrite=True)) + + loaded = list(ZarrGroupSource(zarr_path)) + assert len(loaded) == 1 + assert torch.equal(loaded[0].input, torch.arange(5, dtype=torch.float32)) + assert np.array_equal(loaded[0].target, np.array([1])) + + +def test_zarr_batch_source_roundtrip(tmp_path: Path) -> None: + zarr_path = tmp_path / "batch_rt.zarr" + samples = [Sample(input=np.full((4,), i, dtype=np.float32)) for i in range(3)] + + Flux(samples).to_sink(ZarrBatchSink(zarr_path, shape=[4], overwrite=True)) + + source = ZarrBatchSource(zarr_path) + loaded = list(source) + assert len(loaded) == 3 + assert len(source) == 3 + # Batch sink stores input only — one Sample per row of the leading axis. + assert [int(s.input[0]) for s in loaded] == [0, 1, 2] + assert all(s.target is None for s in loaded) + source.close() + + +def test_zarr_sources_configurable_roundtrip(tmp_path: Path) -> None: + group_src = ZarrGroupSource(tmp_path / "g.zarr", target_key="label") + restored_group = confluid.load(confluid.dump(group_src)) + assert isinstance(restored_group, ZarrGroupSource) + assert restored_group.path == group_src.path + assert restored_group.target_key == "label" + + batch_src = ZarrBatchSource(tmp_path / "b.zarr") + restored_batch = confluid.load(confluid.dump(batch_src)) + assert isinstance(restored_batch, ZarrBatchSource) + assert restored_batch.path == batch_src.path diff --git a/tests/test_target_ops.py b/tests/test_target_ops.py new file mode 100644 index 0000000..80d3d75 --- /dev/null +++ b/tests/test_target_ops.py @@ -0,0 +1,96 @@ +"""Tests for the target movers / encoders (``dataflux.ops.target``).""" + +import pytest + +from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp +from dataflux.sample import Sample + + +# --------------------------------------------------------------------------- # +# MetadataToTargetOp +# --------------------------------------------------------------------------- # +def test_metadata_to_target_moves_value() -> None: + out = MetadataToTargetOp(key="drone")(Sample(input=0, metadata={"drone": "DJI MINI3"})) + assert out.target == "DJI MINI3" + + +def test_metadata_to_target_leaves_metadata_untouched_without_target_key() -> None: + sample = Sample(input=0, metadata={"drone": "DJI MINI3"}) + out = MetadataToTargetOp(key="drone")(sample) + assert set(out.metadata) == {"drone"} + + +def test_metadata_to_target_copies_to_target_key() -> None: + sample = Sample(input=0, metadata={"drone": "DJI MINI3"}) + out = MetadataToTargetOp(key="drone", target_key="raw_label")(sample) + assert out.target == "DJI MINI3" + assert out.metadata["raw_label"] == "DJI MINI3" + + +def test_metadata_to_target_missing_key_raises() -> None: + with pytest.raises(KeyError, match="no key 'drone'"): + MetadataToTargetOp(key="drone")(Sample(input=0, metadata={"other": 1})) + + +# --------------------------------------------------------------------------- # +# EncodeTargetOp +# --------------------------------------------------------------------------- # +def test_encode_target_maps_known_value() -> None: + op = EncodeTargetOp(mapping={"DJI AVATA2": 2, "DJI MINI3": 5}) + assert op(Sample(input=0, target="DJI MINI3")).target == 5 + + +def test_encode_target_class_zero_allowed() -> None: + op = EncodeTargetOp(mapping={"first": 0, "second": 1}) + assert op(Sample(input=0, target="first")).target == 0 + + +def test_encode_target_unknown_raises() -> None: + op = EncodeTargetOp(mapping={"a": 1}) + with pytest.raises(KeyError, match="not in mapping"): + op(Sample(input=0, target="missing")) + + +def test_encode_target_unknown_substitutes_default_when_ignored() -> None: + op = EncodeTargetOp(mapping={"a": 1}, ignore_unknown=True, default=7) + assert op(Sample(input=0, target="missing")).target == 7 + + +def test_encode_target_empty_mapping_rejected() -> None: + with pytest.raises(ValueError, match="at least one entry"): + EncodeTargetOp(mapping={}) + + +# --------------------------------------------------------------------------- # +# DecodeTargetOp +# --------------------------------------------------------------------------- # +def test_decode_target_inverts_encode() -> None: + mapping = {"DJI AVATA2": 2, "DJI MINI3": 5} + inverse = {v: k for k, v in mapping.items()} + sample = Sample(input=0, target="DJI MINI3") + encoded = EncodeTargetOp(mapping=mapping)(sample) + decoded = DecodeTargetOp(mapping=inverse)(encoded) + assert decoded.target == "DJI MINI3" + + +def test_decode_target_unknown_default_is_none() -> None: + op = DecodeTargetOp(mapping={1: "a"}, ignore_unknown=True) + assert op(Sample(input=0, target=999)).target is None + + +def test_decode_target_empty_mapping_rejected() -> None: + with pytest.raises(ValueError, match="at least one entry"): + DecodeTargetOp(mapping={}) + + +# --------------------------------------------------------------------------- # +# Composed chain (the decomposed classification label path) +# --------------------------------------------------------------------------- # +def test_metadata_to_target_then_encode() -> None: + label_to_index = {"DJI AVATA2": 2, "DJI MINI3": 5} + sample = Sample(input=0, metadata={"drone": "DJI AVATA2"}) + sample = MetadataToTargetOp(key="drone", target_key="raw_label")(sample) + sample = EncodeTargetOp(mapping=label_to_index)(sample) + assert sample.target == 2 + # raw label preserved for decode/reporting + assert sample.metadata["raw_label"] == "DJI AVATA2" diff --git a/tests/test_typespec.py b/tests/test_typespec.py index fddbc1a..02ffdb3 100644 --- a/tests/test_typespec.py +++ b/tests/test_typespec.py @@ -1,6 +1,6 @@ """Exhaustive tests for the dataflux type-spec system (matching, inference, JSON, HF bridge).""" -from typing import Any, List, Tuple, cast +from typing import Any, List, Tuple, cast, get_args import numpy as np import pytest @@ -8,9 +8,14 @@ from dataflux.core import Flux from dataflux.sample import FEATURES_KEY, SPEC_KEY, Sample from dataflux.typespec import ( + _DTYPE_FAMILIES, AnyType, ArrayType, Dim, + Dtype, + DtypeFamily, + Framework, + ImageLayout, ListType, MappingType, PythonType, @@ -31,6 +36,30 @@ class _Widget: """Module-level class so its ``__qualname__`` is the bare name (used by infer_type fallback test).""" +# -------------------------------------------------------------------------------------------------- +# Closed Literals (Framework / ImageLayout) a UI / connection-validator can enumerate +# -------------------------------------------------------------------------------------------------- + + +def test_framework_literal_enumerates_supported_frameworks() -> None: + # The point of the Literal over a bare str: choices are readable from the annotation. + assert get_args(Framework) == ("numpy", "torch", "tensorflow") + + +def test_image_layout_literal_enumerates_layouts() -> None: + assert get_args(ImageLayout) == ("CHW", "HWC") + + +def test_dtype_family_literal_matches_family_map() -> None: + # Single source of truth: the DtypeFamily Literal can't drift from the runtime family map. + assert set(get_args(DtypeFamily)) == set(_DTYPE_FAMILIES) + + +def test_dtype_literal_is_exactly_the_family_members() -> None: + # The concrete Dtype Literal is exactly the union of every family's members (no drift). + assert set(get_args(Dtype)) == set().union(*_DTYPE_FAMILIES.values()) + + # -------------------------------------------------------------------------------------------------- # Dim # -------------------------------------------------------------------------------------------------- @@ -129,7 +158,9 @@ def test_array_post_init() -> None: a = ArrayType(shape=(Dim.exact(3), Dim.any())) assert a.ndim == 2 # derived from shape assert isinstance(ArrayType(frameworks={"numpy"}).frameworks, frozenset) # set coerced to frozenset - assert ArrayType(dtype="FLOAT32").dtype == "float32" # normalized + # Off-Literal alias/casing is a *runtime-only* convenience normalized by canonical_dtype; mypy + # rightly flags the non-canonical literal at the call site — that's the point of the Dtype Literal. + assert ArrayType(dtype="FLOAT32").dtype == "float32" # type: ignore[arg-type] # normalized with pytest.raises(ValueError): ArrayType(ndim=3, shape=(Dim.any(),)) @@ -141,7 +172,9 @@ def test_array_image_constructor() -> None: hwc = ArrayType.image("HWC", channels=(1, 4)) assert hwc.shape is not None and hwc.shape[2] == Dim(1, 4, "C") with pytest.raises(ValueError): - ArrayType.image("XYZ") + # Deliberately off-type: the ImageLayout Literal is a static hint, the + # runtime still guards. mypy rightly objects — that's the point. + ArrayType.image("XYZ") # type: ignore[arg-type] def test_array_parse() -> None: @@ -494,7 +527,7 @@ def test_dataflux_op_spec_conformance() -> None: (N.RescaleOp(in_min=0, in_max=255), Sample(input=rgb.copy())), (N.ClipPercentilesOp(), Sample(input=rgb.copy())), (N.ReplaceNonFiniteOp(), Sample(input=rgb.copy())), - (N.ThresholdOp(value=0.5), Sample(input=rgb.copy())), + (N.ThresholdOp(low_level=0.5), Sample(input=rgb.copy())), (N.ConnectedComponentsOp(), Sample(input=(np.random.rand(8, 8) > 0.5))), (T.RescaleOp(in_min=0, in_max=255), Sample(input=__import__("torch").rand(3, 8, 8) * 255)), (T.StandardizeOp(mean=0.5, std=0.5), Sample(input=__import__("torch").rand(3, 8, 8))), From 91c0335e03217753b48a6809112b51690a338ec1 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 31 May 2026 16:58:47 +0200 Subject: [PATCH 005/102] feat: make all @configurable classes lazy + zero-arg; ops/sources/typespec/storage refactor - Lazy/zero-arg convention applied to every @configurable (29 classes): default all ctor params, defer validation/materialization to the use site (__call__ / cached @property / .open()); HuggingFaceSource is the reference (no network in __init__). tests/test_lazy_construction.py pins it. - Pre-existing: generic image ops (ops/image), target ops (ops/target), AnnotationJoinSource rename, typespec/projection closed Literals, DatasetSplit/RangeSource/ConcatSource, ThresholdOp two-bound; intake + hf_core + node-manifest removal. --- AGENTS.md | 2 +- dataflux/core.py | 24 ++++-- dataflux/ops/numpy.py | 51 ++++++----- dataflux/ops/parallel.py | 9 +- dataflux/ops/stash.py | 6 +- dataflux/ops/target.py | 28 +++--- dataflux/ops/tee.py | 5 +- dataflux/ops/torch.py | 22 +++-- dataflux/paired.py | 19 ++-- dataflux/sources.py | 148 +++++++++++++++++++------------- dataflux/storage/directory.py | 3 +- dataflux/storage/hdf5.py | 7 +- dataflux/storage/zarr.py | 17 ++-- tests/test_lazy_construction.py | 63 ++++++++++++++ tests/test_ops.py | 27 ++++-- tests/test_paired.py | 14 +-- tests/test_parallel_op.py | 3 +- tests/test_sources.py | 35 ++++---- tests/test_target_ops.py | 6 +- 19 files changed, 320 insertions(+), 169 deletions(-) create mode 100644 tests/test_lazy_construction.py diff --git a/AGENTS.md b/AGENTS.md index 32932cc..3a62695 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ - **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. - **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY dataflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. diff --git a/dataflux/core.py b/dataflux/core.py index f7fb5f8..87f7de6 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -113,12 +113,16 @@ class FilterOp: Args: p: Predicate ``Sample -> bool``; the sample passes through when it returns ``True``, else is dropped. + Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. """ - def __init__(self, p: Callable[[Sample], bool]): + def __init__(self, p: Optional[Callable[[Sample], bool]] = None): + # Lazy / zero-arg: store config only; a missing predicate is validated lazily in __call__. self.p = p def __call__(self, s: Sample) -> Optional[Sample]: + if self.p is None: + raise ValueError("FilterOp.p (predicate) is not set — provide a Sample->bool callable before use.") return s if self.p(s) else None @@ -128,17 +132,19 @@ class WrappedOp: Args: f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). - s: Which Sample slot to transform — ``"input"``, ``"target"``, or ``"all"`` (the whole Sample). - kw: Extra keyword arguments forwarded to the wrapped callable on every call. + Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. + s: Which Sample slot to transform — ``"input"`` (default), ``"target"``, or ``"all"`` (the whole Sample). + kw: Extra keyword arguments forwarded to the wrapped callable on every call (defaults to none). """ - def __init__(self, f: Union[str, Callable], s: str, kw: Dict[str, Any]): + def __init__(self, f: Union[str, Callable] = "", s: str = "input", kw: Optional[Dict[str, Any]] = None): from dataflux.discovery import get_callable_path - # EXPLICIT: Always store the string path for serialization + # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` + # property). EXPLICIT: always store the string path for serialization. self.f = get_callable_path(f) if callable(f) else f self.s = s - self.kw = kw + self.kw = dict(kw) if kw else {} # Internal cache for the live callable self._func_cache: Optional[Callable] = None @@ -183,10 +189,12 @@ class JointFlux: Args: fluxes: The Flux streams to concatenate; iteration walks them in order and length is their sum. + Defaults to ``None`` ⇒ an empty joint stream (zero-arg construction). """ - def __init__(self, fluxes: List["Flux"]) -> None: - self.fluxes = fluxes + def __init__(self, fluxes: Optional[List["Flux"]] = None) -> None: + # Lazy / zero-arg: store config only; no sub-fluxes ⇒ an empty stream. + self.fluxes = fluxes if fluxes is not None else [] def __iter__(self) -> Iterator[Sample]: """Iterate through all sub-fluxes sequentially.""" diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index e5421c3..ccd4649 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -79,7 +79,8 @@ class StandardizeOp: ACCEPTS = SampleType(input=_NUMERIC_OR_PIL) PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - def __init__(self, mean: Union[float, Sequence[float]], std: Union[float, Sequence[float]]): + def __init__(self, mean: Union[float, Sequence[float]] = 0.0, std: Union[float, Sequence[float]] = 1.0): + # Lazy / zero-arg: store config only. The defaults (mean 0, std 1) are an identity standardize. self.mean = mean self.std = std @@ -143,12 +144,13 @@ class ClipPercentilesOp: PRODUCES = SampleType(input=_NDARRAY) def __init__(self, low: float = 2.0, high: float = 98.0) -> None: - if not (0.0 <= low < high <= 100.0): - raise ValueError(f"ClipPercentilesOp: require 0 <= low < high <= 100; got low={low}, high={high}") + # Lazy / zero-arg: store config only; the bound relationship is validated lazily in __call__. self.low = float(low) self.high = float(high) def __call__(self, sample: Sample) -> Sample: + if not (0.0 <= self.low < self.high <= 100.0): + raise ValueError(f"ClipPercentilesOp: require 0 <= low < high <= 100; got low={self.low}, high={self.high}") arr = _require_ndarray(sample, "ClipPercentilesOp") finite = np.isfinite(arr) if not finite.any(): @@ -169,8 +171,8 @@ class RescaleOp: (``float64`` is preserved). Args: - in_min: Lower edge of the input range. Required. - in_max: Upper edge of the input range, must be ``> in_min``. Required. + in_min: Lower edge of the input range. Default ``0.0``. + in_max: Upper edge of the input range, must be ``> in_min``. Default ``1.0``. out_min: Lower edge of the output range. Default ``0.0``. out_max: Upper edge of the output range, must be ``> out_min``. Default ``1.0``. clip: When True (default), clamp values outside ``[in_min, in_max]`` @@ -182,16 +184,13 @@ class RescaleOp: def __init__( self, - in_min: float, - in_max: float, + in_min: float = 0.0, + in_max: float = 1.0, out_min: float = 0.0, out_max: float = 1.0, clip: bool = True, ) -> None: - if not (in_min < in_max): - raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={in_min}, in_max={in_max}") - if not (out_min < out_max): - raise ValueError(f"RescaleOp: require out_min < out_max; got out_min={out_min}, out_max={out_max}") + # Lazy / zero-arg: store config only; the bound relationships are validated lazily in __call__. self.in_min = float(in_min) self.in_max = float(in_max) self.out_min = float(out_min) @@ -199,6 +198,12 @@ def __init__( self.clip = bool(clip) def __call__(self, sample: Sample) -> Sample: + if not (self.in_min < self.in_max): + raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={self.in_min}, in_max={self.in_max}") + if not (self.out_min < self.out_max): + raise ValueError( + f"RescaleOp: require out_min < out_max; got out_min={self.out_min}, out_max={self.out_max}" + ) arr = sample.input if hasattr(arr, "convert"): arr = np.array(arr) @@ -229,8 +234,7 @@ class ReplaceNonFiniteOp: PRODUCES = SampleType(input=_NDARRAY) def __init__(self, value: Union[float, int, str] = "min") -> None: - if isinstance(value, str) and value not in ("min", "max"): - raise ValueError(f"ReplaceNonFiniteOp: value string must be 'min' or 'max'; got {value!r}") + # Lazy / zero-arg: store config only; the 'min'/'max' string is validated lazily in __call__. self.value = value def __call__(self, sample: Sample) -> Sample: @@ -239,6 +243,8 @@ def __call__(self, sample: Sample) -> Sample: if not non_finite.any(): return sample if isinstance(self.value, str): + if self.value not in ("min", "max"): + raise ValueError(f"ReplaceNonFiniteOp: value string must be 'min' or 'max'; got {self.value!r}") finite = ~non_finite if not finite.any(): logger.warning("ReplaceNonFiniteOp: array is entirely non-finite; passing through") @@ -282,7 +288,8 @@ class ThresholdOp: yield the CLOSED interval ``low_level <= input <= high_level``. At least one of ``low_level`` / ``high_level`` MUST be provided; passing - neither raises ``ValueError`` at construction. + neither raises ``ValueError`` when the op is applied (the zero-arg default is + deferred-valid so the op stays constructible, per the lazy-init convention). Each bound is either a numeric literal or a string expression resolved via :func:`resolve_expression` against ``sample.metadata`` and ``os.environ``: @@ -315,8 +322,8 @@ def __init__( low_op: LowComparison = ">", high_op: HighComparison = "<", ) -> None: - if low_level is None and high_level is None: - raise ValueError("ThresholdOp requires at least one of 'low_level' / 'high_level'") + # Lazy / zero-arg: store config only; the "at least one bound" requirement is validated + # lazily in __call__ so the op stays constructible with no arguments. self.low_level = low_level self.high_level = high_level self.low_op = low_op @@ -349,7 +356,8 @@ def __call__(self, sample: Sample) -> Sample: sample.metadata["threshold_high"] = high below = _HIGH_COMPARISONS[self.high_op](arr, high) mask = below if mask is None else (mask & below) - assert mask is not None # guaranteed by the __init__ presence check + if mask is None: + raise ValueError("ThresholdOp requires at least one of 'low_level' / 'high_level'") return sample._replace(input=mask) @@ -379,14 +387,15 @@ class ConnectedComponentsOp: PRODUCES = SampleType(input=PythonType("list")) def __init__(self, min_area_bins: int = 1, connectivity: int = 4) -> None: - if min_area_bins < 1: - raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") - if connectivity not in (4, 8): - raise ValueError(f"connectivity must be 4 or 8; got {connectivity!r}") + # Lazy / zero-arg: store config only; bounds are validated lazily in __call__. self.min_area_bins = int(min_area_bins) self.connectivity = int(connectivity) def __call__(self, sample: Sample) -> Sample: + if self.min_area_bins < 1: + raise ValueError(f"min_area_bins must be >= 1; got {self.min_area_bins!r}") + if self.connectivity not in (4, 8): + raise ValueError(f"connectivity must be 4 or 8; got {self.connectivity!r}") try: from scipy.ndimage import find_objects, generate_binary_structure, label except ImportError as exc: diff --git a/dataflux/ops/parallel.py b/dataflux/ops/parallel.py index 283a53d..e69caf3 100644 --- a/dataflux/ops/parallel.py +++ b/dataflux/ops/parallel.py @@ -38,10 +38,9 @@ class Parallel: workers: Number of worker processes (spawn context). Must be >= 1. """ - def __init__(self, ops: List[Any], workers: int = 4) -> None: - if workers < 1: - raise ValueError(f"Parallel(workers={workers!r}): must be >= 1") - self.ops = list(ops) + def __init__(self, ops: Optional[List[Any]] = None, workers: int = 4) -> None: + # Lazy / zero-arg: store config only; ``workers >= 1`` is validated lazily in ``stream``. + self.ops = list(ops) if ops else [] self.workers = int(workers) def _materialize_ops(self) -> None: @@ -63,6 +62,8 @@ def __call__(self, sample: Sample) -> Optional[Sample]: def stream(self, samples: Iterable[Optional[Sample]]) -> Iterator[Optional[Sample]]: """Stream-level dispatch with bounded prefetch (in-order yield).""" + if self.workers < 1: + raise ValueError(f"Parallel(workers={self.workers!r}): must be >= 1") self._materialize_ops() ctx = multiprocessing.get_context("spawn") limit = max(2 * self.workers, self.workers + 1) diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py index f80b11b..a2fbc48 100644 --- a/dataflux/ops/stash.py +++ b/dataflux/ops/stash.py @@ -31,7 +31,8 @@ class StashInputOp: don't mutate the shared array in place. """ - def __init__(self, key: str, copy: bool = False) -> None: + def __init__(self, key: str = "", copy: bool = False) -> None: + # Lazy / zero-arg: store config only. self.key = key self.copy = copy @@ -53,7 +54,8 @@ class UnstashInputOp: that no downstream op mutates the array in place. """ - def __init__(self, key: str, copy: bool = True) -> None: + def __init__(self, key: str = "", copy: bool = True) -> None: + # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. self.key = key self.copy = copy diff --git a/dataflux/ops/target.py b/dataflux/ops/target.py index cc2b95a..0612154 100644 --- a/dataflux/ops/target.py +++ b/dataflux/ops/target.py @@ -53,13 +53,15 @@ class MetadataToTargetOp: before :class:`EncodeTargetOp` overwrites it with a class id. Args: - key: Metadata key to read the value from into ``sample.target``. + key: Metadata key to read the value from into ``sample.target`` (defaults to ``""``; a missing + or empty key surfaces as a ``KeyError`` when the op runs, per the lazy-init convention). target_key: When set, the value is also written to ``metadata[target_key]`` (so the raw label survives a later ``EncodeTargetOp`` and can be decoded back). ``None`` (default) leaves ``metadata`` untouched. """ - def __init__(self, key: str, target_key: Optional[str] = None) -> None: + def __init__(self, key: str = "", target_key: Optional[str] = None) -> None: + # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. self.key = str(key) self.target_key = str(target_key) if target_key is not None else None @@ -94,14 +96,17 @@ class EncodeTargetOp: Defaults to ``0``. """ - def __init__(self, mapping: Dict[Any, Any], ignore_unknown: bool = False, default: Any = 0) -> None: - if not mapping: - raise ValueError("EncodeTargetOp: mapping must contain at least one entry.") - self.mapping = dict(mapping) + def __init__( + self, mapping: Optional[Dict[Any, Any]] = None, ignore_unknown: bool = False, default: Any = 0 + ) -> None: + # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + self.mapping = dict(mapping) if mapping else {} self.ignore_unknown = bool(ignore_unknown) self.default = default def __call__(self, sample: Sample) -> Sample: + if not self.mapping: + raise ValueError("EncodeTargetOp: mapping must contain at least one entry.") encoded = _lookup(sample.target, self.mapping, self.ignore_unknown, self.default, "EncodeTargetOp") return sample._replace(target=encoded) @@ -122,14 +127,17 @@ class DecodeTargetOp: Defaults to ``None``. """ - def __init__(self, mapping: Dict[Any, Any], ignore_unknown: bool = False, default: Any = None) -> None: - if not mapping: - raise ValueError("DecodeTargetOp: mapping must contain at least one entry.") - self.mapping = dict(mapping) + def __init__( + self, mapping: Optional[Dict[Any, Any]] = None, ignore_unknown: bool = False, default: Any = None + ) -> None: + # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + self.mapping = dict(mapping) if mapping else {} self.ignore_unknown = bool(ignore_unknown) self.default = default def __call__(self, sample: Sample) -> Sample: + if not self.mapping: + raise ValueError("DecodeTargetOp: mapping must contain at least one entry.") decoded = _lookup(sample.target, self.mapping, self.ignore_unknown, self.default, "DecodeTargetOp") return sample._replace(target=decoded) diff --git a/dataflux/ops/tee.py b/dataflux/ops/tee.py index b00c6bb..8ed9386 100644 --- a/dataflux/ops/tee.py +++ b/dataflux/ops/tee.py @@ -26,8 +26,9 @@ class Tee: ``Sample -> Optional[Sample]`` run in order. """ - def __init__(self, branches: List[List[Any]]) -> None: - self.branches = [list(b) for b in branches] + def __init__(self, branches: Optional[List[List[Any]]] = None) -> None: + # Lazy / zero-arg: store config only; no branches ⇒ __call__ passes the sample through. + self.branches = [list(b) for b in branches] if branches else [] def __call__(self, sample: Sample) -> Optional[Sample]: current: Optional[Sample] = sample diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index 70dcf38..bccf47e 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -65,8 +65,8 @@ class RescaleOp: promoted to ``float32`` (``float64`` is preserved). Args: - in_min: Lower edge of the input range. Required. - in_max: Upper edge of the input range, must be ``> in_min``. Required. + in_min: Lower edge of the input range. Default ``0.0``. + in_max: Upper edge of the input range, must be ``> in_min``. Default ``1.0``. out_min: Lower edge of the output range. Default ``0.0``. out_max: Upper edge of the output range, must be ``> out_min``. Default ``1.0``. clip: When True (default), clamp values outside ``[in_min, in_max]`` @@ -78,16 +78,13 @@ class RescaleOp: def __init__( self, - in_min: float, - in_max: float, + in_min: float = 0.0, + in_max: float = 1.0, out_min: float = 0.0, out_max: float = 1.0, clip: bool = True, ) -> None: - if not (in_min < in_max): - raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={in_min}, in_max={in_max}") - if not (out_min < out_max): - raise ValueError(f"RescaleOp: require out_min < out_max; got out_min={out_min}, out_max={out_max}") + # Lazy / zero-arg: store config only; the bound relationships are validated lazily in __call__. self.in_min = float(in_min) self.in_max = float(in_max) self.out_min = float(out_min) @@ -95,6 +92,12 @@ def __init__( self.clip = bool(clip) def __call__(self, sample: Sample) -> Sample: + if not (self.in_min < self.in_max): + raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={self.in_min}, in_max={self.in_max}") + if not (self.out_min < self.out_max): + raise ValueError( + f"RescaleOp: require out_min < out_max; got out_min={self.out_min}, out_max={self.out_max}" + ) tensor = sample.input if not isinstance(tensor, torch.Tensor): raise TypeError(f"RescaleOp expects a torch.Tensor, got {type(tensor).__name__}") @@ -124,7 +127,8 @@ class StandardizeOp: ACCEPTS = SampleType(input=_TORCH) PRODUCES = SampleType(input=_TORCH_FLOAT) - def __init__(self, mean: Union[float, Sequence[float]], std: Union[float, Sequence[float]]): + def __init__(self, mean: Union[float, Sequence[float]] = 0.0, std: Union[float, Sequence[float]] = 1.0): + # Lazy / zero-arg: store config only. The defaults (mean 0, std 1) are an identity standardize. self.mean = mean self.std = std diff --git a/dataflux/paired.py b/dataflux/paired.py index 7a2d2f8..a6e9947 100644 --- a/dataflux/paired.py +++ b/dataflux/paired.py @@ -114,9 +114,9 @@ class AnnotationJoinSource: def __init__( self, - data: Iterable[Any], - annotations: AnnotationStore, - key_fn: Union[str, Callable[[Sample], str]], + data: Any = None, + annotations: Optional[AnnotationStore] = None, + key_fn: Union[str, Callable[[Sample], str]] = "", policy: Policy = "left_outer", extract_fn: Optional[Union[str, Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]]] = None, prefix: str = "", @@ -126,14 +126,11 @@ def __init__( # those signatures against a broader annotation. data_resolver: Optional[Union[str, Callable[[str, Any], Any]]] = None, ) -> None: - # `annotations` shape is enforced by the AnnotationStore Protocol via - # pydantic at construction. Only the policy-conditional requirement that - # right_driven needs a resolver is checked here (a Protocol can't express it). - if policy == "right_driven" and data_resolver is None: - raise ValueError("policy='right_driven' requires data_resolver") - + # Lazy / zero-arg: store config only. `annotations` shape is enforced by the AnnotationStore + # Protocol via pydantic at construction; the policy-conditional "right_driven needs a resolver" + # requirement (which a Protocol can't express) is validated lazily in `_iter_right_driven`. self.data = data - self.annotations = annotations + self.annotations: AnnotationStore = annotations if annotations is not None else {} self.key_fn = get_callable_path(key_fn) if callable(key_fn) else key_fn self.policy = policy self.extract_fn = get_callable_path(extract_fn) if callable(extract_fn) else extract_fn @@ -206,6 +203,8 @@ def __iter__(self) -> Iterator[Sample]: yield self._attach(sample, record, key) def _iter_right_driven(self) -> Iterator[Sample]: + if self.data_resolver is None: + raise ValueError("policy='right_driven' requires data_resolver") resolver = self._resolved_data_resolver for key in self.annotations.keys(): raw = resolver(key, self.data) diff --git a/dataflux/sources.py b/dataflux/sources.py index a34fb5c..7237832 100644 --- a/dataflux/sources.py +++ b/dataflux/sources.py @@ -25,7 +25,7 @@ def _resolve_metadata_features( - requested: Optional[Any], + requested: Optional[List[str] | str], column_names: Optional[List[str]], input_feature: str, target_feature: str, @@ -77,7 +77,7 @@ def __init__( split: str = "train", input_feature: str = "image", target_feature: str = "label", - metadata_features: Optional[List[str]] = None, + metadata_features: Optional[List[str] | str] = "*", count: Optional[int] = None, name: Optional[str] = None, **kwargs: Any, @@ -220,7 +220,7 @@ class DatasetSplit: arithmetic happens up front; samples are produced on demand. Args: - source: The underlying indexable source. + source: The underlying indexable source (defaults to ``None``; validated lazily on first use). split: View this iterates as a source — ``train`` / ``val`` / ``test`` (``None`` ⇒ ``train``). val_fraction: Fraction of samples assigned to the ``val`` view. Must be in ``(0, 1)``. test_fraction: Fraction of samples assigned to the ``test`` view. Must be in ``(0, 1)``. @@ -229,30 +229,14 @@ class DatasetSplit: def __init__( self, - source: Any, + source: Any = None, split: Optional[SplitName] = None, val_fraction: Optional[float] = None, test_fraction: Optional[float] = None, seed: Optional[int] = None, ) -> None: - if not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): - raise TypeError( - "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" - ) - if split is not None and split not in _SPLIT_NAMES: - raise ValueError(f"split must be one of {_SPLIT_NAMES}; got {split!r}") - if (val_fraction is not None or test_fraction is not None) and seed is None: - raise ValueError("DatasetSplit requires `seed` when a fraction is set, so the partition is reproducible.") - if val_fraction is not None and not (0.0 < val_fraction < 1.0): - raise ValueError(f"val_fraction must be in (0, 1); got {val_fraction}") - if test_fraction is not None and not (0.0 < test_fraction < 1.0): - raise ValueError(f"test_fraction must be in (0, 1); got {test_fraction}") - if (val_fraction or 0.0) + (test_fraction or 0.0) >= 1.0: - raise ValueError( - "val_fraction + test_fraction must be < 1 (to leave a non-empty train split); " - f"got val_fraction={val_fraction}, test_fraction={test_fraction}" - ) - + # Lazy / zero-arg: store config only. All validation is deferred to first materialization + # (``_validate``, invoked from ``_view``) so the source can be configured post-construction. self.source = source self.split = split self.val_fraction = val_fraction @@ -264,6 +248,27 @@ def __init__( # surface as configurable attributes either). self._views: Dict[str, "_SplitView"] = {} + def _validate(self) -> None: + """Validate the (post-construction) configuration. Called lazily before the first partition.""" + source = self.source + if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + if self.split is not None and self.split not in _SPLIT_NAMES: + raise ValueError(f"split must be one of {_SPLIT_NAMES}; got {self.split!r}") + if (self.val_fraction is not None or self.test_fraction is not None) and self.seed is None: + raise ValueError("DatasetSplit requires `seed` when a fraction is set, so the partition is reproducible.") + if self.val_fraction is not None and not (0.0 < self.val_fraction < 1.0): + raise ValueError(f"val_fraction must be in (0, 1); got {self.val_fraction}") + if self.test_fraction is not None and not (0.0 < self.test_fraction < 1.0): + raise ValueError(f"test_fraction must be in (0, 1); got {self.test_fraction}") + if (self.val_fraction or 0.0) + (self.test_fraction or 0.0) >= 1.0: + raise ValueError( + "val_fraction + test_fraction must be < 1 (to leave a non-empty train split); " + f"got val_fraction={self.val_fraction}, test_fraction={self.test_fraction}" + ) + def _partition(self) -> Dict[str, List[int]]: """Deterministically partition the source indices into ``train`` / ``val`` / ``test``. @@ -288,6 +293,7 @@ def _partition(self) -> Dict[str, List[int]]: def _view(self, split: SplitName) -> "_SplitView": if split not in self._views: + self._validate() self._views[split] = _SplitView(self.source, self._partition()[split]) return self._views[split] @@ -352,40 +358,50 @@ class RangeSource: The wrapped source must implement ``__len__`` and ``__getitem__``. Args: - source: The underlying indexable source. + source: The underlying indexable source (defaults to ``None``; validated lazily on first use). start: Inclusive start index (``None`` ⇒ 0; a negative value counts from the end). end: Exclusive end index (``None`` ⇒ len(source); a negative value counts from the end). """ - def __init__(self, source: Any, start: Optional[int] = None, end: Optional[int] = None) -> None: - if not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): - raise TypeError( - "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" - ) + def __init__(self, source: Any = None, start: Optional[int] = None, end: Optional[int] = None) -> None: + # Lazy / zero-arg: store config only; the index arithmetic (and source validation) is deferred + # to the ``indices`` property so the source can be configured post-construction. self.source = source self.start = start self.end = end - n = len(source) - s = 0 if start is None else start - e = n if end is None else end - if s < 0: - s = max(0, n + s) - if e < 0: - e = max(0, n + e) - s = max(0, min(s, n)) - e = max(s, min(e, n)) - self._indices: List[int] = list(range(s, e)) - logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) + self._indices: Optional[List[int]] = None + + @property + def indices(self) -> List[int]: + """The contiguous ``[start:end)`` source indices, computed lazily on first access and cached.""" + if self._indices is None: + source = self.source + if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + n = len(source) + s = 0 if self.start is None else self.start + e = n if self.end is None else self.end + if s < 0: + s = max(0, n + s) + if e < 0: + e = max(0, n + e) + s = max(0, min(s, n)) + e = max(s, min(e, n)) + self._indices = list(range(s, e)) + logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) + return self._indices def __iter__(self) -> Iterator[Sample]: - for idx in self._indices: + for idx in self.indices: yield Sample.from_any(self.source[idx]) def __getitem__(self, index: int) -> Sample: - return Sample.from_any(self.source[self._indices[index]]) + return Sample.from_any(self.source[self.indices[index]]) def __len__(self) -> int: - return len(self._indices) + return len(self.indices) @configurable(category="source") @@ -401,26 +417,38 @@ class ConcatSource: Each sub-source must implement ``__len__`` and ``__getitem__``. Args: - sources: The indexable sources to concatenate, walked in order. + sources: The indexable sources to concatenate, walked in order (defaults to ``None`` ⇒ empty). """ - def __init__(self, sources: List[Any]) -> None: - for i, src in enumerate(sources): - if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): - raise TypeError( - "ConcatSource requires sources supporting __len__ and __getitem__; " - f"source[{i}] is {type(src).__name__}" - ) - self.sources = list(sources) - # Cumulative END offsets, for an O(log k) global-index → (sub-source, local index) map. - self._offsets: List[int] = [] - total = 0 - for src in self.sources: - total += len(src) - self._offsets.append(total) + def __init__(self, sources: Optional[List[Any]] = None) -> None: + # Lazy / zero-arg: store config only; sub-source validation + the cumulative-offset precompute + # are deferred to the ``offsets`` property so sources can be configured post-construction. + self.sources = list(sources) if sources else [] + self._offsets: Optional[List[int]] = None + + @property + def offsets(self) -> List[int]: + """Cumulative END offsets per sub-source, computed lazily on first access and cached. + + Computing them validates each sub-source (``__len__`` / ``__getitem__``); enables an + O(log k) global-index → (sub-source, local index) map. + """ + if self._offsets is None: + offsets: List[int] = [] + total = 0 + for i, src in enumerate(self.sources): + if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): + raise TypeError( + "ConcatSource requires sources supporting __len__ and __getitem__; " + f"source[{i}] is {type(src).__name__}" + ) + total += len(src) + offsets.append(total) + self._offsets = offsets + return self._offsets def __len__(self) -> int: - return self._offsets[-1] if self._offsets else 0 + return self.offsets[-1] if self.offsets else 0 def __getitem__(self, index: int) -> Sample: n = len(self) @@ -428,8 +456,8 @@ def __getitem__(self, index: int) -> Sample: index += n if not 0 <= index < n: raise IndexError(index) - j = bisect.bisect_right(self._offsets, index) - start = self._offsets[j - 1] if j > 0 else 0 + j = bisect.bisect_right(self.offsets, index) + start = self.offsets[j - 1] if j > 0 else 0 return Sample.from_any(self.sources[j][index - start]) def __iter__(self) -> Iterator[Sample]: diff --git a/dataflux/storage/directory.py b/dataflux/storage/directory.py index 53d8517..eb0c4a0 100644 --- a/dataflux/storage/directory.py +++ b/dataflux/storage/directory.py @@ -15,7 +15,8 @@ class DirectorySink(Storage, DataSink): Perfect for irregular data lengths and massive parallel writing. """ - def __init__(self, path: Union[str, Path], overwrite: bool = False, use_npz: bool = True) -> None: + def __init__(self, path: Union[str, Path] = "", overwrite: bool = False, use_npz: bool = True) -> None: + # Lazy / zero-arg: store config only; the directory is created lazily in open(). self.path = Path(path) self.overwrite = overwrite self.use_npz = use_npz diff --git a/dataflux/storage/hdf5.py b/dataflux/storage/hdf5.py index fa89273..2278f62 100644 --- a/dataflux/storage/hdf5.py +++ b/dataflux/storage/hdf5.py @@ -19,10 +19,12 @@ class HDF5Source(Storage, DataSource): def __init__( self, - path: Union[str, Path], + path: Union[str, Path] = "", sample_key: str = "data", target_key: Optional[str] = "target", ) -> None: + # Lazy / zero-arg: store config only; the file is opened lazily in open() (an unset path + # surfaces there, not in __init__). self.path = Path(path) self.sample_key = sample_key self.target_key = target_key @@ -71,10 +73,11 @@ class HDF5Sink(Storage, DataSink): def __init__( self, - path: Union[str, Path], + path: Union[str, Path] = "", compression: Optional[str] = "gzip", overwrite: bool = False, ) -> None: + # Lazy / zero-arg: store config only; the file is opened lazily in open(). self.path = Path(path) self.compression = compression self.overwrite = overwrite diff --git a/dataflux/storage/zarr.py b/dataflux/storage/zarr.py index 68c64c3..f8af8ff 100644 --- a/dataflux/storage/zarr.py +++ b/dataflux/storage/zarr.py @@ -17,7 +17,8 @@ class ZarrGroupSink(Storage, DataSink): Supports variable lengths while keeping data in a single bundle. """ - def __init__(self, path: Union[str, Path], overwrite: bool = False) -> None: + def __init__(self, path: Union[str, Path] = "", overwrite: bool = False) -> None: + # Lazy / zero-arg: store config only; the group is opened lazily in open(). self.path = str(path) self.overwrite = overwrite self._root: Optional[zarr.Group] = None @@ -74,10 +75,11 @@ class ZarrGroupSource(Storage, DataSource): def __init__( self, - path: Union[str, Path], + path: Union[str, Path] = "", sample_key: str = "data", target_key: str = "target", ) -> None: + # Lazy / zero-arg: store config only; the group is opened lazily in open(). self.path = str(path) self.sample_key = sample_key self.target_key = target_key @@ -116,14 +118,16 @@ class ZarrBatchSink(Storage, DataSink): def __init__( self, - path: Union[str, Path], - shape: List[int], + path: Union[str, Path] = "", + shape: Optional[List[int]] = None, dtype: str = "float32", chunks: Optional[List[int]] = None, overwrite: bool = False, ) -> None: + # Lazy / zero-arg: store config only; the array is created lazily in open() (an unset + # path / shape surfaces there). self.path = str(path) - self.shape = tuple(shape) + self.shape = tuple(shape) if shape else () self.dtype = dtype self.chunks = tuple(chunks) if chunks else None self.overwrite = overwrite @@ -173,7 +177,8 @@ class ZarrBatchSource(Storage, DataSource): path: Path to the Zarr store written by ZarrBatchSink (the directory holding the ``data`` array). """ - def __init__(self, path: Union[str, Path]) -> None: + def __init__(self, path: Union[str, Path] = "") -> None: + # Lazy / zero-arg: store config only; the array is opened lazily in open(). self.path = str(path) self._data_arr: Optional[zarr.Array] = None diff --git a/tests/test_lazy_construction.py b/tests/test_lazy_construction.py new file mode 100644 index 0000000..db18b01 --- /dev/null +++ b/tests/test_lazy_construction.py @@ -0,0 +1,63 @@ +"""Pins the "Lazy Initialization & Zero-Arg Construction" convention for ALL dataflux configurables. + +Every ``@configurable`` class in dataflux MUST be constructible with no arguments and do no +functional work in ``__init__`` (no I/O, no network, no eager materialization). This walks the +whole package, discovers every ``@configurable`` class, and asserts ``Cls()`` succeeds — so a +newly-added class that violates the convention (a required ctor arg, or a constructor that opens a +file / loads a dataset) fails here. See confluid ``AGENTS.md`` → "Lazy Initialization & Zero-Arg +Construction" and dataflux ``AGENTS.md`` → "Lazy Evaluation". +""" + +import importlib +import pkgutil +from typing import List + +import pytest + +import dataflux + + +def _all_dataflux_configurables() -> List[type]: + """Import every dataflux submodule and collect the ``@configurable`` classes defined in dataflux.""" + seen: dict = {} + for modinfo in pkgutil.walk_packages(dataflux.__path__, prefix="dataflux."): + try: + module = importlib.import_module(modinfo.name) + except Exception: # pragma: no cover - optional/heavy deps absent in some envs + continue + for obj in vars(module).values(): + if ( + isinstance(obj, type) + and getattr(obj, "__confluid_configurable__", False) + and getattr(obj, "__module__", "").startswith("dataflux") + ): + seen[f"{obj.__module__}.{obj.__qualname__}"] = obj + return list(seen.values()) + + +_CONFIGURABLES = _all_dataflux_configurables() + + +def test_discovery_found_the_configurables() -> None: + # Guard against the walker silently finding nothing (which would make the parametrized + # test below vacuously pass). dataflux has well over a dozen @configurable classes. + assert len(_CONFIGURABLES) >= 20 + + +@pytest.mark.parametrize("cls", _CONFIGURABLES, ids=lambda c: c.__name__) +def test_zero_arg_construction(cls: type) -> None: + # The whole point of the convention: building any configurable must succeed with no arguments + # and do no functional work (so it can be configured post-construction). + instance = cls() + assert instance is not None + + +def test_sources_do_not_materialize_on_construction() -> None: + # The lazy caches stay empty until first use — no dataset load / partition / offset compute + # happens in __init__. + from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource + + assert HuggingFaceSource()._dataset is None + assert DatasetSplit()._views == {} + assert RangeSource()._indices is None + assert ConcatSource()._offsets is None diff --git a/tests/test_ops.py b/tests/test_ops.py index 15fcea1..a463c51 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -135,12 +135,14 @@ def test_raises_on_non_tensor(self) -> None: RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=np.array([1, 2, 3]))) def test_validation_rejects_bad_input_range(self) -> None: + op = RescaleOp(in_min=10.0, in_max=10.0) # lazy: construction succeeds with pytest.raises(ValueError, match="require in_min < in_max"): - RescaleOp(in_min=10.0, in_max=10.0) + op(Sample(input=torch.zeros(2))) def test_validation_rejects_bad_output_range(self) -> None: + op = RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) with pytest.raises(ValueError, match="require out_min < out_max"): - RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) + op(Sample(input=torch.zeros(2))) def test_pipeline_to_tensor_then_rescale(self) -> None: """Integration: ToTensorOp(normalize=False) -> RescaleOp().""" @@ -297,8 +299,9 @@ def test_raises_on_non_ndarray(self) -> None: @pytest.mark.parametrize("low,high", [(50, 50), (60, 50), (-1, 50), (50, 101)]) def test_validation_rejects_bad_bounds(self, low: float, high: float) -> None: + op = np_ops.ClipPercentilesOp(low=low, high=high) # lazy: construction succeeds with pytest.raises(ValueError, match="ClipPercentilesOp: require"): - np_ops.ClipPercentilesOp(low=low, high=high) + op(Sample(input=np.array([1.0, 2.0, 3.0]))) # --------------------------------------------------------------------------- @@ -358,12 +361,14 @@ def test_raises_on_non_ndarray(self) -> None: np_ops.RescaleOp(in_min=0.0, in_max=1.0)(Sample(input=[1.0, 2.0])) def test_validation_rejects_bad_input_range(self) -> None: + op = np_ops.RescaleOp(in_min=10.0, in_max=10.0) # lazy: construction succeeds with pytest.raises(ValueError, match="require in_min < in_max"): - np_ops.RescaleOp(in_min=10.0, in_max=10.0) + op(Sample(input=np.zeros(2))) def test_validation_rejects_bad_output_range(self) -> None: + op = np_ops.RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) with pytest.raises(ValueError, match="require out_min < out_max"): - np_ops.RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) + op(Sample(input=np.zeros(2))) def test_pipeline_rescale_then_to_tensor(self) -> None: """Integration: numpy RescaleOp -> ToTensorOp(normalize=False).""" @@ -414,8 +419,9 @@ def test_raises_on_non_ndarray(self) -> None: np_ops.ReplaceNonFiniteOp()(Sample(input=torch.tensor([1.0]))) def test_validation_rejects_unknown_string(self) -> None: + op = np_ops.ReplaceNonFiniteOp(value="median") # lazy: construction succeeds with pytest.raises(ValueError, match="value string must be 'min' or 'max'"): - np_ops.ReplaceNonFiniteOp(value="median") + op(Sample(input=np.array([1.0, np.inf]))) # --------------------------------------------------------------------------- @@ -699,8 +705,9 @@ def test_high_level_expression(self) -> None: assert out.metadata["threshold_high"] == -20.0 def test_raises_when_no_bounds(self) -> None: + op = np_ops.ThresholdOp() # lazy: construction succeeds (zero-arg) with pytest.raises(ValueError, match="at least one of 'low_level' / 'high_level'"): - np_ops.ThresholdOp() + op(Sample(input=np.zeros(3))) def test_raises_on_non_ndarray(self) -> None: with pytest.raises(TypeError, match="ThresholdOp expects an np.ndarray"): @@ -784,9 +791,11 @@ def test_raises_on_non_2d(self) -> None: np_ops.ConnectedComponentsOp()(Sample(input=np.array([True, False]))) def test_validation_rejects_bad_min_area(self) -> None: + op = np_ops.ConnectedComponentsOp(min_area_bins=0) # lazy: construction succeeds with pytest.raises(ValueError, match="min_area_bins must be >= 1"): - np_ops.ConnectedComponentsOp(min_area_bins=0) + op(Sample(input=np.zeros((2, 2), dtype=bool))) def test_validation_rejects_bad_connectivity(self) -> None: + op = np_ops.ConnectedComponentsOp(connectivity=6) with pytest.raises(ValueError, match="connectivity must be 4 or 8"): - np_ops.ConnectedComponentsOp(connectivity=6) + op(Sample(input=np.zeros((2, 2), dtype=bool))) diff --git a/tests/test_paired.py b/tests/test_paired.py index 14eeedc..9421345 100644 --- a/tests/test_paired.py +++ b/tests/test_paired.py @@ -362,13 +362,15 @@ def test_invalid_policy_raises() -> None: def test_right_driven_requires_data_resolver() -> None: + # Lazy: construction succeeds; the policy-conditional requirement is validated on iteration. + src = AnnotationJoinSource( + data=PairedIndexedSource(), + annotations=DictStore(), + key_fn=sample_id_key, + policy="right_driven", + ) with pytest.raises(ValueError, match="data_resolver"): - AnnotationJoinSource( - data=PairedIndexedSource(), - annotations=DictStore(), - key_fn=sample_id_key, - policy="right_driven", - ) + list(src) def test_right_driven_requires_annotations_keys_method() -> None: diff --git a/tests/test_parallel_op.py b/tests/test_parallel_op.py index 4b142a9..5f9d7cd 100644 --- a/tests/test_parallel_op.py +++ b/tests/test_parallel_op.py @@ -101,8 +101,9 @@ def test_parallel_filter_drops_nones_and_preserves_order() -> None: def test_parallel_workers_must_be_positive() -> None: import pytest + op = Parallel(ops=[double_input], workers=0) # lazy: construction succeeds with pytest.raises(ValueError, match="workers="): - Parallel(ops=[double_input], workers=0) + list(op.stream([])) def test_parallel_close_propagates_to_inner_ops() -> None: diff --git a/tests/test_sources.py b/tests/test_sources.py index 264d7bf..1e45845 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -69,8 +69,9 @@ def test_fraction_mode_different_seeds_differ() -> None: def test_fraction_mode_requires_seed() -> None: + split = DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.1) # lazy ctor with pytest.raises(ValueError, match="seed"): - DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.1) + _ = split.train def test_fraction_mode_rejects_invalid_split() -> None: @@ -80,8 +81,9 @@ def test_fraction_mode_rejects_invalid_split() -> None: def test_fraction_mode_rejects_out_of_range_fraction() -> None: src = IndexedSource(size=10) + split = DatasetSplit(source=src, split="train", val_fraction=1.5, seed=0) # lazy ctor with pytest.raises(ValueError, match="val_fraction"): - DatasetSplit(source=src, split="train", val_fraction=1.5, seed=0) + _ = split.train # --------------------------------------------------------------------------- @@ -111,13 +113,15 @@ def test_three_way_split_partitions_cleanly() -> None: def test_test_fraction_out_of_range_rejected() -> None: + split = DatasetSplit(source=IndexedSource(size=10), split="test", test_fraction=1.5, seed=0) # lazy ctor with pytest.raises(ValueError, match="test_fraction"): - DatasetSplit(source=IndexedSource(size=10), split="test", test_fraction=1.5, seed=0) + _ = split.test def test_val_plus_test_fraction_must_be_under_one() -> None: + split = DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.6, test_fraction=0.5, seed=0) with pytest.raises(ValueError, match="must be < 1"): - DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.6, test_fraction=0.5, seed=0) + _ = split.train # --------------------------------------------------------------------------- @@ -178,12 +182,12 @@ def test_datasetsplit_dropped_range_params() -> None: def test_invalid_source_type() -> None: - with pytest.raises(TypeError, match="__len__"): - - class _Plain: - pass + class _Plain: + pass - DatasetSplit(source=_Plain()) + split = DatasetSplit(source=_Plain()) # lazy: construction succeeds + with pytest.raises(TypeError, match="__len__"): + _ = split.train # --------------------------------------------------------------------------- @@ -218,12 +222,12 @@ def test_range_source_getitem_resolves_through_underlying_source() -> None: def test_range_source_invalid_source_type() -> None: - with pytest.raises(TypeError, match="__len__"): - - class _Plain: - pass + class _Plain: + pass - RangeSource(source=_Plain()) + rng = RangeSource(source=_Plain()) # lazy: construction succeeds + with pytest.raises(TypeError, match="__len__"): + len(rng) def _scale(value: int, factor: int = 1) -> int: @@ -269,8 +273,9 @@ def test_concat_source_empty() -> None: def test_concat_source_rejects_non_indexable() -> None: + cat = ConcatSource(sources=[object()]) # lazy: construction succeeds with pytest.raises(TypeError, match="__len__"): - ConcatSource(sources=[object()]) + len(cat) def test_concat_source_is_splittable() -> None: diff --git a/tests/test_target_ops.py b/tests/test_target_ops.py index 80d3d75..1b5accc 100644 --- a/tests/test_target_ops.py +++ b/tests/test_target_ops.py @@ -57,8 +57,9 @@ def test_encode_target_unknown_substitutes_default_when_ignored() -> None: def test_encode_target_empty_mapping_rejected() -> None: + op = EncodeTargetOp(mapping={}) # lazy: construction succeeds with pytest.raises(ValueError, match="at least one entry"): - EncodeTargetOp(mapping={}) + op(Sample(input=0, target="x")) # --------------------------------------------------------------------------- # @@ -79,8 +80,9 @@ def test_decode_target_unknown_default_is_none() -> None: def test_decode_target_empty_mapping_rejected() -> None: + op = DecodeTargetOp(mapping={}) # lazy: construction succeeds with pytest.raises(ValueError, match="at least one entry"): - DecodeTargetOp(mapping={}) + op(Sample(input=0, target=1)) # --------------------------------------------------------------------------- # From 31aba22618423a0c0e37cb9fbfcf02ff8afa621a Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 1 Jun 2026 09:54:10 +0200 Subject: [PATCH 006/102] =?UTF-8?q?feat:=20add=20NormalizeToUint8Op=20?= =?UTF-8?q?=E2=80=94=20generic=20value=E2=86=92uint8=20op=20decoupled=20fr?= =?UTF-8?q?om=20colormap/PIL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone quantization stage (dB spectrogram / logit map → 8-bit grid) with optional fixed vmin/vmax; value_to_image reuses NormalizeToUint8Op.normalize_to_uint8 for its 2-D-map and float-array paths. README + tests updated. --- AGENTS.md | 2 +- README.md | 9 ++++ dataflux/ops/image.py | 88 ++++++++++++++++++++++++++++++---------- tests/test_categories.py | 4 +- tests/test_image_ops.py | 58 +++++++++++++++++++++++++- 5 files changed, 137 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3a62695..23d6834 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, and the waivefront signal/target ops + `Enable`). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` / `image` (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. +- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. diff --git a/README.md b/README.md index 1857dad..48d5de3 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,15 @@ sample = op(sample) # also publishes image_width_px / image_height_px t # Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): rgb = value_to_image(some_value, colormap="magma", max_size=512) + +# NormalizeToUint8Op: the standalone min-max value -> uint8 quantization step +# (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; +# set them to pin a fixed scale across samples (out-of-range values clamp). +from dataflux.ops.image import NormalizeToUint8Op + +sample = NormalizeToUint8Op()(sample) # auto per-array min/max +sample = NormalizeToUint8Op(vmin=-80.0, vmax=0.0)(sample) # fixed dB window across a dataset +u8 = NormalizeToUint8Op.normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # the backing @staticmethod ``` `Colormap` / `COLORMAPS` / `value_to_image` / `sample_to_image` are re-exported diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py index d10a501..deee7f4 100644 --- a/dataflux/ops/image.py +++ b/dataflux/ops/image.py @@ -18,7 +18,7 @@ need it, so the pure-greyscale path stays matplotlib-free. """ -from typing import Any, Literal, Tuple, get_args +from typing import Any, Literal, Optional, Tuple, get_args import numpy as np import torch @@ -77,24 +77,6 @@ def _apply_colormap(spec_u8: np.ndarray, colormap: Colormap) -> Image.Image: return Image.fromarray(rgb, mode="RGB") -def _to_uint8(arr: np.ndarray) -> np.ndarray: - """Min-max normalize a numeric array to ``uint8`` in ``[0, 255]``. - - Non-finite entries are treated as the finite minimum. A flat array - (``max == min``) maps to all-zeros to avoid a divide-by-zero. - """ - arr = arr.astype(np.float32) - finite = arr[np.isfinite(arr)] - if finite.size == 0: - return np.zeros(arr.shape, dtype=np.uint8) - lo = float(finite.min()) - hi = float(finite.max()) - if hi <= lo: - return np.zeros(arr.shape, dtype=np.uint8) - norm = (np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) - lo) / (hi - lo) - return (np.clip(norm, 0.0, 1.0) * 255.0).astype(np.uint8) - - def _text_to_image(text: str, width: int = 512, height: int = 160) -> np.ndarray: """Render a short string to an ``(H, W, 3)`` uint8 image (non-image fallback).""" img = Image.new("RGB", (width, height), color=(30, 30, 30)) @@ -130,7 +112,7 @@ def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: arr = arr.astype(np.uint8) * 255 if arr.ndim == 2: - return np.array(_apply_colormap(_to_uint8(arr), colormap)) + return np.array(_apply_colormap(NormalizeToUint8Op.normalize_to_uint8(arr), colormap)) if arr.ndim == 3: # Normalize channel position to trailing (HWC). if arr.shape[0] in (1, 3, 4) and arr.shape[2] not in (1, 3, 4): @@ -144,7 +126,7 @@ def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: arr = arr[..., :3] else: # 2 channels (or other) — replicate the first arr = np.repeat(arr[..., :1], 3, axis=2) - return arr if arr.dtype == np.uint8 else _to_uint8(arr) + return arr if arr.dtype == np.uint8 else NormalizeToUint8Op.normalize_to_uint8(arr) return _text_to_image(f"input ndim={arr.ndim}, shape={arr.shape}") @@ -272,10 +254,74 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=img) +@configurable(category="op", group="image") +class NormalizeToUint8Op: + """Min-max normalize ``sample.input`` to a ``uint8`` array in ``[0, 255]``. + + The generic value→``uint8`` conversion step, decoupled from any colormap or + PIL rendering (that is :class:`ConvertToImageOp`). Useful as a standalone + quantization stage — e.g. turning a dB spectrogram or a logit map into a + display-ready 8-bit grid — and as the shared math behind the renderers in + this module (:func:`value_to_image` calls :meth:`normalize_to_uint8` + directly for its 2-D-map and float-array paths). + + By default the scale is taken from the array's own finite min/max (per-array + auto-contrast). Supply ``vmin`` / ``vmax`` to pin a *fixed* range instead so + successive samples are quantized on a common scale (e.g. a constant dB window + across a dataset) — values outside the range clamp to ``0`` / ``255``. + + Non-finite entries (``NaN`` / ``±inf``) are folded to the low / high bound + before scaling; a degenerate range (``vmax <= vmin``, or a flat array under + auto bounds) maps to all-zeros to avoid a divide-by-zero. + + Args: + vmin: Lower bound mapped to ``0``; ``None`` (default) uses the array's finite minimum. + vmax: Upper bound mapped to ``255``; ``None`` (default) uses the array's finite maximum. + """ + + ACCEPTS = SampleType(input=_ArrayType(frameworks={"numpy", "torch"})) + PRODUCES = SampleType(input=_ArrayType(dtype="uint8", frameworks={"numpy"})) + + def __init__(self, vmin: Optional[float] = None, vmax: Optional[float] = None) -> None: + self.vmin = None if vmin is None else float(vmin) + self.vmax = None if vmax is None else float(vmax) + + @staticmethod + def normalize_to_uint8( + arr: np.ndarray, + vmin: Optional[float] = None, + vmax: Optional[float] = None, + ) -> np.ndarray: + """Min-max normalize ``arr`` to ``uint8`` in ``[0, 255]``. + + ``vmin`` / ``vmax`` pin the scale when given (clamping out-of-range + values); otherwise the array's finite min / max are used. Non-finite + entries are folded to the bounds; a degenerate range yields all-zeros. + """ + arr = np.asarray(arr).astype(np.float32) + finite = arr[np.isfinite(arr)] + lo = float(vmin) if vmin is not None else (float(finite.min()) if finite.size else 0.0) + hi = float(vmax) if vmax is not None else (float(finite.max()) if finite.size else 0.0) + if hi <= lo: + return np.zeros(arr.shape, dtype=np.uint8) + filled = np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) + norm = (filled - lo) / (hi - lo) + return (np.clip(norm, 0.0, 1.0) * 255.0).astype(np.uint8) + + def __call__(self, sample: Sample) -> Sample: + if self.vmin is not None and self.vmax is not None and self.vmin >= self.vmax: + raise ValueError(f"NormalizeToUint8Op: vmin must be < vmax; got vmin={self.vmin!r}, vmax={self.vmax!r}") + arr = sample.input + if isinstance(arr, torch.Tensor): + arr = arr.detach().cpu().numpy() + return sample._replace(input=self.normalize_to_uint8(arr, self.vmin, self.vmax)) + + __all__ = [ "Colormap", "COLORMAPS", "ConvertToImageOp", + "NormalizeToUint8Op", "value_to_image", "sample_to_image", ] diff --git a/tests/test_categories.py b/tests/test_categories.py index 7df34c5..1c872bc 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -11,7 +11,7 @@ from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp from dataflux.ops.copy import CopyInputOp -from dataflux.ops.image import ConvertToImageOp +from dataflux.ops.image import ConvertToImageOp, NormalizeToUint8Op from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp from dataflux.ops.parallel import Parallel from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp @@ -82,6 +82,7 @@ def test_op_group_tags() -> None: assert Tee.__confluid_group__ == "compose" assert Parallel.__confluid_group__ == "compose" assert ConvertToImageOp.__confluid_group__ == "image" + assert NormalizeToUint8Op.__confluid_group__ == "image" def test_categories_enumerable_via_registry() -> None: @@ -114,6 +115,7 @@ def test_groups_enumerable_via_registry() -> None: """The registry's group index must surface the tagged ops (``list_classes(group=...)``).""" registry = get_registry() assert {"RescaleOp", "StandardizeOp", "ThresholdOp"} <= registry.list_classes(group="numpy") + assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") assert {"Tee", "Parallel"} <= registry.list_classes(group="compose") assert {"MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp"} <= registry.list_classes(group="structure") # group × category intersect, like task × role. diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py index 81a463d..8588b06 100644 --- a/tests/test_image_ops.py +++ b/tests/test_image_ops.py @@ -13,7 +13,15 @@ import torch from PIL import Image -from dataflux.ops.image import COLORMAPS, Colormap, ConvertToImageOp, _apply_colormap, sample_to_image, value_to_image +from dataflux.ops.image import ( + COLORMAPS, + Colormap, + ConvertToImageOp, + NormalizeToUint8Op, + _apply_colormap, + sample_to_image, + value_to_image, +) from dataflux.sample import Sample @@ -130,6 +138,54 @@ def test_sample_to_image_delegates_to_value_to_image() -> None: assert np.array_equal(sample_to_image(Sample(input=arr)), value_to_image(arr)) +# --------------------------------------------------------------------------- +# NormalizeToUint8Op +# --------------------------------------------------------------------------- + + +def test_normalize_to_uint8_auto_minmax_spans_full_range() -> None: + arr = np.linspace(-3.0, 7.0, 100, dtype=np.float32).reshape(10, 10) + out = NormalizeToUint8Op.normalize_to_uint8(arr) + assert out.dtype == np.uint8 + assert int(out.min()) == 0 and int(out.max()) == 255 + + +def test_normalize_to_uint8_fixed_range_clamps_outside() -> None: + arr = np.array([[-10.0, 0.0, 5.0, 20.0]], dtype=np.float32) + out = NormalizeToUint8Op.normalize_to_uint8(arr, vmin=0.0, vmax=10.0) + # -10 and 0 clamp to 0; 5 is mid (≈127); 20 clamps to 255. + assert list(out.ravel()) == [0, 0, 127, 255] + + +def test_normalize_to_uint8_flat_array_is_all_zero() -> None: + out = NormalizeToUint8Op.normalize_to_uint8(np.full((4, 4), 9.0, dtype=np.float32)) + assert int(out.max()) == 0 + + +def test_normalize_to_uint8_handles_non_finite() -> None: + arr = np.array([[0.0, np.nan, np.inf, -np.inf, 4.0]], dtype=np.float32) + out = NormalizeToUint8Op.normalize_to_uint8(arr) + # NaN/-inf fold to the low bound (0), +inf to the high bound (4 → 255). + assert out[0, 0] == 0 and out[0, 1] == 0 and out[0, 3] == 0 + assert out[0, 2] == 255 and out[0, 4] == 255 + + +def test_normalize_to_uint8_op_converts_sample_input() -> None: + arr = np.linspace(0.0, 1.0, 64, dtype=np.float32).reshape(8, 8) + out = NormalizeToUint8Op()(Sample(input=arr)) + assert out.input.dtype == np.uint8 and out.input.shape == (8, 8) + + +def test_normalize_to_uint8_op_accepts_torch_tensor() -> None: + out = NormalizeToUint8Op()(Sample(input=torch.linspace(0, 1, 16).reshape(4, 4))) + assert isinstance(out.input, np.ndarray) and out.input.dtype == np.uint8 + + +def test_normalize_to_uint8_op_rejects_inverted_range() -> None: + with pytest.raises(ValueError, match="vmin must be < vmax"): + NormalizeToUint8Op(vmin=10.0, vmax=1.0)(Sample(input=np.zeros((2, 2), dtype=np.float32))) + + # --------------------------------------------------------------------------- # Colormap closed-Literal contract # --------------------------------------------------------------------------- From f2ae6ad76e97e79774a955987d3ae15dd1af5f9d Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 1 Jun 2026 14:23:27 +0200 Subject: [PATCH 007/102] feat: add Enable and SampleSinkOp compose ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the modality-neutral Enable (toggle an op-list via one named CLI flag) and SampleSinkOp (adapt a DataSink as a pass-through op) from waivefront into core dataflux.ops — they thread any Sample through any ops and have no signal dependency. Both are zero-arg constructible with lazy validation. Register entry points (dataflux-ops-enable/-sink), pin categories/groups in test_categories, and move test_enable here. --- AGENTS.md | 2 +- dataflux/ops/__init__.py | 6 + dataflux/ops/enable.py | 154 +++++++++++++++++++++++ dataflux/ops/sink.py | 76 ++++++++++++ pyproject.toml | 2 + tests/test_categories.py | 11 +- tests/test_enable.py | 262 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 511 insertions(+), 2 deletions(-) create mode 100644 dataflux/ops/enable.py create mode 100644 dataflux/ops/sink.py create mode 100644 tests/test_enable.py diff --git a/AGENTS.md b/AGENTS.md index 23d6834..3f22ed9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, and the waivefront signal/target ops + `Enable`). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` / `image` (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` (`Tee`/`Parallel`/`Enable`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index 9e9205a..db837cd 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -7,6 +7,8 @@ - dataflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp (tensor) - dataflux.ops.tee: Tee (fan-out branching) - dataflux.ops.parallel: Parallel (worker-pool sub-pipeline) + - dataflux.ops.enable: Enable (toggle an op-list via one named CLI flag) + - dataflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - dataflux.ops.swap: SwapInputTargetOp - dataflux.ops.stash: StashInputOp, UnstashInputOp @@ -17,7 +19,9 @@ """ from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp +from dataflux.ops.enable import Enable from dataflux.ops.parallel import Parallel +from dataflux.ops.sink import SampleSinkOp from dataflux.ops.stash import StashInputOp, UnstashInputOp from dataflux.ops.swap import SwapInputTargetOp from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp @@ -30,10 +34,12 @@ "CopySampleOp", "CopyTargetOp", "DecodeTargetOp", + "Enable", "EncodeTargetOp", "MetadataToTargetOp", "Parallel", "RescaleOp", + "SampleSinkOp", "StandardizeOp", "StashInputOp", "SwapInputTargetOp", diff --git a/dataflux/ops/enable.py b/dataflux/ops/enable.py new file mode 100644 index 0000000..a91b773 --- /dev/null +++ b/dataflux/ops/enable.py @@ -0,0 +1,154 @@ +"""``Enable`` — toggle one or more ops on/off via a single named CLI flag. + +A compose-group op (alongside ``Tee`` / ``Parallel``): wrap an inner op-list +so the whole chain can be switched on or off from one boolean attribute whose +name becomes the CLI flag. Modality-neutral — it threads any ``Sample`` +through any ops — so it lives in core dataflux, not a domain package. +""" + +from typing import List, Optional, Tuple + +from confluid import configurable +from logflow import get_logger + +from dataflux.sample import Sample + +logger = get_logger(__name__) + + +@configurable(category="op", group="compose") +class Enable: + """Wrap one or more ops so they can be toggled on/off via a single named CLI flag. + + ``ops`` is a list; even a single-op guard uses ``ops: [op]``. The wrapper + threads each sample through every op in sequence — same semantics as + listing them inline in ``Flux.ops`` — so visualization chains like + ``ConvertToImageOp`` → ``SaveImageOp`` share one toggle instead + of needing a wrapper per op. + + The toggle flag is supplied in YAML as an *extra* kwarg whose name becomes + the CLI hook — Confluid's post-construction setattr promotes it to an + instance attribute, and Liquify's ``-- `` overrides match + Fluid kwargs by name (see + :func:`liquifai.core._merge_overrides_into_fluids`). + + YAML: + + .. code-block:: yaml + + - !class:dataflux.ops.enable.Enable + visualize: false # ← any boolean attribute name works; this name IS the CLI flag + ops: + - !class:dataflux.ops.image.ConvertToImageOp {} + - !class:waivefront.visualizers.SaveImageOp + output_dir: ./segments_png + + CLI: + + .. code-block:: bash + + marainer process pipeline.yaml --visualize true + marainer process pipeline.yaml --visualize+ # polarity shorthand → True + marainer process pipeline.yaml --visualize- # polarity shorthand → False + + Inner ops stay deferred (not materialized) until the wrapper actually + fires for the first time, so guarding expensive-to-construct ops with + ``Enable(..., visualize=False)`` costs nothing at startup. + + Disambiguating multiple wrappers + -------------------------------- + When two or more ``Enable`` instances live in the same pipeline, give + each a ``name:`` in YAML. That name becomes the preferred identifier + in Confluid's hierarchy (``--help``) and Liquify's override matcher, + so you can toggle them independently: + + .. code-block:: yaml + + - !class:dataflux.ops.enable.Enable + name: overlay # dotted-override key + visualize: false # same attr name is fine — name scopes it + ops: [render-with-overlays, save-to ./debug_png] + - !class:dataflux.ops.enable.Enable + name: labelstudio + visualize: false + ops: [render-clean, save-to ./ls_png] + + CLI: + + .. code-block:: bash + + # Targeted — only the overlay chain fires. + marainer process pipeline.yaml --overlay.visualize true + + # Broadcast — every Fluid with a `visualize` kwarg flips. + marainer process pipeline.yaml --visualize true + + ``name`` is a plain string on the instance; Confluid's post-construction + paradigm setattr's it automatically from YAML with no ctor change. + + Constraints: + * ``ops`` is required and must be a non-empty list — validated **lazily** + on first call (zero-arg construction stays valid per the dataflux + "Lazy Initialization & Zero-Arg Construction" convention). + * Exactly one boolean attribute (other than ``ops`` / ``name`` and + dunders) may be set on the wrapper — that's the toggle. + ``RuntimeError`` is raised on first call if zero or multiple are + present. + + Args: + ops: Non-empty list of callables ``Sample -> Sample`` gated by the toggle. + """ + + def __init__(self, ops: Optional[List] = None) -> None: + # Lazy / zero-arg: store config only; the non-empty requirement is enforced lazily in __call__. + self.ops: List = list(ops) if ops else [] + + def _toggle(self) -> Tuple[str, bool]: + candidates = [ + (k, v) + for k, v in vars(self).items() + if k != "ops" and not k.startswith("_") and not k.startswith("__confluid_") and isinstance(v, bool) + ] + if len(candidates) != 1: + raise RuntimeError( + "Enable requires exactly one boolean toggle attribute (the CLI flag name); " + f"found {len(candidates)}: {[k for k, _ in candidates]}" + ) + return candidates[0] + + @property + def enabled(self) -> bool: + _, value = self._toggle() + return value + + @property + def flag_name(self) -> str: + name, _ = self._toggle() + return name + + def __call__(self, sample: Sample) -> Sample: + if not self.ops: + raise ValueError("Enable requires a non-empty 'ops' list.") + if not self.enabled: + return sample + from confluid import flow + from confluid.fluid import Fluid + + for i, op in enumerate(self.ops): + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + if op is None: + continue + sample = op(sample) + return sample + + def close(self) -> None: + """Propagate close to inner ops that own resources (e.g. SampleSinkOp).""" + for op in self.ops: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + +__all__ = ["Enable"] diff --git a/dataflux/ops/sink.py b/dataflux/ops/sink.py new file mode 100644 index 0000000..32e0ae5 --- /dev/null +++ b/dataflux/ops/sink.py @@ -0,0 +1,76 @@ +"""``SampleSinkOp`` — adapt a :class:`~dataflux.storage.base.DataSink` as a pass-through op. + +Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, the waivefront JSON +sinks …) slot into a ``Sample``-based op chain: on first call it opens the +sink, every call writes the sample and returns it unchanged, and ``close()`` +flushes + closes. Modality-neutral (duck-typed ``open``/``write``/``close``), +so it lives in core dataflux. +""" + +from typing import Any + +from confluid import configurable +from logflow import get_logger + +from dataflux.sample import Sample + +logger = get_logger(__name__) + + +@configurable(category="op", group="sink") +class SampleSinkOp: + """Adapter: wrap a :class:`dataflux.storage.base.DataSink` as a pass-through op. + + Sinks (``JsonPerWindowSink``, ``JsonSink``, ``HDF5Sink`` …) implement the + ``open()`` / ``write(sample)`` / ``close()`` protocol and are normally + attached to a :class:`marainer.processing.DatasetProcessor` as the + flux's terminal sink. This adapter lets the same sinks slot into any + Sample-based op chain — notably the ``ops`` list of + :class:`waivefront.sinks.DetectionPredictionsSink`, where the model's + predictions arrive as a Sample whose metadata carries the new + ``predicted_regions`` and need to be persisted to disk just like a + segment-pipeline output. + + On the first call the adapter calls ``sink.open()`` (when present); each + subsequent call forwards the Sample to ``sink.write(sample)`` and returns + the Sample unchanged. ``close()`` flushes (when present) and closes the + underlying sink — propagated by :class:`dataflux.ops.enable.Enable` and + :class:`waivefront.sinks.DetectionPredictionsSink` at end-of-run. + + YAML:: + + - !class:dataflux.ops.sink.SampleSinkOp + sink: !class:waivefront.sinks.JsonPerWindowSink + output_dir: ./predictions_per_window + + Args: + sink: A DataSink-like object exposing ``write(sample)`` (and optionally ``open``/``flush``/``close``). + """ + + def __init__(self, sink: Any = None) -> None: + # Lazy / zero-arg: store config only; a non-None sink is required lazily in __call__. + self.sink = sink + self._opened = False + + def __call__(self, sample: Sample) -> Sample: + if self.sink is None: + raise ValueError("SampleSinkOp requires a non-None 'sink'.") + if not self._opened: + opener = getattr(self.sink, "open", None) + if callable(opener): + opener() + self._opened = True + self.sink.write(sample) + return sample + + def close(self) -> None: + flush = getattr(self.sink, "flush", None) + if callable(flush): + flush() + closer = getattr(self.sink, "close", None) + if callable(closer): + closer() + self._opened = False + + +__all__ = ["SampleSinkOp"] diff --git a/pyproject.toml b/pyproject.toml index a852a9c..2ca4ec9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ dataflux-core = "dataflux.core" dataflux-sources = "dataflux.sources" dataflux-ops-parallel = "dataflux.ops.parallel" dataflux-ops-tee = "dataflux.ops.tee" +dataflux-ops-enable = "dataflux.ops.enable" +dataflux-ops-sink = "dataflux.ops.sink" dataflux-ops-stash = "dataflux.ops.stash" dataflux-ops-numpy = "dataflux.ops.numpy" dataflux-ops-torch = "dataflux.ops.torch" diff --git a/tests/test_categories.py b/tests/test_categories.py index 1c872bc..df9239b 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -11,9 +11,11 @@ from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp from dataflux.ops.copy import CopyInputOp +from dataflux.ops.enable import Enable from dataflux.ops.image import ConvertToImageOp, NormalizeToUint8Op from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp from dataflux.ops.parallel import Parallel +from dataflux.ops.sink import SampleSinkOp from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp from dataflux.ops.tee import Tee from dataflux.ops.torch import ToTensorOp @@ -61,6 +63,8 @@ def test_op_classes_tagged() -> None: assert StandardizeOp.__confluid_category__ == "op" assert ThresholdOp.__confluid_category__ == "op" assert Tee.__confluid_category__ == "op" + assert Enable.__confluid_category__ == "op" + assert SampleSinkOp.__confluid_category__ == "op" assert MetadataToTargetOp.__confluid_category__ == "op" assert EncodeTargetOp.__confluid_category__ == "op" assert DecodeTargetOp.__confluid_category__ == "op" @@ -81,8 +85,10 @@ def test_op_group_tags() -> None: assert DecodeTargetOp.__confluid_group__ == "structure" assert Tee.__confluid_group__ == "compose" assert Parallel.__confluid_group__ == "compose" + assert Enable.__confluid_group__ == "compose" assert ConvertToImageOp.__confluid_group__ == "image" assert NormalizeToUint8Op.__confluid_group__ == "image" + assert SampleSinkOp.__confluid_group__ == "sink" def test_categories_enumerable_via_registry() -> None: @@ -105,6 +111,8 @@ def test_categories_enumerable_via_registry() -> None: "StandardizeOp", "ThresholdOp", "Tee", + "Enable", + "SampleSinkOp", "MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp", @@ -116,7 +124,8 @@ def test_groups_enumerable_via_registry() -> None: registry = get_registry() assert {"RescaleOp", "StandardizeOp", "ThresholdOp"} <= registry.list_classes(group="numpy") assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") - assert {"Tee", "Parallel"} <= registry.list_classes(group="compose") + assert {"Tee", "Parallel", "Enable"} <= registry.list_classes(group="compose") + assert {"SampleSinkOp"} <= registry.list_classes(group="sink") assert {"MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp"} <= registry.list_classes(group="structure") # group × category intersect, like task × role. assert "Tee" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_enable.py b/tests/test_enable.py new file mode 100644 index 0000000..960911f --- /dev/null +++ b/tests/test_enable.py @@ -0,0 +1,262 @@ +"""Tests for :class:`dataflux.ops.enable.Enable` and :class:`dataflux.ops.sink.SampleSinkOp`. + +These modality-neutral compose helpers moved here from ``waivefront.processing`` — +they thread any ``Sample`` through any ops and have no signal dependency. +""" + +from typing import Any, Dict, List + +import confluid +import pytest + +from dataflux.ops.enable import Enable +from dataflux.ops.sink import SampleSinkOp +from dataflux.sample import Sample + + +class _CountingOp: + """Plain callable that records every invocation on a shared counter dict.""" + + def __init__(self, counter: Dict[str, int]) -> None: + self.counter = counter + + def __call__(self, sample: Sample) -> Sample: + self.counter["calls"] += 1 + new_meta = dict(sample.metadata) + new_meta["counted"] = True + return sample._replace(metadata=new_meta) + + +def _sample() -> Sample: + return Sample(input=None, target=None, metadata={"x": 1}) + + +def _wrap(ops_: Any, **toggle: bool) -> Enable: + """Build an Enable + set the toggle attribute the way Confluid would. + + ``ops_`` accepts either a single callable (wrapped into a 1-list) or a + list, so existing single-op test cases stay concise. + """ + if not isinstance(ops_, list): + ops_ = [ops_] + e = Enable(ops=ops_) + for k, v in toggle.items(): + setattr(e, k, v) + return e + + +def test_enable_disabled_passes_sample_through_untouched() -> None: + counter: Dict[str, int] = {"calls": 0} + wrapped = _wrap(_CountingOp(counter), visualize=False) + out = wrapped(_sample()) + assert counter["calls"] == 0 + assert "counted" not in out.metadata + + +def test_enable_enabled_invokes_inner_op() -> None: + counter: Dict[str, int] = {"calls": 0} + wrapped = _wrap(_CountingOp(counter), visualize=True) + out = wrapped(_sample()) + assert counter["calls"] == 1 + assert out.metadata["counted"] is True + assert wrapped.flag_name == "visualize" + + +def test_enable_requires_exactly_one_boolean_attribute() -> None: + no_toggle = Enable(ops=[lambda s: s]) + with pytest.raises(RuntimeError, match="exactly one boolean toggle"): + no_toggle(_sample()) + + two_toggles = _wrap(lambda s: s, visualize=True, debug=False) + with pytest.raises(RuntimeError, match="exactly one boolean toggle"): + two_toggles(_sample()) + + +def test_enable_flag_name_is_arbitrary() -> None: + """Any kwarg name works — the chosen name is the CLI flag the user types.""" + counter: Dict[str, int] = {"calls": 0} + wrapped = _wrap(_CountingOp(counter), debug_overlay=True) + wrapped(_sample()) + assert counter["calls"] == 1 + assert wrapped.flag_name == "debug_overlay" + + +def test_enable_multi_op_threads_sample_through_each() -> None: + counter_a: Dict[str, int] = {"calls": 0} + counter_b: Dict[str, int] = {"calls": 0} + + class _TagOp: + def __init__(self, tag: str, counter: Dict[str, int]) -> None: + self.tag = tag + self.counter = counter + + def __call__(self, sample: Sample) -> Sample: + self.counter["calls"] += 1 + new_meta = dict(sample.metadata) + tags = list(new_meta.get("tags", [])) + tags.append(self.tag) + new_meta["tags"] = tags + return sample._replace(metadata=new_meta) + + wrapped = _wrap( + [_TagOp("first", counter_a), _TagOp("second", counter_b)], + visualize=True, + ) + out = wrapped(_sample()) + assert counter_a["calls"] == 1 + assert counter_b["calls"] == 1 + assert out.metadata["tags"] == ["first", "second"] + + +def test_enable_multi_op_disabled_skips_entire_chain() -> None: + counter: Dict[str, int] = {"calls": 0} + wrapped = _wrap( + [_CountingOp(counter), _CountingOp(counter)], + visualize=False, + ) + wrapped(_sample()) + assert counter["calls"] == 0 + + +def test_enable_zero_arg_construction_then_rejects_empty_ops_on_call() -> None: + """Zero-arg / empty-ops construction succeeds (lazy convention); the non-empty + requirement is enforced on first call, not in ``__init__``.""" + empty = Enable() # zero-arg construction must work + assert empty.ops == [] + empty.visualize = True # type: ignore[attr-defined] + with pytest.raises(ValueError, match="non-empty 'ops' list"): + empty(_sample()) + + +def test_enable_preserves_name_attr_and_toggle_independence() -> None: + """Two Enable instances with distinct names carry their names through. + + Confirms that (a) ``name`` is a plain string attr that survives + construction + post-construction setattr, (b) a string-valued name + doesn't collide with ``_toggle()``'s bool-attr filter, and (c) the + two wrappers can be toggled independently when each has its own + boolean attribute. + """ + counter_a: Dict[str, int] = {"calls": 0} + counter_b: Dict[str, int] = {"calls": 0} + + overlay = Enable(ops=[_CountingOp(counter_a)]) + overlay.name = "overlay" # type: ignore[attr-defined] # set by Confluid at flow time + overlay.visualize = True # type: ignore[attr-defined] + + ls = Enable(ops=[_CountingOp(counter_b)]) + ls.name = "labelstudio" # type: ignore[attr-defined] + ls.visualize = False # type: ignore[attr-defined] + + overlay(_sample()) + ls(_sample()) + + assert counter_a["calls"] == 1 + assert counter_b["calls"] == 0 + # Names are preserved verbatim; _toggle's bool filter ignores them. + assert overlay.name == "overlay" # type: ignore[attr-defined] + assert ls.name == "labelstudio" # type: ignore[attr-defined] + assert overlay.flag_name == "visualize" + assert ls.flag_name == "visualize" + + +def test_enable_yaml_load_with_cli_style_override(tmp_path: Any) -> None: + """Mimic what Liquify's --visualize true override does to Fluid kwargs.""" + yaml_text = """\ +wrapper: + !class:dataflux.ops.enable.Enable + visualize: false + ops: + - !class:dataflux.ops.copy.CopySampleOp {} +""" + cfg = tmp_path / "enable.yaml" + cfg.write_text(yaml_text) + loaded = confluid.load(cfg) + enable_fluid = loaded["wrapper"] + assert "visualize" in enable_fluid.kwargs + # Liquify CLI override path mutates Fluid.kwargs in-place before flow(). + enable_fluid.kwargs["visualize"] = True + materialized = confluid.flow(enable_fluid) + assert isinstance(materialized, Enable) + assert materialized.enabled is True + assert materialized.flag_name == "visualize" + + +# --- SampleSinkOp & Enable.close propagation -------------------------------- + + +class _RecordingSink: + """Captures the open/write/flush/close lifecycle for assertions.""" + + def __init__(self) -> None: + self.calls: List[str] = [] + self.writes: List[Sample] = [] + + def open(self) -> "_RecordingSink": + self.calls.append("open") + return self + + def write(self, sample: Sample) -> None: + self.calls.append("write") + self.writes.append(sample) + + def flush(self) -> None: + self.calls.append("flush") + + def close(self) -> None: + self.calls.append("close") + + +def test_sample_sink_op_lazy_open_then_writes() -> None: + """``open()`` fires once on first call, write() per call, returning the sample.""" + sink = _RecordingSink() + op = SampleSinkOp(sink=sink) + + s1 = Sample(input=None, target=None, metadata={"i": 0}) + s2 = Sample(input=None, target=None, metadata={"i": 1}) + out1 = op(s1) + out2 = op(s2) + + assert out1 is s1 and out2 is s2 # pass-through + assert sink.calls == ["open", "write", "write"] + assert sink.writes == [s1, s2] + + +def test_sample_sink_op_close_flushes_and_closes() -> None: + """``close()`` calls flush() then close() so buffered sinks finalize cleanly.""" + sink = _RecordingSink() + op = SampleSinkOp(sink=sink) + op(Sample(input=None, target=None, metadata={})) + op.close() + assert sink.calls == ["open", "write", "flush", "close"] + + +def test_sample_sink_op_zero_arg_construction_then_rejects_none_on_call() -> None: + """Zero-arg construction works (lazy convention); a missing sink raises on first call.""" + op = SampleSinkOp() # zero-arg construction must work + assert op.sink is None + with pytest.raises(ValueError, match="non-None 'sink'"): + op(Sample(input=None, target=None, metadata={})) + + +def test_enable_close_propagates_into_inner_ops() -> None: + """Closing an Enable wrapper drives close() on every inner op that owns one.""" + sink = _RecordingSink() + inner = SampleSinkOp(sink=sink) + wrapped = _wrap(inner, visualize=True) + wrapped(Sample(input=None, target=None, metadata={})) + wrapped.close() + assert sink.calls == ["open", "write", "flush", "close"] + + +def test_enable_close_on_disabled_wrapper_still_propagates() -> None: + """Even when toggled off (and thus never invoked), close() must still reach + inner ops in case they were opened independently — it must NEVER raise.""" + sink = _RecordingSink() + inner = SampleSinkOp(sink=sink) + wrapped = _wrap(inner, visualize=False) + # Never called; close still safe. + wrapped.close() + # The SampleSinkOp was never opened, so there's no write — but flush+close + # are forwarded unconditionally by SampleSinkOp.close. + assert sink.calls == ["flush", "close"] From b874aa941974413a778353e5e1987c3043f68569 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 2 Jun 2026 12:31:17 +0200 Subject: [PATCH 008/102] =?UTF-8?q?feat:=20add=20LabelMap=20=E2=80=94=20fi?= =?UTF-8?q?ttable=20class-name=20<->=20id=20map?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dataflux.labels.LabelMap: bidirectional name<->id map, the fittable companion to EncodeTargetOp/DecodeTargetOp. fit() derives a deterministic mapping from a target stream (sklearn LabelEncoder), save/load round-trips marainer's class_names.json, encode_op/decode_op hand back the dataflux ops. - Zero-arg constructible, side-effect-free __init__; non-empty enforced lazily. - Exported from dataflux.__init__; scikit-learn dep (lazy-imported in fit). --- AGENTS.md | 1 + README.md | 24 ++++++++ dataflux/__init__.py | 2 + dataflux/labels.py | 141 ++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 3 +- tests/test_labels.py | 143 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 dataflux/labels.py create mode 100644 tests/test_labels.py diff --git a/AGENTS.md b/AGENTS.md index 3f22ed9..31a47f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,7 @@ - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. Storage classes are bare `@configurable` with **no** discovery `category` — they are YAML `!class:` nodes wired into source/sink slots, not FluxStudio canvas nodes (unlike the `category="source"`/`"op"` classes). **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `dataflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). +- **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`dataflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing dataflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a dataflux dependency for this. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/README.md b/README.md index 48d5de3..2a7e5c1 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,30 @@ full-iteration fallback (just without the skip-decode speedup). `num_classes` is a free function, not a `Flux` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. +### `LabelMap` — fittable name↔id encoding + +When a dataset's `target` is a class **name** rather than an integer id, `LabelMap` turns it into +the pinned encoding the `EncodeTargetOp` / `DecodeTargetOp` need — the *fittable* companion to +those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the +`class_names.json` format, and reload it at eval/predict so every stage shares one ordering: + +```python +from dataflux import LabelMap, Flux + +lm = LabelMap.fit(iter_targets(train_source)) # {"bird": 0, "cat": 1, "dog": 2} +lm.num_classes # 3 +lm.label_names # ["bird", "cat", "dog"] (id -> name) +lm.save("class_names.json") # marainer's class_names.json format + +encoded = Flux(source=train_source, ops=[lm.encode_op()]) # targets are now ints + +# Later, at eval time — reload the SAME ordering instead of refitting: +lm2 = LabelMap.load("class_names.json") +``` + +`LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the +mapping is pinned, so train / eval / predict never disagree. `scikit-learn` backs `fit` (lazy-imported). + ## 🖼 Image Conversion (`dataflux.ops.image`) The single, modality-agnostic "any value → image" layer — generic so every diff --git a/dataflux/__init__.py b/dataflux/__init__.py index 5d1818c..cce1872 100644 --- a/dataflux/__init__.py +++ b/dataflux/__init__.py @@ -3,6 +3,7 @@ """ from dataflux.core import Flux, JointFlux, WrappedOp +from dataflux.labels import LabelMap from dataflux.ops import RescaleOp, StandardizeOp, ToTensorOp from dataflux.paired import AnnotationJoinSource, AnnotationStore from dataflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project @@ -41,6 +42,7 @@ "Framework", "HuggingFaceSource", "JointFlux", + "LabelMap", "ListType", "MappingType", "ProjectionField", diff --git a/dataflux/labels.py b/dataflux/labels.py new file mode 100644 index 0000000..853851d --- /dev/null +++ b/dataflux/labels.py @@ -0,0 +1,141 @@ +"""``LabelMap`` — a bidirectional class-name ↔ integer-id map. + +The *fittable* companion to the config-pinned :class:`~dataflux.ops.target.EncodeTargetOp` / +:class:`~dataflux.ops.target.DecodeTargetOp`. Those ops carry an explicit ``mapping`` that is +**pinned in config, NOT fitted** at run time, so train / eval / predict share one identical +label→id ordering. :class:`LabelMap` is the piece that *produces* such a pinned mapping: + +* :meth:`LabelMap.fit` derives a deterministic name→id mapping from a stream of raw targets + (backed by scikit-learn's ``LabelEncoder``) — the one-time fit that happens at **train** time. +* :meth:`LabelMap.save` / :meth:`LabelMap.load` persist it (in marainer's ``class_names.json`` + format) so **eval / predict** reload the *same* mapping rather than refitting on a subset. +* :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the dataflux ops that apply it. + +So fitting happens once, then the mapping is pinned/persisted — it does NOT contradict the +"mapping pinned in config, not fitted" discipline of the ops; it is how the pin gets created. + +Zero-arg constructible (``LabelMap()`` succeeds with an empty mapping) and side-effect-free in +``__init__`` per the workspace "Lazy Initialization & Zero-Arg Construction" convention; the +non-empty requirement is validated lazily in the properties, not in the constructor. scikit-learn +is imported lazily inside :meth:`fit` so importing dataflux never pulls it in. +""" + +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Union + +from confluid import configurable + +from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp + + +@configurable +class LabelMap: + """Bidirectional class-name ↔ integer-id map (the fittable companion to ``EncodeTargetOp``). + + Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream + via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`label_names`, builds the + :class:`~dataflux.ops.target.EncodeTargetOp` / :class:`~dataflux.ops.target.DecodeTargetOp` + that apply it, and round-trips to disk in marainer's ``class_names.json`` format. + + Args: + mapping: Explicit name→id lookup, e.g. ``{"cat": 0, "dog": 1}``. ``None`` (default) builds an + empty map — valid to construct (zero-arg convention), but the properties raise until it + is populated (by passing a mapping, or via :meth:`fit` / :meth:`from_label_names`). + """ + + def __init__(self, mapping: Optional[Dict[str, int]] = None) -> None: + # Lazy / zero-arg: store config only. An empty map is a valid object; the non-empty + # requirement is enforced lazily in the properties, never here. + self.mapping: Dict[str, int] = {str(k): int(v) for k, v in mapping.items()} if mapping else {} + + def _require(self) -> Dict[str, int]: + if not self.mapping: + raise ValueError( + "LabelMap is empty — pass a `mapping`, or build one via LabelMap.fit(targets) / " + "LabelMap.from_label_names(names) / LabelMap.load(path) before use." + ) + return self.mapping + + @property + def num_classes(self) -> int: + """Class count = ``max(id) + 1`` (covers the largest id even if some don't appear).""" + return max(self._require().values()) + 1 + + @property + def label_names(self) -> List[str]: + """``id → name`` list (index == class id). Ids without a name fall back to ``str(id)``.""" + inverse = self.inverse + return [inverse.get(i, str(i)) for i in range(self.num_classes)] + + @property + def inverse(self) -> Dict[int, str]: + """``id → name`` lookup (the inverse of :attr:`mapping`).""" + return {v: k for k, v in self._require().items()} + + def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTargetOp: + """Return an :class:`~dataflux.ops.target.EncodeTargetOp` that maps name → id via this map.""" + return EncodeTargetOp(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) + + def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTargetOp: + """Return a :class:`~dataflux.ops.target.DecodeTargetOp` that maps id → name via this map.""" + return DecodeTargetOp(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) + + @classmethod + def fit(cls, targets: Iterable[Any]) -> "LabelMap": + """Fit a deterministic name→id map from a stream of raw targets via sklearn ``LabelEncoder``. + + Ordering is scikit-learn's sorted-unique ordering, so the same set of labels always yields + the same mapping — train and (a refit on the same labels at) eval agree. In practice eval + should :meth:`load` the pinned training map rather than refit on a subset. + + Args: + targets: Iterable of raw labels (strings, or anything ``str``-coercible). Must be non-empty. + """ + from sklearn.preprocessing import LabelEncoder + + labels = [str(t) for t in targets] + if not labels: + raise ValueError("LabelMap.fit: no targets to fit on (empty stream).") + encoder = LabelEncoder() + encoder.fit(labels) + return cls(mapping={str(name): int(idx) for idx, name in enumerate(encoder.classes_)}) + + @classmethod + def from_label_names(cls, names: Sequence[str]) -> "LabelMap": + """Build a map from an ordered ``id → name`` list (the inverse of :attr:`label_names`). + + Args: + names: Ordered class names; the list index becomes the class id. Must be non-empty. + """ + if not names: + raise ValueError("LabelMap.from_label_names: `names` is empty.") + return cls(mapping={str(name): int(i) for i, name in enumerate(names)}) + + def save(self, path: Union[str, Path]) -> None: + """Persist as ``{"class_names": [...], "num_classes": N}`` — marainer's ``class_names.json`` format. + + Args: + path: Destination file. Parent directories are created as needed. + """ + out = Path(path).expanduser() + out.parent.mkdir(parents=True, exist_ok=True) + payload = {"class_names": self.label_names, "num_classes": self.num_classes} + out.write_text(json.dumps(payload, indent=2, sort_keys=True)) + + @classmethod + def load(cls, path: Union[str, Path]) -> "LabelMap": + """Restore from a ``class_names.json``-shaped file written by :meth:`save` or marainer. + + Args: + path: Source file shaped ``{"class_names": [...]}`` (the ``num_classes`` key is optional; + the ordering of ``class_names`` is authoritative). + """ + data = json.loads(Path(path).expanduser().read_text()) + names = data.get("class_names") + if not names: + raise ValueError(f"LabelMap.load: {path} has no non-empty 'class_names' list.") + return cls.from_label_names([str(n) for n in names]) + + +__all__ = ["LabelMap"] diff --git a/pyproject.toml b/pyproject.toml index 2ca4ec9..bd7caa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = [ "albumentations", "orjson", "fsspec", - "cloudpathlib" + "cloudpathlib", + "scikit-learn" # LabelMap.fit() uses sklearn.preprocessing.LabelEncoder (lazy-imported) ] requires-python = ">=3.12" diff --git a/tests/test_labels.py b/tests/test_labels.py new file mode 100644 index 0000000..6e273a2 --- /dev/null +++ b/tests/test_labels.py @@ -0,0 +1,143 @@ +"""Tests for :class:`dataflux.labels.LabelMap` — the fittable name↔id label map.""" + +import json + +import pytest + +from dataflux.labels import LabelMap +from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp +from dataflux.sample import Sample + +# --------------------------------------------------------------------------- +# Construction & lazy validation +# --------------------------------------------------------------------------- + + +def test_zero_arg_construction_is_empty() -> None: + # Zero-arg / lazy-init convention: building succeeds, the map is just empty. + lm = LabelMap() + assert lm.mapping == {} + + +def test_empty_map_properties_raise() -> None: + lm = LabelMap() + with pytest.raises(ValueError): + _ = lm.num_classes + with pytest.raises(ValueError): + _ = lm.label_names + with pytest.raises(ValueError): + _ = lm.inverse + + +def test_explicit_mapping_coerces_types() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + assert lm.mapping == {"cat": 0, "dog": 1} + assert lm.num_classes == 2 + assert lm.label_names == ["cat", "dog"] + assert lm.inverse == {0: "cat", 1: "dog"} + + +# --------------------------------------------------------------------------- +# fit() — deterministic sorted ordering via sklearn LabelEncoder +# --------------------------------------------------------------------------- + + +def test_fit_uses_sorted_ordering() -> None: + lm = LabelMap.fit(["dog", "cat", "dog", "bird", "cat"]) + # sklearn LabelEncoder sorts classes lexicographically. + assert lm.label_names == ["bird", "cat", "dog"] + assert lm.mapping == {"bird": 0, "cat": 1, "dog": 2} + assert lm.num_classes == 3 + + +def test_fit_coerces_non_strings() -> None: + lm = LabelMap.fit([1, 2, 1, 3]) + assert lm.label_names == ["1", "2", "3"] + + +def test_fit_empty_raises() -> None: + with pytest.raises(ValueError): + LabelMap.fit([]) + + +# --------------------------------------------------------------------------- +# from_label_names — inverse of label_names +# --------------------------------------------------------------------------- + + +def test_from_label_names_round_trip() -> None: + names = ["bird", "cat", "dog"] + lm = LabelMap.from_label_names(names) + assert lm.label_names == names + assert lm.mapping == {"bird": 0, "cat": 1, "dog": 2} + + +def test_from_label_names_empty_raises() -> None: + with pytest.raises(ValueError): + LabelMap.from_label_names([]) + + +# --------------------------------------------------------------------------- +# encode_op / decode_op produce working dataflux ops +# --------------------------------------------------------------------------- + + +def test_encode_op_encodes_target() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + op = lm.encode_op() + assert isinstance(op, EncodeTargetOp) + out = op(Sample(input=None, target="dog", metadata={})) + assert out.target == 1 + + +def test_decode_op_inverts_encoding() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + op = lm.decode_op() + assert isinstance(op, DecodeTargetOp) + out = op(Sample(input=None, target=0, metadata={})) + assert out.target == "cat" + + +def test_encode_op_ignore_unknown() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + op = lm.encode_op(ignore_unknown=True, default=-1) + out = op(Sample(input=None, target="fish", metadata={})) + assert out.target == -1 + + +# --------------------------------------------------------------------------- +# Persistence — same format as marainer's class_names.json +# --------------------------------------------------------------------------- + + +def test_save_load_round_trip(tmp_path: object) -> None: + lm = LabelMap.fit(["dog", "cat", "bird"]) + path = tmp_path / "class_names.json" # type: ignore[operator] + lm.save(path) + restored = LabelMap.load(path) + assert restored.mapping == lm.mapping + assert restored.label_names == lm.label_names + + +def test_save_writes_class_names_payload(tmp_path: object) -> None: + lm = LabelMap.from_label_names(["a", "b", "c"]) + path = tmp_path / "class_names.json" # type: ignore[operator] + lm.save(path) + data = json.loads(path.read_text()) # type: ignore[attr-defined] + assert data == {"class_names": ["a", "b", "c"], "num_classes": 3} + + +def test_load_reads_marainer_written_file(tmp_path: object) -> None: + # A class_names.json written by marainer's _write_class_names is byte-compatible. + path = tmp_path / "class_names.json" # type: ignore[operator] + path.write_text(json.dumps({"class_names": ["x", "y"], "num_classes": 2})) # type: ignore[attr-defined] + lm = LabelMap.load(path) + assert lm.mapping == {"x": 0, "y": 1} + assert lm.num_classes == 2 + + +def test_load_missing_class_names_raises(tmp_path: object) -> None: + path = tmp_path / "bad.json" # type: ignore[operator] + path.write_text(json.dumps({"num_classes": 2})) # type: ignore[attr-defined] + with pytest.raises(ValueError): + LabelMap.load(path) From 018f6b5d2c1f85829aa7af98ba02ec0ddca41e2c Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 2 Jun 2026 17:19:09 +0200 Subject: [PATCH 009/102] feat: widen Sample.metadata to dict|list[dict] with .meta/.batch_meta accessors Sample.metadata is now Union[Dict, List[Dict]] so a Sample distinguishes a single item (one dict) from a batch (a list of per-item dicts, as the collate functions produce). Adds Sample.is_batched plus the narrowing accessors .meta (the dict; raises on a batch) and .batch_meta (the list; raises on a single); describe()/with_type() guard the batch form. Rewrites all per-sample .metadata[...] dict-access through .meta. --- AGENTS.md | 1 + dataflux/core.py | 8 ++-- dataflux/ops/copy.py | 6 +-- dataflux/ops/image.py | 6 +-- dataflux/ops/numpy.py | 14 +++---- dataflux/ops/stash.py | 4 +- dataflux/ops/target.py | 9 ++--- dataflux/paired.py | 2 +- dataflux/projection.py | 2 +- dataflux/sample.py | 73 +++++++++++++++++++++++++++------ dataflux/storage/directory.py | 4 +- dataflux/storage/hdf5.py | 2 +- dataflux/storage/zarr.py | 4 +- examples/paired_annotations.py | 31 +++++++------- examples/storage_roundtrip.py | 6 +-- tests/test_enable.py | 10 ++--- tests/test_image_ops.py | 8 ++-- tests/test_ops.py | 52 ++++++++++++------------ tests/test_paired.py | 74 +++++++++++++++++----------------- tests/test_projection.py | 2 +- tests/test_sample.py | 51 ++++++++++++++++++++++- tests/test_sources.py | 2 +- tests/test_storage.py | 16 ++++---- tests/test_target_ops.py | 6 +-- tests/test_typespec.py | 12 +++--- 25 files changed, 251 insertions(+), 154 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 31a47f0..79dbf6a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,7 @@ # DataFlux Mandates - **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. +- **`Sample.metadata` Is `dict` (single) OR `list[dict]` (batch) — Narrow via `.meta` / `.batch_meta`:** The `metadata` field is `Metadata = Union[Dict[str, Any], List[Dict[str, Any]]]`. A **single** item carries one `dict` (the normal pipeline form every source/op produces and consumes); a **batch** carries a `list` of per-item dicts (one per stacked item), produced by the collate functions (`marainer.collate.collate_fn_with_metadata`, `sonair.classification.classification_collate_fn`) when N samples are stacked into one Sample for the model/loss/predictions-sinks. `Sample.is_batched` (= `isinstance(metadata, list)`) is the single source of truth for telling them apart. Per-sample code MUST read/mutate metadata through the narrowing accessor **`sample.meta`** (returns the dict, raises `TypeError` on a batch) — `sample.meta[key]` / `sample.meta[key] = v`; batch consumers use **`sample.batch_meta`** (returns the list, raises on a single). NEVER index the raw `sample.metadata` Union directly (mypy rejects `Union[...][str]`). NOTE the batch convention is per-collate: marainer/sonair stack into the **list** form (`is_batched` True); deltaid's `segmentation_collate_fn` instead nests under a **dict** `metadata={"per_sample": [...]}` (so `is_batched` is False there — use `.meta["per_sample"]`). `describe()`/`with_type()` operate on single samples only (a batch infers / raises). Pins: `tests/test_sample.py` (batch vs single, `.meta`/`.batch_meta` guards). - **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY dataflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. diff --git a/dataflux/core.py b/dataflux/core.py index 87f7de6..dd13460 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -50,13 +50,13 @@ def _refresh_type(sample: Sample, op: Any) -> Sample: dataset), an op that declares ``PRODUCES`` refreshes it; an op that declares none drops it so :meth:`Sample.describe` falls back to inference rather than reporting a stale type. """ - if not any(key in sample.metadata for key in TYPE_KEYS): + if not any(key in sample.meta for key in TYPE_KEYS): return sample produces = getattr(op, "PRODUCES", None) if produces is not None: features_key, spec_key = _serialized_type_keys(produces) - return sample._replace(metadata={**sample.metadata, FEATURES_KEY: features_key, SPEC_KEY: spec_key}) - return sample._replace(metadata={k: v for k, v in sample.metadata.items() if k not in TYPE_KEYS}) + return sample._replace(metadata={**sample.meta, FEATURES_KEY: features_key, SPEC_KEY: spec_key}) + return sample._replace(metadata={k: v for k, v in sample.meta.items() if k not in TYPE_KEYS}) def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: @@ -514,5 +514,5 @@ def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: yield Sample( input=sample.input if "input" in want else None, target=sample.target if "target" in want else None, - metadata=sample.metadata if "metadata" in want else {}, + metadata=sample.meta if "metadata" in want else {}, ) diff --git a/dataflux/ops/copy.py b/dataflux/ops/copy.py index c569b33..d5944ca 100644 --- a/dataflux/ops/copy.py +++ b/dataflux/ops/copy.py @@ -22,7 +22,7 @@ def __call__(self, sample: Sample) -> Sample: return Sample( input=copy.deepcopy(sample.input), target=copy.deepcopy(sample.target), - metadata=copy.deepcopy(sample.metadata), + metadata=copy.deepcopy(sample.meta), ) @@ -44,12 +44,12 @@ def __call__(self, sample: Sample) -> Sample: @configurable(category="op", group="structure") class CopyMetadataOp: - """Deepcopy ``sample.metadata``. + """Deepcopy ``sample.meta``. The replacement dict is a fresh object, so subsequent in-place writes on the new metadata won't be seen by other holders of the old dict. """ def __call__(self, sample: Sample) -> Sample: - new_meta: Any = copy.deepcopy(sample.metadata) + new_meta: Any = copy.deepcopy(sample.meta) return sample._replace(metadata=new_meta) diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py index deee7f4..e129bad 100644 --- a/dataflux/ops/image.py +++ b/dataflux/ops/image.py @@ -209,7 +209,7 @@ class ConvertToImageOp: array's row 0 is the *bottom* of the desired image (a spectrogram stores row 0 = f_min but display wants f_max at the top, so overlay pixel math lines up). The final ``image_width_px`` / ``image_height_px`` are published - to ``sample.metadata`` so downstream consumers (e.g. a detector + to ``sample.meta`` so downstream consumers (e.g. a detector back-projecting pixel boxes to signal regions) can read the raster size. Args: @@ -249,8 +249,8 @@ def __call__(self, sample: Sample) -> Sample: else: img = Image.fromarray(_bound_longest_side(rgb, self.max_size)) - sample.metadata["image_width_px"] = img.width - sample.metadata["image_height_px"] = img.height + sample.meta["image_width_px"] = img.width + sample.meta["image_height_px"] = img.height return sample._replace(input=img) diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index ccd4649..e84f95f 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -21,7 +21,7 @@ def resolve_expression(value: str, sample: Sample) -> str: - """Substitute ``{key}`` from ``sample.metadata`` and ``$NAME`` from ``os.environ``. + """Substitute ``{key}`` from ``sample.meta`` and ``$NAME`` from ``os.environ``. Returns the substituted string verbatim — the caller is responsible for any further casting (e.g. ``float(...)`` for a numeric expression). @@ -45,12 +45,12 @@ def _repl(match: "re.Match[str]") -> str: meta_key = match.group(1) env_name = match.group(2) if meta_key is not None: - if meta_key not in sample.metadata: + if meta_key not in sample.meta: raise KeyError( f"resolve_expression: metadata key {meta_key!r} missing in {value!r}; " - f"available keys: {sorted(sample.metadata)}" + f"available keys: {sorted(sample.meta)}" ) - return str(sample.metadata[meta_key]) + return str(sample.meta[meta_key]) assert env_name is not None if env_name not in os.environ: raise KeyError(f"resolve_expression: environment variable {env_name!r} missing in {value!r}") @@ -292,7 +292,7 @@ class ThresholdOp: deferred-valid so the op stays constructible, per the lazy-init convention). Each bound is either a numeric literal or a string expression resolved via - :func:`resolve_expression` against ``sample.metadata`` and ``os.environ``: + :func:`resolve_expression` against ``sample.meta`` and ``os.environ``: * ``5.5`` or ``"5.5"`` — fixed bound * ``"{reference_snr_level}"`` — looks up ``metadata["reference_snr_level"]`` @@ -349,11 +349,11 @@ def __call__(self, sample: Sample) -> Sample: mask: Optional[np.ndarray] = None if self.low_level is not None: low = self._resolve(self.low_level, sample) - sample.metadata["threshold_low"] = low + sample.meta["threshold_low"] = low mask = _LOW_COMPARISONS[self.low_op](arr, low) if self.high_level is not None: high = self._resolve(self.high_level, sample) - sample.metadata["threshold_high"] = high + sample.meta["threshold_high"] = high below = _HIGH_COMPARISONS[self.high_op](arr, high) mask = below if mask is None else (mask & below) if mask is None: diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py index a2fbc48..fed7930 100644 --- a/dataflux/ops/stash.py +++ b/dataflux/ops/stash.py @@ -37,7 +37,7 @@ def __init__(self, key: str = "", copy: bool = False) -> None: self.copy = copy def __call__(self, sample: Sample) -> Sample: - sample.metadata[self.key] = _copy.deepcopy(sample.input) if self.copy else sample.input + sample.meta[self.key] = _copy.deepcopy(sample.input) if self.copy else sample.input return sample @@ -60,7 +60,7 @@ def __init__(self, key: str = "", copy: bool = True) -> None: self.copy = copy def __call__(self, sample: Sample) -> Sample: - value = sample.metadata[self.key] + value = sample.meta[self.key] if self.copy: value = _copy.deepcopy(value) return sample._replace(input=value) diff --git a/dataflux/ops/target.py b/dataflux/ops/target.py index 0612154..170d1c9 100644 --- a/dataflux/ops/target.py +++ b/dataflux/ops/target.py @@ -66,14 +66,13 @@ def __init__(self, key: str = "", target_key: Optional[str] = None) -> None: self.target_key = str(target_key) if target_key is not None else None def __call__(self, sample: Sample) -> Sample: - if self.key not in sample.metadata: + if self.key not in sample.meta: raise KeyError( - f"MetadataToTargetOp: sample.metadata has no key {self.key!r}. " - f"Available keys: {sorted(sample.metadata)}" + f"MetadataToTargetOp: sample.meta has no key {self.key!r}. " f"Available keys: {sorted(sample.meta)}" ) - value = sample.metadata[self.key] + value = sample.meta[self.key] if self.target_key is not None: - sample.metadata[self.target_key] = value + sample.meta[self.target_key] = value return sample._replace(target=value) diff --git a/dataflux/paired.py b/dataflux/paired.py index a6e9947..940ba57 100644 --- a/dataflux/paired.py +++ b/dataflux/paired.py @@ -168,7 +168,7 @@ def _resolved_data_resolver(self) -> Callable[[str, Any], Any]: return self._data_resolver_cache def _attach(self, sample: Sample, record: Optional[Dict[str, Any]], key: str) -> Sample: - metadata = dict(sample.metadata) + metadata = dict(sample.meta) metadata["annotation_key"] = key metadata["annotated"] = record is not None diff --git a/dataflux/projection.py b/dataflux/projection.py index 4faaacf..b881729 100644 --- a/dataflux/projection.py +++ b/dataflux/projection.py @@ -77,7 +77,7 @@ def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample yield Sample( input=s.input if INPUT in want else None, target=s.target if TARGET in want else None, - metadata=s.metadata if METADATA in want else {}, + metadata=s.meta if METADATA in want else {}, ) diff --git a/dataflux/sample.py b/dataflux/sample.py index 4998844..9307f65 100644 --- a/dataflux/sample.py +++ b/dataflux/sample.py @@ -1,5 +1,5 @@ import json -from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Tuple, Union, cast if TYPE_CHECKING: # pragma: no cover - typing only from dataflux.typespec import SampleType @@ -12,6 +12,13 @@ SPEC_KEY = "__spec__" TYPE_KEYS = (FEATURES_KEY, SPEC_KEY) +# A Sample's metadata is EITHER a single ``dict`` (one item — the normal pipeline form every op +# produces/consumes) OR a ``list`` of per-item dicts (a BATCH — produced by the collate functions when +# stacking N samples into one). The two forms are how a Sample distinguishes a single item from a batch: +# per-sample ops always see (and require) the dict form; the list form appears only AFTER collate, in the +# batched Sample fed to the model / loss / predictions sinks, and never flows back through a per-sample op. +Metadata = Union[Dict[str, Any], List[Dict[str, Any]]] + # Standardized Sample: (input, target, metadata) # This allows DataFlux to handle complex pipelines while remaining @@ -19,32 +26,74 @@ class Sample(NamedTuple): input: Any target: Any = None - metadata: Dict[str, Any] = {} + metadata: Metadata = {} - def to_tuple(self) -> Tuple[Any, Any, Dict[str, Any]]: + def to_tuple(self) -> Tuple[Any, Any, Metadata]: return (self.input, self.target, self.metadata) + @property + def is_batched(self) -> bool: + """True if this Sample holds a BATCH — ``metadata`` is a ``list`` of per-item dicts (one per + stacked item, as the collate functions produce); False for a single item (``metadata`` is a + ``dict``). The single source of truth for telling batch from single.""" + return isinstance(self.metadata, list) + + @property + def meta(self) -> Dict[str, Any]: + """The single-item metadata **dict** — the narrowing accessor per-sample ops/sources use to + read or mutate ``metadata`` (``sample.meta[key]`` / ``sample.meta[key] = v``). Returns the same + underlying dict (mutation propagates). Raises ``TypeError`` on a batched Sample, where there is + no single dict — use :attr:`batch_meta` instead.""" + if isinstance(self.metadata, list): + raise TypeError( + "Sample.meta is the single-item metadata dict, but this Sample is batched (metadata is a " + "list of per-item dicts) — use Sample.batch_meta." + ) + return self.metadata + + @property + def batch_meta(self) -> List[Dict[str, Any]]: + """The per-item metadata **list** of a batched Sample (one dict per stacked item) — the + narrowing accessor batch consumers (collate-fed losses / predictions sinks) use. Raises + ``TypeError`` on a single Sample, whose metadata is one dict — use :attr:`meta` instead.""" + if not isinstance(self.metadata, list): + raise TypeError( + "Sample.batch_meta is the per-item metadata list of a batch, but this Sample is single " + "(metadata is one dict) — use Sample.meta." + ) + return self.metadata + def describe(self) -> "SampleType": """Return this sample's :class:`~dataflux.typespec.SampleType`. Prefers the stored type (the reserved metadata keys, set explicitly via :meth:`with_type` or - carried by a serialized dataset); otherwise infers it from the live ``input`` / ``target``. + carried by a serialized dataset); otherwise infers it from the live ``input`` / ``target``. A + batched sample carries no per-reserved-key type, so it always infers from the live data. """ from dataflux.typespec import SampleType, infer_sample_type - raw_features = self.metadata.get(FEATURES_KEY) - raw_extras = self.metadata.get(SPEC_KEY) - if raw_features is not None or raw_extras is not None: - features = json.loads(raw_features) if isinstance(raw_features, str) else (raw_features or {}) - extras = json.loads(raw_extras) if isinstance(raw_extras, str) else raw_extras - return SampleType.from_hf_features(features, extras) + meta = self.metadata + if isinstance(meta, dict): + raw_features = meta.get(FEATURES_KEY) + raw_extras = meta.get(SPEC_KEY) + if raw_features is not None or raw_extras is not None: + features = json.loads(raw_features) if isinstance(raw_features, str) else (raw_features or {}) + extras = json.loads(raw_extras) if isinstance(raw_extras, str) else raw_extras + return SampleType.from_hf_features(features, extras) return infer_sample_type(self) def with_type(self, sample_type: "SampleType") -> "Sample": """Return a copy carrying ``sample_type`` in the reserved metadata keys (copy-on-write, so the - original sample's metadata is not mutated).""" + original sample's metadata is not mutated). Only defined for a single (non-batched) sample — + a batch carries no single stored type.""" + if self.is_batched: + raise TypeError( + "Sample.with_type is only defined for a single (non-batched) sample; this Sample carries " + "list (batched) metadata." + ) + base = cast(Dict[str, Any], self.metadata) features, extras = sample_type.to_hf_features() - metadata = {**self.metadata, FEATURES_KEY: json.dumps(features.to_dict()), SPEC_KEY: json.dumps(extras)} + metadata = {**base, FEATURES_KEY: json.dumps(features.to_dict()), SPEC_KEY: json.dumps(extras)} return self._replace(metadata=metadata) @classmethod diff --git a/dataflux/storage/directory.py b/dataflux/storage/directory.py index eb0c4a0..d760d63 100644 --- a/dataflux/storage/directory.py +++ b/dataflux/storage/directory.py @@ -36,9 +36,9 @@ def write(self, sample: Sample) -> None: sample_dir.mkdir(parents=True, exist_ok=True) # 1. Save Metadata (YAML via Confluid) - if sample.metadata: + if sample.meta: meta_path = sample_dir / "metadata.yaml" - meta_path.write_text(confluid.dump(sample.metadata)) + meta_path.write_text(confluid.dump(sample.meta)) # 2. Save Input and Target (Numpy) if self.use_npz: diff --git a/dataflux/storage/hdf5.py b/dataflux/storage/hdf5.py index 2278f62..ffe3c96 100644 --- a/dataflux/storage/hdf5.py +++ b/dataflux/storage/hdf5.py @@ -121,7 +121,7 @@ def write(self, sample: Sample) -> None: # and the str() fallback would silently truncate the array — so it is written as its own # dataset under a per-sample group ``{prefix}_meta/`` (the "/" makes h5py auto-create the # group; arbitrary metadata keys are safe as dataset names). HDF5Source merges both back. - for k, v in sample.metadata.items(): + for k, v in sample.meta.items(): if isinstance(v, (np.ndarray, torch.Tensor)): arr = to_numpy(v) m_kwargs = {} diff --git a/dataflux/storage/zarr.py b/dataflux/storage/zarr.py index f8af8ff..b8bab30 100644 --- a/dataflux/storage/zarr.py +++ b/dataflux/storage/zarr.py @@ -49,8 +49,8 @@ def write(self, sample: Sample) -> None: grp.create_array("target", data=to_numpy(sample.target), overwrite=True) # 2. Save metadata as Zarr attributes (.zattrs) - if sample.metadata: - grp.attrs.update(sample.metadata) + if sample.meta: + grp.attrs.update(sample.meta) self._counter += 1 diff --git a/examples/paired_annotations.py b/examples/paired_annotations.py index 0d051a4..f7500d5 100644 --- a/examples/paired_annotations.py +++ b/examples/paired_annotations.py @@ -62,11 +62,11 @@ def keys(self) -> Any: # Module-level callables so they survive Confluid YAML round-trip via # dataflux.discovery.resolve_callable("examples.paired_annotations:"). def window_key(sample: Sample) -> str: - return f"{sample.metadata['pack_id']}:win{sample.metadata['window_start_sample']:08d}" + return f"{sample.meta['pack_id']}:win{sample.meta['window_start_sample']:08d}" def pack_key(sample: Sample) -> str: - return str(sample.metadata["pack_id"]) + return str(sample.meta["pack_id"]) def resolve_by_window_key(key: str, data: WindowedSource) -> Sample: @@ -78,9 +78,9 @@ def resolve_by_window_key(key: str, data: WindowedSource) -> Sample: def slice_intervals(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: """Trim per-pack time intervals down to each window's range.""" - samplerate = sample.metadata["samplerate"] - win_start = sample.metadata["window_start_sample"] / samplerate - win_end = sample.metadata["window_end_sample"] / samplerate + samplerate = sample.meta["samplerate"] + win_start = sample.meta["window_start_sample"] / samplerate + win_end = sample.meta["window_end_sample"] / samplerate trimmed = [] for iv in record.get("intervals", []): s, e = max(iv["start_s"], win_start), min(iv["end_s"], win_end) @@ -102,9 +102,9 @@ def scenario_a_binary_first() -> None: paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key) for s in paired: - flag = "ANNOTATED" if s.metadata["annotated"] else " -" - label = s.metadata.get("label", "") - print(f" [{flag}] window_start={s.metadata['window_start_sample']:>4} label={label!r}") + flag = "ANNOTATED" if s.meta["annotated"] else " -" + label = s.meta.get("label", "") + print(f" [{flag}] window_start={s.meta['window_start_sample']:>4} label={label!r}") def scenario_b_annotation_first() -> None: @@ -121,7 +121,7 @@ def scenario_b_annotation_first() -> None: paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key, policy="inner") for s in paired: - print(f" key={s.metadata['annotation_key']:<30} label={s.metadata['label']!r}") + print(f" key={s.meta['annotation_key']:<30} label={s.meta['label']!r}") print(f" -> {len(list(paired))} samples (out of {len(data)} in data)") @@ -135,8 +135,7 @@ def scenario_c1_broadcast() -> None: for s in paired: print( - f" win={s.metadata['window_start_sample']:>4} " - f"drone={s.metadata['drone']!r} operator={s.metadata['operator']!r}" + f" win={s.meta['window_start_sample']:>4} " f"drone={s.meta['drone']!r} operator={s.meta['operator']!r}" ) @@ -164,11 +163,11 @@ def scenario_c2_slicing() -> None: ) for s in paired: - win = s.metadata["window_start_sample"] - if s.metadata["annotated"]: - iv = s.metadata["intervals"][0] + win = s.meta["window_start_sample"] + if s.meta["annotated"]: + iv = s.meta["intervals"][0] print( - f" win_start={win:>4} drone={s.metadata['drone']!r} " + f" win_start={win:>4} drone={s.meta['drone']!r} " f"active=[{iv['start_s']*1e6:.1f}us, {iv['end_s']*1e6:.1f}us]" ) else: @@ -196,7 +195,7 @@ def scenario_d_right_driven() -> None: ) for s in paired: - print(f" key={s.metadata['annotation_key']:<30} label={s.metadata['label']!r}") + print(f" key={s.meta['annotation_key']:<30} label={s.meta['label']!r}") def scenario_e_confluid_roundtrip() -> None: diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py index 64cc8fc..3be2b92 100644 --- a/examples/storage_roundtrip.py +++ b/examples/storage_roundtrip.py @@ -34,8 +34,8 @@ def main() -> None: HDF5Sink(h5, overwrite=True) ) loaded = next(iter(HDF5Source(h5))) - print(f" mask round-trips exact : {np.array_equal(loaded.metadata['mask'], mask)}") - print(f" scalar metadata kept : snr={loaded.metadata['snr']}") + print(f" mask round-trips exact : {np.array_equal(loaded.meta['mask'], mask)}") + print(f" scalar metadata kept : snr={loaded.meta['snr']}") # 2. Zarr group source — full input/target/metadata round-trip. print("\n--- Zarr group: ZarrGroupSink -> ZarrGroupSource ---") @@ -47,7 +47,7 @@ def main() -> None: Flux(samples).to_sink(ZarrGroupSink(zg, overwrite=True)) for s in ZarrGroupSource(zg): tgt = None if s.target is None else s.target.tolist() - print(f" input={s.input.tolist()} target={tgt} id={s.metadata['id']!r}") + print(f" input={s.input.tolist()} target={tgt} id={s.meta['id']!r}") # 3. Zarr batch source — stacked uniform array, input only. print("\n--- Zarr batch: ZarrBatchSink -> ZarrBatchSource ---") diff --git a/tests/test_enable.py b/tests/test_enable.py index 960911f..b80a970 100644 --- a/tests/test_enable.py +++ b/tests/test_enable.py @@ -22,7 +22,7 @@ def __init__(self, counter: Dict[str, int]) -> None: def __call__(self, sample: Sample) -> Sample: self.counter["calls"] += 1 - new_meta = dict(sample.metadata) + new_meta = dict(sample.meta) new_meta["counted"] = True return sample._replace(metadata=new_meta) @@ -50,7 +50,7 @@ def test_enable_disabled_passes_sample_through_untouched() -> None: wrapped = _wrap(_CountingOp(counter), visualize=False) out = wrapped(_sample()) assert counter["calls"] == 0 - assert "counted" not in out.metadata + assert "counted" not in out.meta def test_enable_enabled_invokes_inner_op() -> None: @@ -58,7 +58,7 @@ def test_enable_enabled_invokes_inner_op() -> None: wrapped = _wrap(_CountingOp(counter), visualize=True) out = wrapped(_sample()) assert counter["calls"] == 1 - assert out.metadata["counted"] is True + assert out.meta["counted"] is True assert wrapped.flag_name == "visualize" @@ -92,7 +92,7 @@ def __init__(self, tag: str, counter: Dict[str, int]) -> None: def __call__(self, sample: Sample) -> Sample: self.counter["calls"] += 1 - new_meta = dict(sample.metadata) + new_meta = dict(sample.meta) tags = list(new_meta.get("tags", [])) tags.append(self.tag) new_meta["tags"] = tags @@ -105,7 +105,7 @@ def __call__(self, sample: Sample) -> Sample: out = wrapped(_sample()) assert counter_a["calls"] == 1 assert counter_b["calls"] == 1 - assert out.metadata["tags"] == ["first", "second"] + assert out.meta["tags"] == ["first", "second"] def test_enable_multi_op_disabled_skips_entire_chain() -> None: diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py index 8588b06..ca46877 100644 --- a/tests/test_image_ops.py +++ b/tests/test_image_ops.py @@ -39,16 +39,16 @@ def test_convert_2d_map_to_exact_size_pil_and_publishes_dims() -> None: out = ConvertToImageOp(colormap="gray", width=128, height=256)(_sample(arr)) assert isinstance(out.input, Image.Image) assert out.input.size == (128, 256) - assert out.metadata["image_width_px"] == 128 - assert out.metadata["image_height_px"] == 256 + assert out.meta["image_width_px"] == 128 + assert out.meta["image_height_px"] == 256 def test_convert_max_size_path_bounds_longest_side() -> None: out = ConvertToImageOp(max_size=256)(_sample(np.zeros((1000, 400), dtype=np.float32))) assert max(out.input.size) == 256 # Dims are published from the actual rendered raster. - assert out.metadata["image_width_px"] == out.input.width - assert out.metadata["image_height_px"] == out.input.height + assert out.meta["image_width_px"] == out.input.width + assert out.meta["image_height_px"] == out.input.height def test_convert_flip_vertical_mirrors_top_to_bottom() -> None: diff --git a/tests/test_ops.py b/tests/test_ops.py index a463c51..4cffc0b 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -69,7 +69,7 @@ def test_preserves_target_and_metadata(self) -> None: arr = np.zeros((28, 28), dtype=np.uint8) result = ToTensorOp()(Sample(input=arr, target=5, metadata={"k": "v"})) assert result.target == 5 - assert result.metadata == {"k": "v"} + assert result.meta == {"k": "v"} # --------------------------------------------------------------------------- @@ -128,7 +128,7 @@ def test_preserves_target_and_metadata(self) -> None: tensor = torch.tensor([128.0]) result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor, target=7, metadata={"key": "val"})) assert result.target == 7 - assert result.metadata == {"key": "val"} + assert result.meta == {"key": "val"} def test_raises_on_non_tensor(self) -> None: with pytest.raises(TypeError, match="RescaleOp expects a torch.Tensor"): @@ -193,7 +193,7 @@ def test_preserves_target_and_metadata(self) -> None: tensor = torch.tensor([5.0]) result = StandardizeOp(mean=0.0, std=1.0)(Sample(input=tensor, target=3, metadata={"a": 1})) assert result.target == 3 - assert result.metadata == {"a": 1} + assert result.meta == {"a": 1} def test_raises_on_non_tensor(self) -> None: with pytest.raises(TypeError, match="StandardizeOp expects a torch.Tensor"): @@ -244,7 +244,7 @@ def test_preserves_target_and_metadata(self) -> None: arr = np.array([5.0], dtype=np.float32) result = np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=arr, target=3, metadata={"a": 1})) assert result.target == 3 - assert result.metadata == {"a": 1} + assert result.meta == {"a": 1} def test_raises_on_non_ndarray(self) -> None: with pytest.raises(TypeError, match="StandardizeOp expects an np.ndarray"): @@ -354,7 +354,7 @@ def test_preserves_target_and_metadata(self) -> None: arr = np.array([128.0], dtype=np.float32) result = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr, target=7, metadata={"key": "val"})) assert result.target == 7 - assert result.metadata == {"key": "val"} + assert result.meta == {"key": "val"} def test_raises_on_non_ndarray(self) -> None: with pytest.raises(TypeError, match="RescaleOp expects an np.ndarray"): @@ -434,17 +434,17 @@ def test_two_branches_share_metadata(self) -> None: sample = Sample(input=np.array([1.0]), target=None, metadata={}) def writer_a(s: Sample) -> Sample: - s.metadata["a"] = 1 + s.meta["a"] = 1 return s def writer_b(s: Sample) -> Sample: - assert s.metadata["a"] == 1 # branch A's write is visible - s.metadata["b"] = 2 + assert s.meta["a"] == 1 # branch A's write is visible + s.meta["b"] = 2 return s out = Tee(branches=[[writer_a], [writer_b]])(sample) assert out is not None - assert out.metadata == {"a": 1, "b": 2} + assert out.meta == {"a": 1, "b": 2} def test_branches_run_sequentially(self) -> None: from typing import Callable @@ -496,15 +496,15 @@ def test_copy_sample_deepcopies_all_fields(self) -> None: out = CopySampleOp()(sample) assert out.input is not sample.input assert out.target is not sample.target - assert out.metadata is not sample.metadata - assert out.metadata["k"] is not sample.metadata["k"] + assert out.meta is not sample.meta + assert out.meta["k"] is not sample.meta["k"] def test_copy_input_only_copies_input(self) -> None: sample = Sample(input=np.array([1.0]), target=[5], metadata={"k": "v"}) out = CopyInputOp()(sample) assert out.input is not sample.input assert out.target is sample.target - assert out.metadata is sample.metadata + assert out.meta is sample.meta def test_copy_target_only_copies_target(self) -> None: sample = Sample(input=[1, 2], target=[10, 20], metadata={}) @@ -516,7 +516,7 @@ def test_copy_metadata_breaks_aliasing(self) -> None: meta = {"k": [1]} sample = Sample(input=None, target=None, metadata=meta) out = CopyMetadataOp()(sample) - out.metadata["k"].append(2) + out.meta["k"].append(2) assert meta["k"] == [1] @@ -531,7 +531,7 @@ def test_swaps_input_and_target(self) -> None: out = SwapInputTargetOp()(sample) assert out.input == 2 assert out.target == 1 - assert out.metadata == {"k": "v"} + assert out.meta == {"k": "v"} # --------------------------------------------------------------------------- @@ -544,15 +544,15 @@ def test_stash_aliases_by_default(self) -> None: arr = np.array([1.0, 2.0]) sample = Sample(input=arr, target=None, metadata={}) out = StashInputOp(key="snap")(sample) - assert out.metadata["snap"] is arr + assert out.meta["snap"] is arr assert out.input is arr def test_stash_with_copy_deepcopies(self) -> None: arr = np.array([1.0, 2.0]) sample = Sample(input=arr, target=None, metadata={}) out = StashInputOp(key="snap", copy=True)(sample) - assert out.metadata["snap"] is not arr - np.testing.assert_array_equal(out.metadata["snap"], arr) + assert out.meta["snap"] is not arr + np.testing.assert_array_equal(out.meta["snap"], arr) def test_unstash_default_copies_to_isolate_branches(self) -> None: arr = np.array([1.0, 2.0]) @@ -622,23 +622,23 @@ def test_numeric_low_level(self) -> None: arr = np.array([0.0, 1.0, 2.0, 3.0]) out = np_ops.ThresholdOp(low_level=1.5)(Sample(input=arr, metadata={})) np.testing.assert_array_equal(out.input, [False, False, True, True]) - assert out.metadata["threshold_low"] == 1.5 - assert "threshold_high" not in out.metadata + assert out.meta["threshold_low"] == 1.5 + assert "threshold_high" not in out.meta def test_numeric_high_level(self) -> None: arr = np.array([0.0, 1.0, 2.0, 3.0]) out = np_ops.ThresholdOp(high_level=1.5)(Sample(input=arr, metadata={})) np.testing.assert_array_equal(out.input, [True, True, False, False]) - assert out.metadata["threshold_high"] == 1.5 - assert "threshold_low" not in out.metadata + assert out.meta["threshold_high"] == 1.5 + assert "threshold_low" not in out.meta def test_band_low_and_high(self) -> None: arr = np.array([0.0, 1.0, 2.0, 3.0, 4.0]) out = np_ops.ThresholdOp(low_level=1.0, high_level=3.0)(Sample(input=arr, metadata={})) # strictly between 1.0 and 3.0 (default open interval: > and <) np.testing.assert_array_equal(out.input, [False, False, True, False, False]) - assert out.metadata["threshold_low"] == 1.0 - assert out.metadata["threshold_high"] == 3.0 + assert out.meta["threshold_low"] == 1.0 + assert out.meta["threshold_high"] == 3.0 def test_low_level_inclusive(self) -> None: arr = np.array([0.0, 1.0, 2.0]) @@ -682,14 +682,14 @@ def test_metadata_lookup(self) -> None: sample = Sample(input=arr, target=None, metadata={"reference_snr_level": -25.0}) out = np_ops.ThresholdOp(low_level="{reference_snr_level}")(sample) np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.metadata["threshold_low"] == -25.0 + assert out.meta["threshold_low"] == -25.0 def test_metadata_lookup_with_negation(self) -> None: arr = np.array([-50.0, -30.0, -10.0]) sample = Sample(input=arr, target=None, metadata={"reference_snr_level": 30.0}) out = np_ops.ThresholdOp(low_level="-{reference_snr_level}")(sample) np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.metadata["threshold_low"] == -30.0 + assert out.meta["threshold_low"] == -30.0 def test_env_lookup(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DATAFLUX_TEST_THRESHOLD", "1.0") @@ -702,7 +702,7 @@ def test_high_level_expression(self) -> None: sample = Sample(input=arr, target=None, metadata={"ceiling": -20.0}) out = np_ops.ThresholdOp(high_level="{ceiling}")(sample) np.testing.assert_array_equal(out.input, [True, True, False]) - assert out.metadata["threshold_high"] == -20.0 + assert out.meta["threshold_high"] == -20.0 def test_raises_when_no_bounds(self) -> None: op = np_ops.ThresholdOp() # lazy: construction succeeds (zero-arg) diff --git a/tests/test_paired.py b/tests/test_paired.py index 9421345..ffdba5b 100644 --- a/tests/test_paired.py +++ b/tests/test_paired.py @@ -51,7 +51,7 @@ def keys(self) -> Any: # Module-level callables so resolve_callable can find them def sample_id_key(sample: Sample) -> str: - return str(sample.metadata["id"]) + return str(sample.meta["id"]) def identity_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: @@ -64,7 +64,7 @@ def none_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, A def odd_only_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: """Return the record only for odd-id samples, None otherwise.""" - idx = int(str(sample.metadata["id"])[1:]) + idx = int(str(sample.meta["id"])[1:]) if idx % 2 == 1: return record return None @@ -90,7 +90,7 @@ def test_left_outer_emits_all_data() -> None: assert len(samples) == 4 assert [s.input for s in samples] == [0, 10, 20, 30] - assert [s.metadata["annotated"] for s in samples] == [True, False, True, False] + assert [s.meta["annotated"] for s in samples] == [True, False, True, False] def test_left_outer_flattens_record_into_metadata() -> None: @@ -100,11 +100,11 @@ def test_left_outer_flattens_record_into_metadata() -> None: samples = list(paired) - assert samples[0].metadata["label"] == "dog" - assert samples[0].metadata["confidence"] == 0.9 - assert samples[0].metadata["annotation_key"] == "s0" - assert "label" not in samples[1].metadata - assert samples[1].metadata["annotation_key"] == "s1" + assert samples[0].meta["label"] == "dog" + assert samples[0].meta["confidence"] == 0.9 + assert samples[0].meta["annotation_key"] == "s0" + assert "label" not in samples[1].meta + assert samples[1].meta["annotation_key"] == "s1" def test_left_outer_preserves_original_metadata() -> None: @@ -113,8 +113,8 @@ def test_left_outer_preserves_original_metadata() -> None: paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) sample = list(paired)[0] - assert sample.metadata["id"] == "s0" # data metadata survived - assert sample.metadata["label"] == "x" + assert sample.meta["id"] == "s0" # data metadata survived + assert sample.meta["label"] == "x" def test_left_outer_prefix() -> None: @@ -123,8 +123,8 @@ def test_left_outer_prefix() -> None: paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, prefix="ann_") sample = list(paired)[0] - assert sample.metadata["ann_label"] == "x" - assert "label" not in sample.metadata + assert sample.meta["ann_label"] == "x" + assert "label" not in sample.meta def test_left_outer_store_full_under() -> None: @@ -138,9 +138,9 @@ def test_left_outer_store_full_under() -> None: ) sample = list(paired)[0] - assert sample.metadata["raw_annotation"] == {"label": "x", "score": 0.5} + assert sample.meta["raw_annotation"] == {"label": "x", "score": 0.5} # Still flattened too - assert sample.metadata["label"] == "x" + assert sample.meta["label"] == "x" def test_left_outer_len_delegates_to_data() -> None: @@ -157,12 +157,12 @@ def test_left_outer_getitem_matched_and_unmatched() -> None: paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) matched = paired[1] - assert matched.metadata["annotated"] is True - assert matched.metadata["label"] == "y" + assert matched.meta["annotated"] is True + assert matched.meta["label"] == "y" unmatched = paired[0] - assert unmatched.metadata["annotated"] is False - assert "label" not in unmatched.metadata + assert unmatched.meta["annotated"] is False + assert "label" not in unmatched.meta # --------------------------------------------------------------------------- @@ -178,8 +178,8 @@ def test_inner_emits_only_matched() -> None: samples = list(paired) assert len(samples) == 2 - assert {s.metadata["annotation_key"] for s in samples} == {"s0", "s3"} - assert all(s.metadata["annotated"] for s in samples) + assert {s.meta["annotation_key"] for s in samples} == {"s0", "s3"} + assert all(s.meta["annotated"] for s in samples) def test_inner_len_is_cached_scan() -> None: @@ -218,8 +218,8 @@ def test_extract_fn_transforms_record() -> None: samples = list(paired) - assert samples[0].metadata["label"] == "a" - assert samples[1].metadata["label"] == "b" + assert samples[0].meta["label"] == "a" + assert samples[1].meta["label"] == "b" def test_extract_fn_returning_none_marks_unannotated() -> None: @@ -235,7 +235,7 @@ def test_extract_fn_returning_none_marks_unannotated() -> None: samples = list(paired) # Every sample emitted under left_outer; only odd ones are annotated - assert [s.metadata["annotated"] for s in samples] == [False, True, False] + assert [s.meta["annotated"] for s in samples] == [False, True, False] def test_extract_fn_with_inner_policy_filters() -> None: @@ -251,7 +251,7 @@ def test_extract_fn_with_inner_policy_filters() -> None: samples = list(paired) - assert {s.metadata["annotation_key"] for s in samples} == {"s1", "s3"} + assert {s.meta["annotation_key"] for s in samples} == {"s1", "s3"} def test_extract_fn_none_suppresses_flattening() -> None: @@ -265,8 +265,8 @@ def test_extract_fn_none_suppresses_flattening() -> None: ) sample = list(paired)[0] - assert sample.metadata["annotated"] is False - assert "label" not in sample.metadata + assert sample.meta["annotated"] is False + assert "label" not in sample.meta # --------------------------------------------------------------------------- @@ -286,8 +286,8 @@ def test_coarser_key_broadcasts_to_all_matching_samples() -> None: samples = list(paired) - assert all(s.metadata["annotated"] for s in samples) - assert all(s.metadata["drone"] == "dji_mavic" for s in samples) + assert all(s.meta["annotated"] for s in samples) + assert all(s.meta["drone"] == "dji_mavic" for s in samples) # --------------------------------------------------------------------------- @@ -309,7 +309,7 @@ def test_right_driven_iterates_annotation_keys() -> None: samples = list(paired) assert len(samples) == 2 - assert [s.metadata["annotation_key"] for s in samples] == ["s0", "s3"] + assert [s.meta["annotation_key"] for s in samples] == ["s0", "s3"] assert [s.input for s in samples] == [0, 30] @@ -341,7 +341,7 @@ def test_right_driven_skips_when_extract_fn_returns_none() -> None: samples = list(paired) - assert {s.metadata["annotation_key"] for s in samples} == {"s1", "s3"} + assert {s.meta["annotation_key"] for s in samples} == {"s1", "s3"} # --------------------------------------------------------------------------- @@ -422,7 +422,7 @@ def test_key_fn_accepts_string_path() -> None: ) sample = list(paired)[0] - assert sample.metadata["label"] == "x" + assert sample.meta["label"] == "x" def test_extract_fn_accepts_string_path() -> None: @@ -436,7 +436,7 @@ def test_extract_fn_accepts_string_path() -> None: ) sample = list(paired)[0] - assert sample.metadata["label"] == "x" + assert sample.meta["label"] == "x" def test_callable_is_stored_as_string() -> None: @@ -467,9 +467,9 @@ def test_chained_paired_sources_compose() -> None: samples = list(full_paired) # Every sample has drone (from pack), s1 also has event - assert all(s.metadata["drone"] == "mavic" for s in samples) - assert samples[1].metadata["event"] == "takeoff" - assert "event" not in samples[0].metadata + assert all(s.meta["drone"] == "mavic" for s in samples) + assert samples[1].meta["event"] == "takeoff" + assert "event" not in samples[0].meta # --------------------------------------------------------------------------- @@ -491,7 +491,7 @@ def test_confluid_roundtrip_preserves_behavior() -> None: yaml_state = confluid.dump(paired) restored = confluid.load(yaml_state) - original_result = [(s.input, s.metadata.get("ann_label"), s.metadata["annotated"]) for s in paired] - restored_result = [(s.input, s.metadata.get("ann_label"), s.metadata["annotated"]) for s in restored] + original_result = [(s.input, s.meta.get("ann_label"), s.meta["annotated"]) for s in paired] + restored_result = [(s.input, s.meta.get("ann_label"), s.meta["annotated"]) for s in restored] assert original_result == restored_result diff --git a/tests/test_projection.py b/tests/test_projection.py index 82807bf..ff498d0 100644 --- a/tests/test_projection.py +++ b/tests/test_projection.py @@ -55,7 +55,7 @@ def test_project_fallback_nulls_unrequested_fields() -> None: out = list(project(_plain_source(), (TARGET,))) assert [s.target for s in out] == [0, 2] assert all(s.input is None for s in out) - assert all(s.metadata == {} for s in out) + assert all(s.meta == {} for s in out) def test_project_fallback_input_only() -> None: diff --git a/tests/test_sample.py b/tests/test_sample.py index d3c6fa6..7763550 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -1,6 +1,7 @@ from typing import Any, cast import numpy as np +import pytest from dataflux.sample import Sample @@ -11,7 +12,7 @@ def test_sample_from_any() -> None: s = Sample.from_any(d) assert np.array_equal(s.input, cast(Any, d["input"])) assert s.target == 1 - assert s.metadata["id"] == "test" + assert s.meta["id"] == "test" # 2. From tuple (input, target) t = (np.array([3, 4]), 0) @@ -36,3 +37,51 @@ def test_sample_to_tuple() -> None: s = Sample(input=1, target=2, metadata={"a": 3}) t = s.to_tuple() assert t == (1, 2, {"a": 3}) + + +# --- Batch vs single metadata (Union schema) ------------------------------- + + +def test_single_sample_metadata_is_a_dict() -> None: + s = Sample(input=1, target=0, metadata={"id": "a"}) + assert s.is_batched is False + assert s.meta == {"id": "a"} + assert s.meta["id"] == "a" + + +def test_batched_sample_metadata_is_a_list_of_dicts() -> None: + # The collate form: one Sample carrying N stacked items + per-item metadata dicts. + batch = Sample(input=[1, 2], target=[0, 1], metadata=[{"id": "a"}, {"id": "b"}]) + assert batch.is_batched is True + assert batch.batch_meta == [{"id": "a"}, {"id": "b"}] + assert [m["id"] for m in batch.batch_meta] == ["a", "b"] + + +def test_meta_accessor_raises_on_a_batch() -> None: + batch = Sample(input=[1], target=[0], metadata=[{"id": "a"}]) + with pytest.raises(TypeError, match="batched"): + _ = batch.meta + + +def test_batch_meta_accessor_raises_on_a_single_sample() -> None: + single = Sample(input=1, target=0, metadata={"id": "a"}) + with pytest.raises(TypeError, match="single"): + _ = single.batch_meta + + +def test_describe_falls_back_to_inference_on_a_batch() -> None: + # A batched sample carries no per-reserved-key stored type -> describe infers (no crash on the list). + batch = Sample(input=np.zeros((2, 4)), target=np.array([0, 1]), metadata=[{}, {}]) + assert batch.describe() is not None + + +def test_with_type_rejects_a_batch() -> None: + from dataflux.typespec import infer_sample_type + + single = Sample(input=np.zeros((4,)), target=0, metadata={}) + typed = single.with_type(infer_sample_type(single)) # single sample: OK + assert typed.describe() is not None + + batch = Sample(input=np.zeros((2, 4)), target=np.array([0, 1]), metadata=[{}, {}]) + with pytest.raises(TypeError, match="batched"): + batch.with_type(infer_sample_type(single)) diff --git a/tests/test_sources.py b/tests/test_sources.py index 1e45845..aa02d69 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -542,7 +542,7 @@ def test_hf_source_iter_metadata_features_star_expands_on_real_dataset() -> None samples = list(src) assert [s.input for s in samples] == [0, 1, 2] - md = samples[0].metadata + md = samples[0].meta assert md["id"] == "r0" and md["src"] == "a" assert "image" not in md and "label" not in md # input/target excluded from metadata assert md["hf_path"] == "fake/ds" and md["hf_split"] == "train" diff --git a/tests/test_storage.py b/tests/test_storage.py index 75bf741..a8bf800 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -63,13 +63,13 @@ def test_hdf5_array_metadata_roundtrip(tmp_path: Path) -> None: assert len(loaded) == 2 # Scalar metadata round-trips via attributes. - assert loaded[0].metadata["id"] == "a" - assert loaded[0].metadata["samplerate"] == 100.0 - assert loaded[1].metadata["id"] == "b" + assert loaded[0].meta["id"] == "a" + assert loaded[0].meta["samplerate"] == 100.0 + assert loaded[1].meta["id"] == "b" # Array metadata round-trips exactly (no truncation). - assert np.array_equal(loaded[0].metadata["mask"], mask) + assert np.array_equal(loaded[0].meta["mask"], mask) # The sample without array metadata has no spurious mask key. - assert "mask" not in loaded[1].metadata + assert "mask" not in loaded[1].meta def test_hdf5_array_metadata_no_compression(tmp_path: Path) -> None: @@ -82,7 +82,7 @@ def test_hdf5_array_metadata_no_compression(tmp_path: Path) -> None: sink.close() loaded = list(HDF5Source(h5_path)) - assert np.array_equal(loaded[0].metadata["mask"], mask) + assert np.array_equal(loaded[0].meta["mask"], mask) def test_zarr_group_storage(tmp_path: Path) -> None: @@ -212,8 +212,8 @@ def test_zarr_group_source_roundtrip(tmp_path: Path) -> None: assert np.array_equal(loaded[0].target, np.array([1])) assert loaded[1].target is None # Metadata round-trips via group attributes. - assert loaded[0].metadata["id"] == "a" - assert loaded[1].metadata["id"] == "b" + assert loaded[0].meta["id"] == "a" + assert loaded[1].meta["id"] == "b" source.close() diff --git a/tests/test_target_ops.py b/tests/test_target_ops.py index 1b5accc..f1fa71a 100644 --- a/tests/test_target_ops.py +++ b/tests/test_target_ops.py @@ -17,14 +17,14 @@ def test_metadata_to_target_moves_value() -> None: def test_metadata_to_target_leaves_metadata_untouched_without_target_key() -> None: sample = Sample(input=0, metadata={"drone": "DJI MINI3"}) out = MetadataToTargetOp(key="drone")(sample) - assert set(out.metadata) == {"drone"} + assert set(out.meta) == {"drone"} def test_metadata_to_target_copies_to_target_key() -> None: sample = Sample(input=0, metadata={"drone": "DJI MINI3"}) out = MetadataToTargetOp(key="drone", target_key="raw_label")(sample) assert out.target == "DJI MINI3" - assert out.metadata["raw_label"] == "DJI MINI3" + assert out.meta["raw_label"] == "DJI MINI3" def test_metadata_to_target_missing_key_raises() -> None: @@ -95,4 +95,4 @@ def test_metadata_to_target_then_encode() -> None: sample = EncodeTargetOp(mapping=label_to_index)(sample) assert sample.target == 2 # raw label preserved for decode/reporting - assert sample.metadata["raw_label"] == "DJI AVATA2" + assert sample.meta["raw_label"] == "DJI AVATA2" diff --git a/tests/test_typespec.py b/tests/test_typespec.py index 02ffdb3..8fdf736 100644 --- a/tests/test_typespec.py +++ b/tests/test_typespec.py @@ -458,9 +458,9 @@ def test_sample_with_type_and_describe_stored() -> None: declared = SampleType(input=ArrayType.image("CHW", 3, "float32", "torch")) s = Sample(input=np.zeros((3, 8, 8), dtype=np.float32), metadata={"id": 7}) typed_s = s.with_type(declared) - assert FEATURES_KEY in typed_s.metadata and SPEC_KEY in typed_s.metadata - assert typed_s.metadata["id"] == 7 # pre-existing metadata preserved - assert s.metadata == {"id": 7} and FEATURES_KEY not in s.metadata # copy-on-write: original untouched + assert FEATURES_KEY in typed_s.meta and SPEC_KEY in typed_s.meta + assert typed_s.meta["id"] == 7 # pre-existing metadata preserved + assert s.meta == {"id": 7} and FEATURES_KEY not in s.meta # copy-on-write: original untouched rt = typed_s.describe() assert rt.accepts(declared) and declared.accepts(rt) @@ -488,7 +488,7 @@ def _run(sample: Sample, op: Any) -> Sample: def test_pipeline_does_not_stamp_untracked_samples() -> None: out = _run(Sample(input=np.array([1, 2, 3])), _ToFloat64Op()) - assert FEATURES_KEY not in out.metadata and SPEC_KEY not in out.metadata + assert FEATURES_KEY not in out.meta and SPEC_KEY not in out.meta def test_pipeline_refreshes_stored_type_from_produces() -> None: @@ -496,7 +496,7 @@ def test_pipeline_refreshes_stored_type_from_produces() -> None: SampleType(input=ArrayType(ndim=1, dtype="int64", frameworks={"numpy"})) ) out = _run(stamped, _ToFloat64Op()) - assert FEATURES_KEY in out.metadata + assert FEATURES_KEY in out.meta assert out.describe().accepts(_ToFloat64Op.PRODUCES) assert _ToFloat64Op.PRODUCES.accepts(out.describe()) @@ -506,7 +506,7 @@ def test_pipeline_drops_stored_type_when_op_has_no_produces() -> None: SampleType(input=ArrayType(ndim=1, dtype="int64", frameworks={"numpy"})) ) out = _run(stamped, _UntypedOp()) - assert FEATURES_KEY not in out.metadata and SPEC_KEY not in out.metadata + assert FEATURES_KEY not in out.meta and SPEC_KEY not in out.meta # describe() falls back to inference -> still correct, just not "stored" assert isinstance(out.describe().input, ArrayType) From 518792982b75e7987060e591a33a19b2d893f6c2 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 4 Jun 2026 15:23:55 +0200 Subject: [PATCH 010/102] feat: detection target ops + ToTensorOp mode + shared bbox helper - CocoToTorchVisionDetectionOp (HF/COCO objects -> {boxes xyxy, labels}) - MasksToDetectionBoxesOp (segmentation mask -> boxes; per-instance or CC) - factor connected_component_bboxes out of ConnectedComponentsOp (shared) - ToTensorOp(mode=...) coerces PIL via Image.convert before arraying (RGBA fix) --- AGENTS.md | 2 +- dataflux/ops/__init__.py | 10 ++- dataflux/ops/numpy.py | 81 +++++++++++-------- dataflux/ops/target.py | 171 ++++++++++++++++++++++++++++++++++++++- dataflux/ops/torch.py | 11 ++- tests/test_categories.py | 22 ++++- tests/test_ops.py | 17 ++++ tests/test_target_ops.py | 122 +++++++++++++++++++++++++++- 8 files changed, 392 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 79dbf6a..ac20ed1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` (`Tee`/`Parallel`/`Enable`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` (`Tee`/`Parallel`/`Enable`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index db837cd..a224ce7 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -24,7 +24,13 @@ from dataflux.ops.sink import SampleSinkOp from dataflux.ops.stash import StashInputOp, UnstashInputOp from dataflux.ops.swap import SwapInputTargetOp -from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp +from dataflux.ops.target import ( + CocoToTorchVisionDetectionOp, + DecodeTargetOp, + EncodeTargetOp, + MasksToDetectionBoxesOp, + MetadataToTargetOp, +) from dataflux.ops.tee import Tee from dataflux.ops.torch import RescaleOp, StandardizeOp, ToTensorOp @@ -37,6 +43,8 @@ "Enable", "EncodeTargetOp", "MetadataToTargetOp", + "CocoToTorchVisionDetectionOp", + "MasksToDetectionBoxesOp", "Parallel", "RescaleOp", "SampleSinkOp", diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index e84f95f..6f72d4e 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -361,6 +361,52 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=mask) +def connected_component_bboxes( + mask: np.ndarray, min_area_bins: int = 1, connectivity: int = 4 +) -> List[Tuple[int, int, int, int]]: + """Label connected ``True`` regions of a 2-D bool mask → ``(row_min, row_max, col_min, col_max)`` inclusive tuples. + + Components smaller than ``min_area_bins`` are dropped. ``connectivity`` is ``4`` + (orthogonal neighbors) or ``8`` (orthogonal + diagonal). This is the shared scipy + core behind :class:`ConnectedComponentsOp` (signal-domain bin bboxes on ``input``) + AND :class:`dataflux.ops.target.MasksToDetectionBoxesOp` (its ``connected=True`` + mode, which lifts the tuples to xyxy-pixel detection boxes). Requires ``scipy`` + (``pip install data-flux[vision]``). + """ + if min_area_bins < 1: + raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") + if connectivity not in (4, 8): + raise ValueError(f"connectivity must be 4 or 8; got {connectivity!r}") + try: + from scipy.ndimage import find_objects, generate_binary_structure, label + except ImportError as exc: + raise ImportError( + "connected-components labeling requires scipy. " + "Install with `pip install data-flux[vision]` or add scipy to your environment." + ) from exc + + structure = generate_binary_structure(2, 1 if connectivity == 4 else 2) + labels, n_components = label(mask, structure=structure) + bboxes: List[Tuple[int, int, int, int]] = [] + if n_components > 0: + for idx, sl in enumerate(find_objects(labels), start=1): + if sl is None: + continue + row_slice, col_slice = sl + area = int((labels[row_slice, col_slice] == idx).sum()) + if area < min_area_bins: + continue + bboxes.append( + ( + int(row_slice.start), + int(row_slice.stop) - 1, + int(col_slice.start), + int(col_slice.stop) - 1, + ) + ) + return bboxes + + @configurable(category="op", group="numpy") class ConnectedComponentsOp: """Label connected ``True`` regions of a boolean mask into bin-bbox tuples. @@ -392,41 +438,12 @@ def __init__(self, min_area_bins: int = 1, connectivity: int = 4) -> None: self.connectivity = int(connectivity) def __call__(self, sample: Sample) -> Sample: - if self.min_area_bins < 1: - raise ValueError(f"min_area_bins must be >= 1; got {self.min_area_bins!r}") - if self.connectivity not in (4, 8): - raise ValueError(f"connectivity must be 4 or 8; got {self.connectivity!r}") - try: - from scipy.ndimage import find_objects, generate_binary_structure, label - except ImportError as exc: - raise ImportError( - "ConnectedComponentsOp requires scipy. " - "Install with `pip install data-flux[vision]` or add scipy to your environment." - ) from exc - mask = sample.input if not isinstance(mask, np.ndarray): raise TypeError(f"ConnectedComponentsOp expects an np.ndarray on sample.input, got {type(mask).__name__}") if mask.ndim != 2: raise ValueError(f"ConnectedComponentsOp expects a 2-D mask; got shape {mask.shape}") - - structure = generate_binary_structure(2, 1 if self.connectivity == 4 else 2) - labels, n_components = label(mask, structure=structure) - bboxes: List[Tuple[int, int, int, int]] = [] - if n_components > 0: - for idx, sl in enumerate(find_objects(labels), start=1): - if sl is None: - continue - row_slice, col_slice = sl - area = int((labels[row_slice, col_slice] == idx).sum()) - if area < self.min_area_bins: - continue - bboxes.append( - ( - int(row_slice.start), - int(row_slice.stop) - 1, - int(col_slice.start), - int(col_slice.stop) - 1, - ) - ) + # Shared scipy core (also used by dataflux.ops.target.MasksToDetectionBoxesOp); + # validates min_area_bins / connectivity and raises the scipy ImportError. + bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) return sample._replace(input=bboxes) diff --git a/dataflux/ops/target.py b/dataflux/ops/target.py index 170d1c9..d4d018d 100644 --- a/dataflux/ops/target.py +++ b/dataflux/ops/target.py @@ -11,18 +11,27 @@ whatever labels happen to appear, so train / eval / predict share one identical ordering. -These are deliberately small, value-agnostic plumbing ops (no ``ACCEPTS`` / +The first three are deliberately small, value-agnostic plumbing ops (no ``ACCEPTS`` / ``PRODUCES`` contract, like ``copy`` / ``swap`` / ``stash``). The encoded value is written verbatim (e.g. a plain ``int``); wrap it into a framework tensor downstream (e.g. a collate function) when a loss needs one. + +* :class:`CocoToTorchVisionDetectionOp` is the one structured-target op here: it turns a + HuggingFace / COCO ``objects`` annotation (``{bbox, category}``) into the torchvision + detection target ``{"boxes": xyxy, "labels"}`` (torch tensors). It is the generic, + image-detection counterpart of waivefront's signal-domain ``RegionsToDetectionBoxesOp``. """ -from typing import Any, Dict, Optional +from typing import Any, Dict, Literal, Optional from confluid import configurable from dataflux.sample import Sample +#: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo +#: fails at the call site and UIs / form-specs enumerate the choices. +BBoxFormat = Literal["xywh", "xyxy", "cxcywh"] + def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: Any, op_name: str) -> Any: """Return ``mapping[value]``, or ``default`` when missing and ``ignore_unknown``. @@ -141,4 +150,160 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(target=decoded) -__all__ = ["MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp"] +@configurable(category="op", group="structure") +class CocoToTorchVisionDetectionOp: + """Convert a HuggingFace / COCO ``objects`` annotation to a torchvision detection target. + + HuggingFace object-detection datasets (e.g. ``cppe-5``) carry per-image annotations as an + ``objects`` mapping — ``{"bbox": [[...], ...], "category": [...], ...}`` — where each box is, + by COCO convention, ``[x, y, w, h]`` in absolute pixels and ``category`` is an integer class + id. ``HuggingFaceSource(target_feature="objects")`` lands that mapping verbatim on + ``sample.target``; this op rewrites it to the shape ``raidar.detection.detection_collate_fn`` + and the detection trainer consume:: + + sample.target = {"boxes": [N, 4] float32 xyxy-pixel, "labels": [N] int64} + + The modality-neutral, image-detection counterpart of waivefront's signal-domain + :class:`~waivefront.targets.RegionsToDetectionBoxesOp` (which projects time/frequency + regions) — it lives in core dataflux because the COCO→xyxy conversion is fully generic. + The input image is left untouched (tensorize it with :class:`~dataflux.ops.torch.ToTensorOp`). + An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract + torchvision detectors accept). + + Args: + bbox_key: Key in the objects mapping holding per-box coordinates (default ``"bbox"``). + category_key: Key holding the per-box integer class ids (default ``"category"``). + bbox_format: Box layout in pixels — ``xywh`` (COCO, default), ``xyxy``, or ``cxcywh``; output is xyxy. + label_offset: Added to each class id (default ``0``). Set ``1`` to reserve class ``0`` for background. + """ + + def __init__( + self, + bbox_key: str = "bbox", + category_key: str = "category", + bbox_format: BBoxFormat = "xywh", + label_offset: int = 0, + ) -> None: + # Lazy / zero-arg: store config only. The objects-shaped target is validated in __call__. + self.bbox_key = str(bbox_key) + self.category_key = str(category_key) + self.bbox_format = bbox_format + self.label_offset = int(label_offset) + + def __call__(self, sample: Sample) -> Sample: + # torch is imported lazily so this module stays import-light for the value-agnostic + # plumbing ops above (which need no framework). + import torch + + objects = sample.target + if not isinstance(objects, dict): + raise TypeError( + f"CocoToTorchVisionDetectionOp: sample.target must be a COCO/HF objects mapping " + f"(a dict with {self.bbox_key!r}/{self.category_key!r}); got {type(objects).__name__}. " + "Wire HuggingFaceSource(target_feature='objects') upstream." + ) + raw_boxes = objects.get(self.bbox_key) or [] + raw_labels = objects.get(self.category_key) or [] + + if len(raw_boxes): + boxes = torch.as_tensor(raw_boxes, dtype=torch.float32).reshape(-1, 4) + if self.bbox_format == "xywh": # COCO: top-left + size + x, y, w, h = boxes.unbind(-1) + boxes = torch.stack([x, y, x + w, y + h], dim=-1) + elif self.bbox_format == "cxcywh": # center + size + cx, cy, w, h = boxes.unbind(-1) + boxes = torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1) + # "xyxy": already in the output layout + else: + boxes = torch.zeros((0, 4), dtype=torch.float32) + + if len(raw_labels): + labels = torch.as_tensor(list(raw_labels), dtype=torch.int64).reshape(-1) + self.label_offset + else: + labels = torch.zeros((0,), dtype=torch.int64) + + return sample._replace(target={"boxes": boxes, "labels": labels}) + + +@configurable(category="op", group="structure") +class MasksToDetectionBoxesOp: + """Convert a segmentation MASK on ``sample.target`` to a torchvision detection target. + + Reads a 2-D integer mask (PIL ``L`` image or ndarray) and rewrites ``sample.target`` to + ``{"boxes": [N,4] float32 xyxy-pixel, "labels": [N] int64}`` — the tight per-object box. This is + the derivation the official torchvision **Penn-Fudan** object-detection tutorial performs (the + dataset ships masks, not boxes). Two object-separation modes: + + * ``connected=False`` (default) — an **instance mask**: each distinct non-zero pixel value is one + object (box = the tight extent of ``mask == value``). Penn-Fudan's ``instance_id`` mask (pixels + ``1..N``, one per pedestrian) is exactly this — exact even when objects touch. + * ``connected=True`` — a **binary / semantic mask**: binarize (non-zero), then split into connected + components via :func:`dataflux.ops.numpy.connected_component_bboxes` (one box per blob). Use for + a semantic mask (all objects share one value) or a model's predicted foreground mask. + + Every box gets class id ``label`` (one foreground class; class 0 stays background — so a 1-class + dataset like Penn-Fudan derives ``num_classes = 2``). The input image is left untouched (tensorize + with :class:`~dataflux.ops.torch.ToTensorOp` ``mode="RGB"``). An empty mask yields empty ``[0,4]`` / + ``[0]`` tensors (the negative-example contract torchvision detectors accept). + + Args: + label: Foreground class id assigned to every derived box (default ``1``; class 0 = background). + connected: True = connected-components on a binary mask; False (default) = each non-zero value is one instance. + min_area: Drop objects whose mask area (in pixels) is below this (default ``1``). + connectivity: Connected-components neighborhood when ``connected=True`` — ``4`` or ``8`` (default ``4``). + """ + + def __init__(self, label: int = 1, connected: bool = False, min_area: int = 1, connectivity: int = 4) -> None: + # Lazy / zero-arg: store config only; the mask shape is validated in __call__. + self.label = int(label) + self.connected = bool(connected) + self.min_area = int(min_area) + self.connectivity = int(connectivity) + + def __call__(self, sample: Sample) -> Sample: + import numpy as np + import torch + + mask = sample.target + if hasattr(mask, "convert"): # PIL image (e.g. an 'L' instance mask) + mask = np.array(mask) + mask = np.asarray(mask) + if mask.ndim != 2: + raise TypeError( + f"MasksToDetectionBoxesOp: sample.target must be a 2-D segmentation mask " + f"(PIL 'L' image or 2-D array); got shape {getattr(mask, 'shape', None)}. " + "Wire HuggingFaceSource(target_feature='') upstream." + ) + + boxes: list = [] + if self.connected: + from dataflux.ops.numpy import connected_component_bboxes + + # row/col-inclusive (r0,r1,c0,c1) → xyxy-pixel (x0,y0,x1,y1) with exclusive far edge. + for r0, r1, c0, c1 in connected_component_bboxes(mask != 0, self.min_area, self.connectivity): + boxes.append((float(c0), float(r0), float(c1 + 1), float(r1 + 1))) + else: + for value in np.unique(mask): + if int(value) == 0: + continue + ys, xs = np.where(mask == value) + if int(ys.size) < self.min_area: + continue + boxes.append((float(xs.min()), float(ys.min()), float(xs.max() + 1), float(ys.max() + 1))) + + if boxes: + boxes_t = torch.tensor(boxes, dtype=torch.float32) + labels_t = torch.full((len(boxes),), self.label, dtype=torch.int64) + else: + boxes_t = torch.zeros((0, 4), dtype=torch.float32) + labels_t = torch.zeros((0,), dtype=torch.int64) + return sample._replace(target={"boxes": boxes_t, "labels": labels_t}) + + +__all__ = [ + "MetadataToTargetOp", + "EncodeTargetOp", + "DecodeTargetOp", + "CocoToTorchVisionDetectionOp", + "MasksToDetectionBoxesOp", +] diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index bccf47e..14803e4 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -1,4 +1,4 @@ -from typing import Sequence, Union +from typing import Optional, Sequence, Union import numpy as np import torch @@ -18,20 +18,25 @@ class ToTensorOp: Args: normalize: When ``True``, scale integer pixel inputs into the ``[0, 1]`` float range during conversion. + mode: Optional PIL mode to convert to (e.g. "RGB" forces 3 channels); None (default) arrays as-is. """ ACCEPTS = SampleType(input=UnionType((PythonType("PIL.Image.Image"), ArrayType(frameworks={"numpy"})))) PRODUCES = SampleType(input=_TORCH) - def __init__(self, normalize: bool = True): + def __init__(self, normalize: bool = True, mode: Optional[str] = None): self.normalize = normalize + self.mode = mode def __call__(self, sample: Sample) -> Sample: img = sample.input # Handle PIL / PngImageFile if hasattr(img, "convert"): - # Ensure grayscale or RGB as needed, but for generic we just array it + # Optionally coerce the PIL mode (e.g. "RGB") so a mixed-mode dataset + # (RGBA / grayscale / palette samples) yields a uniform channel count. + if self.mode is not None: + img = img.convert(self.mode) img = np.array(img) # Convert to Tensor diff --git a/tests/test_categories.py b/tests/test_categories.py index df9239b..fb1440e 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -16,7 +16,13 @@ from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp from dataflux.ops.parallel import Parallel from dataflux.ops.sink import SampleSinkOp -from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp +from dataflux.ops.target import ( + CocoToTorchVisionDetectionOp, + DecodeTargetOp, + EncodeTargetOp, + MasksToDetectionBoxesOp, + MetadataToTargetOp, +) from dataflux.ops.tee import Tee from dataflux.ops.torch import ToTensorOp from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource @@ -68,6 +74,8 @@ def test_op_classes_tagged() -> None: assert MetadataToTargetOp.__confluid_category__ == "op" assert EncodeTargetOp.__confluid_category__ == "op" assert DecodeTargetOp.__confluid_category__ == "op" + assert CocoToTorchVisionDetectionOp.__confluid_category__ == "op" + assert MasksToDetectionBoxesOp.__confluid_category__ == "op" def test_op_group_tags() -> None: @@ -83,6 +91,8 @@ def test_op_group_tags() -> None: assert MetadataToTargetOp.__confluid_group__ == "structure" assert EncodeTargetOp.__confluid_group__ == "structure" assert DecodeTargetOp.__confluid_group__ == "structure" + assert CocoToTorchVisionDetectionOp.__confluid_group__ == "structure" + assert MasksToDetectionBoxesOp.__confluid_group__ == "structure" assert Tee.__confluid_group__ == "compose" assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" @@ -116,6 +126,8 @@ def test_categories_enumerable_via_registry() -> None: "MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp", + "CocoToTorchVisionDetectionOp", + "MasksToDetectionBoxesOp", } <= registry.list_classes(category="op") @@ -126,6 +138,12 @@ def test_groups_enumerable_via_registry() -> None: assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") assert {"Tee", "Parallel", "Enable"} <= registry.list_classes(group="compose") assert {"SampleSinkOp"} <= registry.list_classes(group="sink") - assert {"MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp"} <= registry.list_classes(group="structure") + assert { + "MetadataToTargetOp", + "EncodeTargetOp", + "DecodeTargetOp", + "CocoToTorchVisionDetectionOp", + "MasksToDetectionBoxesOp", + } <= registry.list_classes(group="structure") # group × category intersect, like task × role. assert "Tee" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_ops.py b/tests/test_ops.py index 4cffc0b..cd4d6ff 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -71,6 +71,23 @@ def test_preserves_target_and_metadata(self) -> None: assert result.target == 5 assert result.meta == {"k": "v"} + def test_mode_rgb_forces_three_channels_from_mixed_pil_modes(self) -> None: + # A mixed-mode image dataset (RGBA / grayscale / palette) → uniform 3-channel + # RGB for a fixed-channel model (e.g. torchvision Faster R-CNN's 3-ch normalize). + op = ToTensorOp(mode="RGB") + for mode, arr in ( + ("RGBA", np.zeros((8, 8, 4), dtype=np.uint8)), + ("L", np.zeros((8, 8), dtype=np.uint8)), + ("RGB", np.zeros((8, 8, 3), dtype=np.uint8)), + ): + out = op(Sample(input=Image.fromarray(arr, mode=mode))) + assert out.input.shape == (3, 8, 8), f"{mode} → {tuple(out.input.shape)}" + + def test_mode_none_leaves_channels_as_is(self) -> None: + # Default mode=None arrays the image verbatim — RGBA stays 4-channel. + rgba = Image.fromarray(np.zeros((8, 8, 4), dtype=np.uint8), mode="RGBA") + assert ToTensorOp()(Sample(input=rgba)).input.shape == (4, 8, 8) + # --------------------------------------------------------------------------- # Torch RescaleOp diff --git a/tests/test_target_ops.py b/tests/test_target_ops.py index f1fa71a..6d8cade 100644 --- a/tests/test_target_ops.py +++ b/tests/test_target_ops.py @@ -1,8 +1,16 @@ """Tests for the target movers / encoders (``dataflux.ops.target``).""" +import numpy as np import pytest - -from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp +from PIL import Image + +from dataflux.ops.target import ( + CocoToTorchVisionDetectionOp, + DecodeTargetOp, + EncodeTargetOp, + MasksToDetectionBoxesOp, + MetadataToTargetOp, +) from dataflux.sample import Sample @@ -96,3 +104,113 @@ def test_metadata_to_target_then_encode() -> None: assert sample.target == 2 # raw label preserved for decode/reporting assert sample.meta["raw_label"] == "DJI AVATA2" + + +# --------------------------------------------------------------------------- # +# CocoToTorchVisionDetectionOp (HF / COCO objects -> {boxes xyxy, labels}) +# --------------------------------------------------------------------------- # +def _objects_sample(bbox: object, category: object) -> Sample: + """A Sample shaped like ``HuggingFaceSource(target_feature='objects')`` output.""" + return Sample(input="img", target={"bbox": bbox, "category": category}, metadata={}) + + +def test_objects_to_boxes_xywh_to_xyxy_and_dtypes() -> None: + op = CocoToTorchVisionDetectionOp() # default bbox_format="xywh" + out = op(_objects_sample([[10, 20, 30, 40]], [2])) + # COCO [x,y,w,h]=[10,20,30,40] -> xyxy [10,20,40,60] + assert out.target["boxes"].tolist() == [[10.0, 20.0, 40.0, 60.0]] + assert out.target["labels"].tolist() == [2] + assert str(out.target["boxes"].dtype) == "torch.float32" + assert str(out.target["labels"].dtype) == "torch.int64" + + +def test_objects_to_boxes_label_offset_for_background_class() -> None: + # label_offset=1 shifts 0-indexed dataset categories to torchvision foreground ids 1..K. + op = CocoToTorchVisionDetectionOp(label_offset=1) + out = op(_objects_sample([[0, 0, 4, 4], [1, 1, 2, 2]], [0, 3])) + assert out.target["labels"].tolist() == [1, 4] + + +def test_objects_to_boxes_xyxy_passthrough() -> None: + op = CocoToTorchVisionDetectionOp(bbox_format="xyxy") + out = op(_objects_sample([[1, 2, 3, 4]], [0])) + assert out.target["boxes"].tolist() == [[1.0, 2.0, 3.0, 4.0]] + + +def test_objects_to_boxes_cxcywh() -> None: + op = CocoToTorchVisionDetectionOp(bbox_format="cxcywh") + # center (50,50), size (20,40) -> [40,30,60,70] + out = op(_objects_sample([[50, 50, 20, 40]], [1])) + assert out.target["boxes"].tolist() == [[40.0, 30.0, 60.0, 70.0]] + + +def test_objects_to_boxes_empty_annotation_yields_empty_tensors() -> None: + op = CocoToTorchVisionDetectionOp() + out = op(_objects_sample([], [])) + assert tuple(out.target["boxes"].shape) == (0, 4) + assert tuple(out.target["labels"].shape) == (0,) + + +def test_objects_to_boxes_custom_keys() -> None: + op = CocoToTorchVisionDetectionOp(bbox_key="boxes", category_key="labels") + out = op(Sample(input="i", target={"boxes": [[0, 0, 2, 2]], "labels": [5]}, metadata={})) + assert out.target["boxes"].tolist() == [[0.0, 0.0, 2.0, 2.0]] + assert out.target["labels"].tolist() == [5] + + +def test_objects_to_boxes_rejects_non_mapping_target() -> None: + with pytest.raises(TypeError, match="objects mapping"): + CocoToTorchVisionDetectionOp()(Sample(input="i", target=[1, 2, 3], metadata={})) + + +# --------------------------------------------------------------------------- # +# MasksToDetectionBoxesOp (segmentation mask -> {boxes xyxy, labels}) +# --------------------------------------------------------------------------- # +def _instance_mask() -> np.ndarray: + """Two objects: instance id 1 at rows 2-4/cols 1-3, id 2 at rows 6-8/cols 7-10.""" + m = np.zeros((10, 12), dtype=np.uint8) + m[2:5, 1:4] = 1 + m[6:9, 7:11] = 2 + return m + + +def test_masks_instance_mode_per_id_bbox_and_dtypes() -> None: + op = MasksToDetectionBoxesOp(label=1) # default connected=False + out = op(Sample(input="img", target=Image.fromarray(_instance_mask(), mode="L"), metadata={})) + # row/col extents → xyxy with exclusive far edge. + assert sorted(out.target["boxes"].tolist()) == [[1.0, 2.0, 4.0, 5.0], [7.0, 6.0, 11.0, 9.0]] + assert out.target["labels"].tolist() == [1, 1] + assert str(out.target["boxes"].dtype) == "torch.float32" + assert str(out.target["labels"].dtype) == "torch.int64" + + +def test_masks_label_assigns_class_id() -> None: + out = MasksToDetectionBoxesOp(label=3)(Sample(input="i", target=_instance_mask(), metadata={})) + assert out.target["labels"].tolist() == [3, 3] + + +def test_masks_connected_mode_splits_semantic_blobs() -> None: + # A SEMANTIC mask (both objects = 1) — connected components separate the two blobs. + sem = (_instance_mask() > 0).astype(np.uint8) + out = MasksToDetectionBoxesOp(connected=True)(Sample(input="i", target=sem, metadata={})) + assert sorted(out.target["boxes"].tolist()) == [[1.0, 2.0, 4.0, 5.0], [7.0, 6.0, 11.0, 9.0]] + assert out.target["labels"].tolist() == [1, 1] + + +def test_masks_min_area_drops_small_instances() -> None: + m = np.zeros((8, 8), dtype=np.uint8) + m[0, 0] = 1 # area 1 + m[4:7, 4:7] = 2 # area 9 + out = MasksToDetectionBoxesOp(min_area=2)(Sample(input="i", target=m, metadata={})) + assert out.target["boxes"].tolist() == [[4.0, 4.0, 7.0, 7.0]] + + +def test_masks_empty_mask_yields_empty_tensors() -> None: + out = MasksToDetectionBoxesOp()(Sample(input="i", target=np.zeros((5, 5), dtype=np.uint8), metadata={})) + assert tuple(out.target["boxes"].shape) == (0, 4) + assert tuple(out.target["labels"].shape) == (0,) + + +def test_masks_rejects_non_2d_target() -> None: + with pytest.raises(TypeError, match="2-D segmentation mask"): + MasksToDetectionBoxesOp()(Sample(input="i", target=np.zeros((4, 4, 3), dtype=np.uint8), metadata={})) From d60307eab9d58d33803958100be509ebc1a54d2d Mon Sep 17 00:00:00 2001 From: gertbehi Date: Tue, 9 Jun 2026 14:16:49 +0200 Subject: [PATCH 011/102] =?UTF-8?q?feat:=20add=20RandomApply=20=E2=80=94?= =?UTF-8?q?=20Bernoulli=20gate=20op=20in=20dataflux.ops.compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps any Sample→Sample callable behind a coin flip (probability=0.5). Confluid Fluid ops are resolved lazily on first call and then cached. Registered as @configurable(category="op", group="compose", random=True) and wired into the confluid.configurables entry-point group. 8 unit tests covering zero-arg construction, deterministic extremes (probability=0/1), no-op passthrough, Fluid lazy resolution, and confluid registry membership. --- dataflux/ops/__init__.py | 13 +++++-- dataflux/ops/random_apply.py | 67 ++++++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_random_apply.py | 74 ++++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 dataflux/ops/random_apply.py create mode 100644 tests/test_random_apply.py diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index a224ce7..d2cd5ef 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -3,11 +3,14 @@ Submodules: - dataflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, - ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp (ndarray) - - dataflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp (tensor) + ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp, SqueezeOp, + UnsqueezeOp (ndarray) + - dataflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp, SqueezeOp, + UnsqueezeOp (tensor) - dataflux.ops.tee: Tee (fan-out branching) - dataflux.ops.parallel: Parallel (worker-pool sub-pipeline) - dataflux.ops.enable: Enable (toggle an op-list via one named CLI flag) + - dataflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - dataflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - dataflux.ops.swap: SwapInputTargetOp @@ -21,6 +24,7 @@ from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp from dataflux.ops.enable import Enable from dataflux.ops.parallel import Parallel +from dataflux.ops.random_apply import RandomApply from dataflux.ops.sink import SampleSinkOp from dataflux.ops.stash import StashInputOp, UnstashInputOp from dataflux.ops.swap import SwapInputTargetOp @@ -32,7 +36,7 @@ MetadataToTargetOp, ) from dataflux.ops.tee import Tee -from dataflux.ops.torch import RescaleOp, StandardizeOp, ToTensorOp +from dataflux.ops.torch import RescaleOp, SqueezeOp, StandardizeOp, ToTensorOp, UnsqueezeOp __all__ = [ "CopyInputOp", @@ -46,12 +50,15 @@ "CocoToTorchVisionDetectionOp", "MasksToDetectionBoxesOp", "Parallel", + "RandomApply", "RescaleOp", "SampleSinkOp", + "SqueezeOp", "StandardizeOp", "StashInputOp", "SwapInputTargetOp", "Tee", "ToTensorOp", "UnstashInputOp", + "UnsqueezeOp", ] diff --git a/dataflux/ops/random_apply.py b/dataflux/ops/random_apply.py new file mode 100644 index 0000000..d472aec --- /dev/null +++ b/dataflux/ops/random_apply.py @@ -0,0 +1,67 @@ +"""``RandomApply`` — apply an op with a given probability. + +A compose-group op (alongside ``Enable`` / ``Tee`` / ``Parallel``): +wrap any single ``Sample → Sample`` op so it fires only *p* fraction of +the time. Samples that are skipped pass through unchanged. + +Modality-neutral — it threads any ``Sample`` through any op — so it lives +in core dataflux, not a domain package. +""" + +import random +from typing import Optional, cast + +from confluid import configurable +from logflow import get_logger + +from dataflux.sample import Sample + +logger = get_logger(__name__) + + +@configurable(category="op", group="compose", random=True) +class RandomApply: + """Gate any op behind a Bernoulli coin flip. + + On each call, a uniform ``U ~ [0, 1)`` is drawn; if ``U < probability`` + the inner ``op`` is applied, otherwise the sample passes through unchanged. + + ``op`` is flowed lazily on first use (Confluid ``!class:`` / ``!lazy:`` + markers are resolved at call-time, not at construction), so building a + ``RandomApply()`` with no arguments costs nothing. + + YAML: + + .. code-block:: yaml + + - !class:dataflux.ops.random_apply.RandomApply + probability: 0.5 + op: !class:dataflux.ops.numpy.RescaleOp + in_min: -1.0 + in_max: 1.0 + + Args: + op: Inner ``Sample → Sample`` callable to gate. Defaults to ``None`` + (identity); validated lazily on first call. + probability: Gate probability in ``[0, 1]``. ``0.0`` = never apply; + ``1.0`` = always apply. Defaults to ``0.5``. + """ + + def __init__(self, op: Optional[object] = None, probability: float = 0.5) -> None: + self.op = op + self.probability = probability + + def __call__(self, sample: Sample) -> Sample: + if self.op is None: + raise ValueError("RandomApply requires 'op' to be set before calling.") + if random.random() >= self.probability: + return sample + from confluid import flow + from confluid.fluid import Fluid + + op = flow(self.op) if isinstance(self.op, Fluid) else self.op + self.op = op # cache the flowed op so we only flow once + return cast(Sample, op(sample)) # type: ignore[operator] + + +__all__ = ["RandomApply"] diff --git a/pyproject.toml b/pyproject.toml index bd7caa0..f103318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dataflux-sources = "dataflux.sources" dataflux-ops-parallel = "dataflux.ops.parallel" dataflux-ops-tee = "dataflux.ops.tee" dataflux-ops-enable = "dataflux.ops.enable" +dataflux-ops-random-apply = "dataflux.ops.random_apply" dataflux-ops-sink = "dataflux.ops.sink" dataflux-ops-stash = "dataflux.ops.stash" dataflux-ops-numpy = "dataflux.ops.numpy" diff --git a/tests/test_random_apply.py b/tests/test_random_apply.py new file mode 100644 index 0000000..5d64528 --- /dev/null +++ b/tests/test_random_apply.py @@ -0,0 +1,74 @@ +"""Tests for dataflux.ops.random_apply.RandomApply.""" + +import pytest + +from dataflux.ops.random_apply import RandomApply +from dataflux.sample import Sample + + +def _s(v: int = 0) -> Sample: + return Sample(input=v, metadata={}) + + +class _BumpOp: + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + 1) + + +def test_zero_arg_construction() -> None: + assert RandomApply() is not None + + +def test_probability_zero_never_applies() -> None: + op = RandomApply(op=_BumpOp(), probability=0.0) + for _ in range(20): + out = op(_s(0)) + assert out.input == 0 + + +def test_probability_one_always_applies() -> None: + op = RandomApply(op=_BumpOp(), probability=1.0) + for _ in range(20): + out = op(_s(0)) + assert out.input == 1 + + +def test_raises_when_op_is_none() -> None: + op = RandomApply(probability=1.0) + with pytest.raises(ValueError, match="op"): + op(_s()) + + +def test_is_marked_random() -> None: + assert getattr(RandomApply, "__confluid_random__", False) is True + + +def test_is_registered_configurable() -> None: + from confluid.registry import resolve_class # type: ignore[import-not-found] + + path = f"{RandomApply.__module__}.{RandomApply.__qualname__}" + assert resolve_class(path) is RandomApply + + +def test_flows_confluid_fluid_op_lazily() -> None: + from confluid import configurable + from confluid.fluid import Class + + @configurable + class _Inner: + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + 10) + + fluid_op = Class(_Inner) + op = RandomApply(op=fluid_op, probability=1.0) + out = op(_s(5)) + assert out.input == 15 + # second call reuses the cached flowed op + out2 = op(_s(5)) + assert out2.input == 15 + + +def test_sample_passes_through_unchanged_when_skipped() -> None: + s = _s(42) + op = RandomApply(op=_BumpOp(), probability=0.0) + assert op(s) is s From 4f79e10317bdcdd707def7b2d54656fa63018cf5 Mon Sep 17 00:00:00 2001 From: gertbehi Date: Tue, 9 Jun 2026 17:08:06 +0200 Subject: [PATCH 012/102] fix: make ThresholdOp robust against None and blank-string bounds - _resolve() raises ValueError if called with None (guard against misuse; __call__ already filters None via self.low_level/high_level is not None, but the signature now reflects it) - __call__ also strips blank STRING values (empty widget) to None before the None check, so leaving a FluxStudio STRING widget empty silently disables that bound --- dataflux/ops/numpy.py | 198 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 5 deletions(-) diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index 6f72d4e..6d321a6 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -125,6 +125,46 @@ def _require_ndarray(sample: Sample, op_name: str) -> np.ndarray: return arr +@configurable(category="op", group="numpy") +class SqueezeOp: + """Remove size-1 axes from an ``np.ndarray``. + + Args: + axis: Axis index to remove. When ``None`` (default), all size-1 axes are removed. + When specified, the axis must have size 1 (numpy raises ``ValueError`` otherwise). + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: Optional[int] = None) -> None: + self.axis = axis + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "SqueezeOp") + out = np.squeeze(arr) if self.axis is None else np.squeeze(arr, axis=self.axis) + return sample._replace(input=out) + + +@configurable(category="op", group="numpy") +class UnsqueezeOp: + """Insert a size-1 axis at the specified position in an ``np.ndarray``. + + Args: + axis: Axis index at which the new dimension is inserted. Default ``0``. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: int = 0) -> None: + self.axis = axis + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "UnsqueezeOp") + return sample._replace(input=np.expand_dims(arr, axis=self.axis)) + + @configurable(category="op", group="numpy") class ClipPercentilesOp: """Clip ``sample.input`` to ``[p_low, p_high]`` percentiles of finite values. @@ -329,7 +369,9 @@ def __init__( self.low_op = low_op self.high_op = high_op - def _resolve(self, bound: Union[float, int, str], sample: Sample) -> float: + def _resolve(self, bound: Optional[Union[float, int, str]], sample: Sample) -> float: + if bound is None: + raise ValueError("ThresholdOp._resolve called with None — bound was not filtered by __call__") if isinstance(bound, (int, float)): return float(bound) if not isinstance(bound, str): @@ -346,13 +388,21 @@ def __call__(self, sample: Sample) -> Sample: arr = sample.input if not isinstance(arr, np.ndarray): raise TypeError(f"ThresholdOp expects an np.ndarray on sample.input, got {type(arr).__name__}") + low_level = self.low_level + high_level = self.high_level + # Treat empty string (blank STRING widget left unset) as None ("disabled"). + if isinstance(low_level, str) and low_level.strip() == "": + low_level = None + if isinstance(high_level, str) and high_level.strip() == "": + high_level = None + mask: Optional[np.ndarray] = None - if self.low_level is not None: - low = self._resolve(self.low_level, sample) + if low_level is not None: + low = self._resolve(low_level, sample) sample.meta["threshold_low"] = low mask = _LOW_COMPARISONS[self.low_op](arr, low) - if self.high_level is not None: - high = self._resolve(self.high_level, sample) + if high_level is not None: + high = self._resolve(high_level, sample) sample.meta["threshold_high"] = high below = _HIGH_COMPARISONS[self.high_op](arr, high) mask = below if mask is None else (mask & below) @@ -407,6 +457,144 @@ def connected_component_bboxes( return bboxes +@configurable(category="op", group="numpy") +class MinOp: + """Reduce ``sample.input`` to its minimum value, ignoring NaN. + + Args: + axis: Axis along which to compute the minimum. ``None`` (default) reduces over all axes. + keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: Optional[int] = None, keepdims: bool = False) -> None: + self.axis = axis + self.keepdims = bool(keepdims) + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "MinOp") + return sample._replace(input=np.nanmin(arr, axis=self.axis, keepdims=self.keepdims)) + + +@configurable(category="op", group="numpy") +class MaxOp: + """Reduce ``sample.input`` to its maximum value, ignoring NaN. + + Args: + axis: Axis along which to compute the maximum. ``None`` (default) reduces over all axes. + keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: Optional[int] = None, keepdims: bool = False) -> None: + self.axis = axis + self.keepdims = bool(keepdims) + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "MaxOp") + return sample._replace(input=np.nanmax(arr, axis=self.axis, keepdims=self.keepdims)) + + +@configurable(category="op", group="numpy") +class MedianOp: + """Reduce ``sample.input`` to its median value, ignoring NaN. + + Args: + axis: Axis along which to compute the median. ``None`` (default) reduces over all axes. + keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: Optional[int] = None, keepdims: bool = False) -> None: + self.axis = axis + self.keepdims = bool(keepdims) + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "MedianOp") + return sample._replace(input=np.nanmedian(arr, axis=self.axis, keepdims=self.keepdims)) + + +@configurable(category="op", group="numpy") +class PercentileOp: + """Reduce ``sample.input`` to a 2-element array ``[p_low, p_high]``, ignoring NaN. + + Output shape when ``axis=None``: ``(2,)`` scalar pair. When ``axis=k``: + ``(2, …)`` stacked along a new leading dimension. + + Args: + low: Lower percentile in ``[0, 100]``. Default ``5.0``. + high: Upper percentile in ``[0, 100]``, should be ``> low``. Default ``95.0``. + axis: Axis along which to compute the percentiles. ``None`` (default) reduces over all axes. + keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__( + self, + low: float = 5.0, + high: float = 95.0, + axis: Optional[int] = None, + keepdims: bool = False, + ) -> None: + self.low = float(low) + self.high = float(high) + self.axis = axis + self.keepdims = bool(keepdims) + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "PercentileOp") + p_low = np.nanpercentile(arr, self.low, axis=self.axis, keepdims=self.keepdims) + p_high = np.nanpercentile(arr, self.high, axis=self.axis, keepdims=self.keepdims) + return sample._replace(input=np.stack([p_low, p_high])) + + +@configurable(category="op", group="numpy") +class StatsOp: + """Compute summary statistics of ``sample.input`` and record them in metadata; input is passed through unchanged. + + Writes five scalar float keys to ``sample.metadata``: ``{prefix}min``, + ``{prefix}max``, ``{prefix}median``, ``{prefix}p_low``, ``{prefix}p_high``. + NaN values are excluded from all computations. + + Chain anywhere in a pipeline without disrupting the data flow — useful for + inspecting distribution properties during development or for downstream + normalisation decisions. + + Args: + low: Lower percentile bound (0–100). Default ``5.0``. + high: Upper percentile bound (0–100). Default ``95.0``. + prefix: Optional string prepended to every metadata key, e.g. ``"input_"`` to + distinguish multiple ``StatsOp`` invocations in one pipeline. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, low: float = 5.0, high: float = 95.0, prefix: str = "") -> None: + self.low = float(low) + self.high = float(high) + self.prefix = prefix + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "StatsOp") + p = self.prefix + meta = dict(sample.meta) + meta[f"{p}min"] = float(np.nanmin(arr)) + meta[f"{p}max"] = float(np.nanmax(arr)) + meta[f"{p}median"] = float(np.nanmedian(arr)) + meta[f"{p}p_low"] = float(np.nanpercentile(arr, self.low)) + meta[f"{p}p_high"] = float(np.nanpercentile(arr, self.high)) + return sample._replace(metadata=meta) + + @configurable(category="op", group="numpy") class ConnectedComponentsOp: """Label connected ``True`` regions of a boolean mask into bin-bbox tuples. From 2968a68ef0f0572922f53be5a4e1849d889496cc Mon Sep 17 00:00:00 2001 From: gertbehi Date: Thu, 11 Jun 2026 21:47:07 +0200 Subject: [PATCH 013/102] feat: Add TransformChain op for sequentially applying a list of operations - Introduced TransformChain class to group a sequence of ops into a single unit. - Each op in the chain is applied in order, and if any op returns None, the chain stops early. - Added SqueezeOp and UnsqueezeOp for tensor dimension manipulation in PyTorch. - Implemented StashTargetOp and UnstashTargetOp for managing sample targets in metadata. - Enhanced RandomApply to support reproducibility with a random_state parameter. - Updated tests to cover new functionality and ensure compatibility with existing ops. --- AGENTS.md | 2 +- dataflux/ops/__init__.py | 9 +- dataflux/ops/image.py | 4 +- dataflux/ops/numpy.py | 23 +++- dataflux/ops/random_apply.py | 14 +- dataflux/ops/stash.py | 57 +++++++- dataflux/ops/torch.py | 45 +++++++ dataflux/ops/transform_chain.py | 84 ++++++++++++ dataflux/typespec.py | 78 +++++++++++ pyproject.toml | 1 + tests/test_categories.py | 9 +- tests/test_node_docs.py | 2 + tests/test_ops.py | 228 ++++++++++++++++++++++++++++++++ tests/test_random_apply.py | 40 ++++++ tests/test_transform_chain.py | 157 ++++++++++++++++++++++ tests/test_typespec.py | 119 ++++++++++++++++- 16 files changed, 856 insertions(+), 16 deletions(-) create mode 100644 dataflux/ops/transform_chain.py create mode 100644 tests/test_transform_chain.py diff --git a/AGENTS.md b/AGENTS.md index ac20ed1..9ff65b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` (`Tee`/`Parallel`/`Enable`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index d2cd5ef..008ad59 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -12,9 +12,10 @@ - dataflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - dataflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - dataflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) + - dataflux.ops.transform_chain: TransformChain (sequential op-chain grouping) - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - dataflux.ops.swap: SwapInputTargetOp - - dataflux.ops.stash: StashInputOp, UnstashInputOp + - dataflux.ops.stash: StashInputOp, UnstashInputOp, StashTargetOp, UnstashTargetOp - dataflux.ops.target: MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp (target field) Flat imports default to torch variants for the data ops; flow / copy / @@ -26,7 +27,7 @@ from dataflux.ops.parallel import Parallel from dataflux.ops.random_apply import RandomApply from dataflux.ops.sink import SampleSinkOp -from dataflux.ops.stash import StashInputOp, UnstashInputOp +from dataflux.ops.stash import StashInputOp, StashTargetOp, UnstashInputOp, UnstashTargetOp from dataflux.ops.swap import SwapInputTargetOp from dataflux.ops.target import ( CocoToTorchVisionDetectionOp, @@ -37,6 +38,7 @@ ) from dataflux.ops.tee import Tee from dataflux.ops.torch import RescaleOp, SqueezeOp, StandardizeOp, ToTensorOp, UnsqueezeOp +from dataflux.ops.transform_chain import TransformChain __all__ = [ "CopyInputOp", @@ -56,9 +58,12 @@ "SqueezeOp", "StandardizeOp", "StashInputOp", + "StashTargetOp", "SwapInputTargetOp", "Tee", + "TransformChain", "ToTensorOp", "UnstashInputOp", + "UnstashTargetOp", "UnsqueezeOp", ] diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py index e129bad..ebded40 100644 --- a/dataflux/ops/image.py +++ b/dataflux/ops/image.py @@ -306,7 +306,9 @@ def normalize_to_uint8( return np.zeros(arr.shape, dtype=np.uint8) filled = np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) norm = (filled - lo) / (hi - lo) - return (np.clip(norm, 0.0, 1.0) * 255.0).astype(np.uint8) + # np.asarray (not .astype) so the return type is ndarray under stub + # versions where clip-arithmetic degrades to Any. + return np.asarray(np.clip(norm, 0.0, 1.0) * 255.0, dtype=np.uint8) def __call__(self, sample: Sample) -> Sample: if self.vmin is not None and self.vmax is not None and self.vmin >= self.vmax: diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index 6d321a6..1182a46 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -388,6 +388,7 @@ def __call__(self, sample: Sample) -> Sample: arr = sample.input if not isinstance(arr, np.ndarray): raise TypeError(f"ThresholdOp expects an np.ndarray on sample.input, got {type(arr).__name__}") + low_level = self.low_level high_level = self.high_level # Treat empty string (blank STRING widget left unset) as None ("disabled"). @@ -399,13 +400,25 @@ def __call__(self, sample: Sample) -> Sample: mask: Optional[np.ndarray] = None if low_level is not None: low = self._resolve(low_level, sample) - sample.meta["threshold_low"] = low - mask = _LOW_COMPARISONS[self.low_op](arr, low) + if np.isnan(low): + logger.warning( + f"ThresholdOp: resolved low_level is NaN; no values will be above the threshold. " + f"Expression was {self.low_level!r} resolved to {low!r}" + ) + else: + sample.meta["threshold_low"] = low + mask = _LOW_COMPARISONS[self.low_op](arr, low) if high_level is not None: high = self._resolve(high_level, sample) - sample.meta["threshold_high"] = high - below = _HIGH_COMPARISONS[self.high_op](arr, high) - mask = below if mask is None else (mask & below) + if np.isnan(high): + logger.warning( + f"ThresholdOp: resolved high_level is NaN; no values will be below the threshold. " + f"Expression was {self.high_level!r} resolved to {high!r}" + ) + else: + sample.meta["threshold_high"] = high + below = _HIGH_COMPARISONS[self.high_op](arr, high) + mask = below if mask is None else (mask & below) if mask is None: raise ValueError("ThresholdOp requires at least one of 'low_level' / 'high_level'") return sample._replace(input=mask) diff --git a/dataflux/ops/random_apply.py b/dataflux/ops/random_apply.py index d472aec..958ed43 100644 --- a/dataflux/ops/random_apply.py +++ b/dataflux/ops/random_apply.py @@ -45,16 +45,26 @@ class RandomApply: (identity); validated lazily on first call. probability: Gate probability in ``[0, 1]``. ``0.0`` = never apply; ``1.0`` = always apply. Defaults to ``0.5``. + random_state: Seed for the Bernoulli gate RNG. ``None`` = non-deterministic (default). """ - def __init__(self, op: Optional[object] = None, probability: float = 0.5) -> None: + def __init__( + self, + op: Optional[object] = None, + probability: float = 0.5, + random_state: Optional[int] = None, + ) -> None: self.op = op self.probability = probability + self.random_state = random_state + self._gate_rng: Optional[random.Random] = None def __call__(self, sample: Sample) -> Sample: if self.op is None: raise ValueError("RandomApply requires 'op' to be set before calling.") - if random.random() >= self.probability: + if self._gate_rng is None: + self._gate_rng = random.Random(self.random_state) + if self._gate_rng.random() >= self.probability: return sample from confluid import flow from confluid.fluid import Fluid diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py index fed7930..bbd032f 100644 --- a/dataflux/ops/stash.py +++ b/dataflux/ops/stash.py @@ -1,11 +1,15 @@ -"""Stash / unstash ``sample.input`` to / from ``metadata``. +"""Stash / unstash ``sample.input`` / ``sample.target`` to / from ``metadata``. Use ``StashInputOp(key)`` to snapshot the current ``sample.input`` under a metadata key without changing ``sample.input``. Use ``UnstashInputOp(key)`` later (e.g. inside another ``Tee`` branch) to restore that value into -``sample.input``. +``sample.input``. ``StashTargetOp`` / ``UnstashTargetOp`` are the exact +``sample.target`` counterparts — together the family is what lets a branchy +canvas graph compile to ONE sequential op-list (FluxStudio's DAG→sequential +ops-export restores the fork-point input/target between branches and +translates a Mix-style fan-in into unstashes). -``UnstashInputOp`` defaults to ``copy=True`` (deepcopy) so two branches +The ``Unstash*Op``\\ s default to ``copy=True`` (deepcopy) so two branches that both unstash the same key are independent — each gets its own array to mutate. Without the copy, an in-place op like ``ClipPercentilesOp`` in the first branch would silently corrupt the stashed value seen by the @@ -64,3 +68,50 @@ def __call__(self, sample: Sample) -> Sample: if self.copy: value = _copy.deepcopy(value) return sample._replace(input=value) + + +@configurable(category="op", group="structure") +class StashTargetOp: + """Copy ``sample.target`` into ``metadata[key]``; ``sample.target`` unchanged. + + Args: + key: Metadata key to write. + copy: When ``True``, deepcopy ``sample.target`` before stashing. + Defaults to ``False`` (cheap pointer alias) — the typical case + is that downstream ops use ``sample._replace(target=...)`` and + don't mutate the shared value in place. + """ + + def __init__(self, key: str = "", copy: bool = False) -> None: + # Lazy / zero-arg: store config only. + self.key = key + self.copy = copy + + def __call__(self, sample: Sample) -> Sample: + sample.meta[self.key] = _copy.deepcopy(sample.target) if self.copy else sample.target + return sample + + +@configurable(category="op", group="structure") +class UnstashTargetOp: + """Set ``sample.target := metadata[key]``. + + Args: + key: Metadata key to read. + copy: When ``True`` (default), deepcopy the stashed value before + assigning. This prevents two branches that unstash the same + key from corrupting each other through downstream in-place + mutations. Set ``False`` only when the caller has audited + that no downstream op mutates the value in place. + """ + + def __init__(self, key: str = "", copy: bool = True) -> None: + # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. + self.key = key + self.copy = copy + + def __call__(self, sample: Sample) -> Sample: + value = sample.meta[self.key] + if self.copy: + value = _copy.deepcopy(value) + return sample._replace(target=value) diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index 14803e4..0e2ccef 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -114,6 +114,51 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=out) +@configurable(category="op", group="torch") +class SqueezeOp: + """Remove size-1 dimensions from a ``torch.Tensor``. + + Args: + dim: Axis index to remove. When ``None`` (default), all size-1 dimensions are removed. + When specified, the dimension must have size 1; otherwise the tensor is returned unchanged + (matching ``torch.squeeze`` semantics). + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH) + + def __init__(self, dim: Optional[int] = None) -> None: + self.dim = dim + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"SqueezeOp expects a torch.Tensor, got {type(tensor).__name__}") + out = torch.squeeze(tensor) if self.dim is None else torch.squeeze(tensor, self.dim) + return sample._replace(input=out) + + +@configurable(category="op", group="torch") +class UnsqueezeOp: + """Insert a size-1 dimension at the specified position in a ``torch.Tensor``. + + Args: + dim: Axis index at which the new dimension is inserted. Default ``0``. + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH) + + def __init__(self, dim: int = 0) -> None: + self.dim = dim + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"UnsqueezeOp expects a torch.Tensor, got {type(tensor).__name__}") + return sample._replace(input=torch.unsqueeze(tensor, self.dim)) + + @configurable(category="op", group="torch") class StandardizeOp: """ diff --git a/dataflux/ops/transform_chain.py b/dataflux/ops/transform_chain.py new file mode 100644 index 0000000..81e21d5 --- /dev/null +++ b/dataflux/ops/transform_chain.py @@ -0,0 +1,84 @@ +"""``TransformChain`` — group a sequence of ops into a single named unit. + +A compose-group op (alongside ``Enable`` / ``Tee`` / ``Parallel``): +wrap an ordered list of ``Sample → Sample`` callables so they appear as +one node in FluxStudio (dynamic ``op_0``, ``op_1``, … ``DATAFLUX_OP`` +inputs instead of N wired ``DATAFLUX_SAMPLE`` connections) and one named +block in a Confluid YAML. + +Unlike ``Enable`` there is no boolean gate — the chain always fires. +Unlike ``Parallel`` there is no worker pool — ops run sequentially in the +calling thread. If any op returns ``None`` the chain stops early and +propagates ``None`` (consistent with ``FilterOp`` / ``Tee`` semantics). +""" + +from typing import List, Optional + +from confluid import configurable +from logflow import get_logger + +from dataflux.sample import Sample + +logger = get_logger(__name__) + + +@configurable(category="op", group="compose") +class TransformChain: + """Apply a fixed sequence of ops to every sample, always. + + Wrap a list of ops into one named unit so they appear as a single node + in FluxStudio (dynamic ``op_0``, ``op_1``, … ``DATAFLUX_OP`` inputs) + and one block in Confluid YAML instead of N separate connections. + + If any op in the chain returns ``None`` the remaining ops are skipped + and ``None`` is propagated (consistent with ``FilterOp`` semantics — + the sample is dropped). + + Inner ops keep full autonomy over their own randomness; ``TransformChain`` + itself is deterministic. Nest a + :class:`~dataflux.ops.random_apply.RandomApply` inside the chain to + gate individual ops stochastically. + + YAML example:: + + - !class:dataflux.ops.transform_chain.TransformChain + ops: + - !class:dataflux.ops.random_apply.RandomApply + op: !class:waivefront.torchsig.processing.AWGNOp {} + probability: 0.8 + - !class:dataflux.ops.torch.ToTensorOp {} + + Args: + ops: Ordered list of callables ``Sample -> Optional[Sample]`` applied + in sequence. Defaults to ``[]`` (identity — the chain passes + every sample through unchanged). + """ + + def __init__(self, ops: Optional[List] = None) -> None: + self.ops: List = list(ops) if ops else [] + + def __call__(self, sample: Sample) -> Optional[Sample]: + from confluid import flow + from confluid.fluid import Fluid + + current: Optional[Sample] = sample + for i, op in enumerate(self.ops): + if current is None: + return None + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + if op is None: + continue + current = op(current) + return current + + def close(self) -> None: + """Propagate close() to inner ops that own resources (e.g. SampleSinkOp).""" + for op in self.ops: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + +__all__ = ["TransformChain"] diff --git a/dataflux/typespec.py b/dataflux/typespec.py index 9c65de5..5b0cb30 100644 --- a/dataflux/typespec.py +++ b/dataflux/typespec.py @@ -256,6 +256,15 @@ def compatible(self, other: "Dim") -> bool: return False return True + def __str__(self) -> str: + if self.min is None and self.max is None: + return "any" + if self.min == self.max: + return str(self.min) + lo = str(self.min) if self.min is not None else "0" + hi = str(self.max) if self.max is not None else "∞" + return f"{lo}–{hi}" + def to_dict(self) -> Dict[str, Any]: return {"min": self.min, "max": self.max, "name": self.name} @@ -359,6 +368,42 @@ def parse( fws = frozenset({framework}) if framework else None return cls(ndim=len(dims), shape=tuple(dims), dtype=dtype, frameworks=fws, semantic=semantic) + def __str__(self) -> str: + parts: List[str] = [] + if self.frameworks: + parts.append("/".join(sorted(self.frameworks))) + if self.dtype: + parts.append(str(self.dtype)) + if self.shape is not None: + parts.append(f"shape=({', '.join(str(d) for d in self.shape)})") + elif self.ndim is not None: + parts.append(f"rank-{self.ndim}") + return f"array[{', '.join(parts)}]" if parts else "array" + + def explain_mismatch(self, producer: "ArrayType") -> List[str]: + """Return human-readable reasons why this consumer does not accept *producer*.""" + reasons: List[str] = [] + if self.frameworks is not None: + if producer.frameworks is None: + reasons.append(f"framework unknown in upstream (op requires {'/'.join(sorted(self.frameworks))})") + elif not (producer.frameworks <= self.frameworks): + exp = "/".join(sorted(self.frameworks)) + got = "/".join(sorted(producer.frameworks)) + reasons.append(f"framework mismatch: op requires {exp}, upstream produces {got}") + if self.ndim is not None and producer.ndim is not None and producer.ndim != self.ndim: + reasons.append(f"rank mismatch: op requires rank {self.ndim}, upstream has rank {producer.ndim}") + if self.dtype is not None and producer.dtype is not None: + if not _dtype_accepts(self.dtype, producer.dtype): + reasons.append(f"dtype mismatch: op requires {self.dtype}, upstream produces {producer.dtype}") + elif self.dtype is not None and producer.dtype is None: + reasons.append(f"dtype unknown in upstream (op requires {self.dtype})") + if self.shape is not None and producer.shape is not None and len(self.shape) == len(producer.shape): + for i, (cdim, pdim) in enumerate(zip(self.shape, producer.shape)): + if not cdim.accepts(pdim): + name = f" ({cdim.name})" if cdim.name else "" + reasons.append(f"axis {i}{name}: op requires size {cdim}, upstream has size {pdim}") + return reasons + def to_dict(self) -> Dict[str, Any]: return { "kind": "array", @@ -447,6 +492,19 @@ def compatible(self, producer: "SampleType") -> bool: self.target, producer.target, permissive=True ) + def explain_mismatch(self, producer: "SampleType") -> str: + """Return a human-readable explanation of why ``self.accepts(producer)`` is False. + + Walks each slot (input / target) and collects all failing conditions — framework, + rank, dtype, and per-axis shape — then returns them as a single comma-separated + sentence so the caller can embed it directly in an error message. + """ + reasons: List[str] = [] + for slot, cspec, pspec in (("input", self.input, producer.input), ("target", self.target, producer.target)): + slot_reasons = _explain_slot_mismatch(cspec, pspec) + reasons.extend(f"{slot} {r}" for r in slot_reasons) + return "; ".join(reasons) if reasons else "incompatible types (no specific reason derived)" + def to_dict(self) -> Dict[str, Any]: return {"kind": "sample", "input": type_to_dict(self.input), "target": type_to_dict(self.target)} @@ -502,6 +560,26 @@ def compatible(consumer: TypeSpec, producer: TypeSpec) -> bool: return _accepts(consumer, producer, permissive=True) +def _explain_slot_mismatch(consumer: TypeSpec, producer: TypeSpec) -> List[str]: + """Return human-readable reasons why *consumer* does not strictly accept *producer* for one slot.""" + if isinstance(consumer, AnyType) or isinstance(producer, AnyType): + return [] + if isinstance(consumer, ArrayType) and isinstance(producer, ArrayType): + return consumer.explain_mismatch(producer) + if isinstance(consumer, UnionType): + # None of the union members accepted — collect reasons from the closest member. + all_reasons = [_explain_slot_mismatch(m, producer) for m in consumer.members] + # Pick the member with fewest (most specific) reasons as the most helpful. + best = min(all_reasons, key=lambda r: (len(r) == 0, len(r)), default=[]) + return best if best else ["incompatible union types"] + if isinstance(consumer, PythonType) and isinstance(producer, PythonType): + if consumer.qualname != producer.qualname: + return [f"type mismatch: op requires {consumer.qualname}, upstream produces {producer.qualname}"] + return [] + # Framework/kind-level mismatch (e.g. ArrayType vs PythonType). + return [f"kind mismatch: op requires {type(consumer).__name__}, upstream produces {type(producer).__name__}"] + + def _accepts(consumer: TypeSpec, producer: TypeSpec, *, permissive: bool) -> bool: if isinstance(consumer, AnyType): return True diff --git a/pyproject.toml b/pyproject.toml index f103318..bbdf4d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dataflux-ops-parallel = "dataflux.ops.parallel" dataflux-ops-tee = "dataflux.ops.tee" dataflux-ops-enable = "dataflux.ops.enable" dataflux-ops-random-apply = "dataflux.ops.random_apply" +dataflux-ops-transform-chain = "dataflux.ops.transform_chain" dataflux-ops-sink = "dataflux.ops.sink" dataflux-ops-stash = "dataflux.ops.stash" dataflux-ops-numpy = "dataflux.ops.numpy" diff --git a/tests/test_categories.py b/tests/test_categories.py index fb1440e..e82fd76 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -16,6 +16,7 @@ from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp from dataflux.ops.parallel import Parallel from dataflux.ops.sink import SampleSinkOp +from dataflux.ops.stash import StashTargetOp, UnstashTargetOp from dataflux.ops.target import ( CocoToTorchVisionDetectionOp, DecodeTargetOp, @@ -25,6 +26,7 @@ ) from dataflux.ops.tee import Tee from dataflux.ops.torch import ToTensorOp +from dataflux.ops.transform_chain import TransformChain from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource @@ -70,6 +72,7 @@ def test_op_classes_tagged() -> None: assert ThresholdOp.__confluid_category__ == "op" assert Tee.__confluid_category__ == "op" assert Enable.__confluid_category__ == "op" + assert TransformChain.__confluid_category__ == "op" assert SampleSinkOp.__confluid_category__ == "op" assert MetadataToTargetOp.__confluid_category__ == "op" assert EncodeTargetOp.__confluid_category__ == "op" @@ -88,6 +91,8 @@ def test_op_group_tags() -> None: assert ThresholdOp.__confluid_group__ == "numpy" assert ToTensorOp.__confluid_group__ == "torch" assert CopyInputOp.__confluid_group__ == "structure" + assert StashTargetOp.__confluid_group__ == "structure" + assert UnstashTargetOp.__confluid_group__ == "structure" assert MetadataToTargetOp.__confluid_group__ == "structure" assert EncodeTargetOp.__confluid_group__ == "structure" assert DecodeTargetOp.__confluid_group__ == "structure" @@ -96,6 +101,7 @@ def test_op_group_tags() -> None: assert Tee.__confluid_group__ == "compose" assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" + assert TransformChain.__confluid_group__ == "compose" assert ConvertToImageOp.__confluid_group__ == "image" assert NormalizeToUint8Op.__confluid_group__ == "image" assert SampleSinkOp.__confluid_group__ == "sink" @@ -128,6 +134,7 @@ def test_categories_enumerable_via_registry() -> None: "DecodeTargetOp", "CocoToTorchVisionDetectionOp", "MasksToDetectionBoxesOp", + "TransformChain", } <= registry.list_classes(category="op") @@ -136,7 +143,7 @@ def test_groups_enumerable_via_registry() -> None: registry = get_registry() assert {"RescaleOp", "StandardizeOp", "ThresholdOp"} <= registry.list_classes(group="numpy") assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") - assert {"Tee", "Parallel", "Enable"} <= registry.list_classes(group="compose") + assert {"Tee", "Parallel", "Enable", "TransformChain"} <= registry.list_classes(group="compose") assert {"SampleSinkOp"} <= registry.list_classes(group="sink") assert { "MetadataToTargetOp", diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index b494f28..2cd4456 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -18,6 +18,7 @@ from dataflux.ops.tee import Tee from dataflux.ops.torch import StandardizeOp as TorchStandardizeOp from dataflux.ops.torch import ToTensorOp +from dataflux.ops.transform_chain import TransformChain from dataflux.sources import HuggingFaceSource _NODE_CLASSES = [ @@ -35,6 +36,7 @@ MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp, + TransformChain, ] diff --git a/tests/test_ops.py b/tests/test_ops.py index cd4d6ff..1c8d9ac 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -13,12 +13,16 @@ CopySampleOp, CopyTargetOp, RescaleOp, + SqueezeOp, StandardizeOp, StashInputOp, + StashTargetOp, SwapInputTargetOp, Tee, ToTensorOp, + UnsqueezeOp, UnstashInputOp, + UnstashTargetOp, ) from dataflux.ops import numpy as np_ops from dataflux.sample import Sample @@ -594,6 +598,53 @@ def test_two_unstashes_with_in_place_mutation_dont_corrupt(self) -> None: np.testing.assert_array_equal(b.input, [1.0, 2.0, 3.0]) +# --------------------------------------------------------------------------- +# StashTargetOp / UnstashTargetOp +# --------------------------------------------------------------------------- + + +class TestStashUnstashTarget: + def test_stash_target_aliases_by_default(self) -> None: + arr = np.array([1.0, 2.0]) + sample = Sample(input=None, target=arr, metadata={}) + out = StashTargetOp(key="snap")(sample) + assert out.meta["snap"] is arr + assert out.target is arr + + def test_stash_target_with_copy_deepcopies(self) -> None: + arr = np.array([1.0, 2.0]) + sample = Sample(input=None, target=arr, metadata={}) + out = StashTargetOp(key="snap", copy=True)(sample) + assert out.meta["snap"] is not arr + np.testing.assert_array_equal(out.meta["snap"], arr) + + def test_unstash_target_default_copies_to_isolate_branches(self) -> None: + arr = np.array([1.0, 2.0]) + sample = Sample(input=None, target=None, metadata={"snap": arr}) + out = UnstashTargetOp(key="snap")(sample) + assert out.target is not arr + np.testing.assert_array_equal(out.target, arr) + + def test_unstash_target_no_copy_aliases(self) -> None: + arr = np.array([1.0, 2.0]) + sample = Sample(input=None, target=None, metadata={"snap": arr}) + out = UnstashTargetOp(key="snap", copy=False)(sample) + assert out.target is arr + + def test_unstash_target_missing_key_raises_lazily(self) -> None: + sample = Sample(input=None, target=None, metadata={}) + with pytest.raises(KeyError): + UnstashTargetOp(key="nope")(sample) + + def test_stash_restore_round_trip_preserves_fork_target(self) -> None: + """The DAG→sequential pattern: snapshot at a fork, restore after a branch replaced it.""" + sample = Sample(input=None, target="fork-target", metadata={}) + stashed = StashTargetOp(key="fork")(sample) + branched = stashed._replace(target="branch-target") + restored = UnstashTargetOp(key="fork")(branched) + assert restored.target == "fork-target" + + # --------------------------------------------------------------------------- # numpy.resolve_expression # --------------------------------------------------------------------------- @@ -816,3 +867,180 @@ def test_validation_rejects_bad_connectivity(self) -> None: op = np_ops.ConnectedComponentsOp(connectivity=6) with pytest.raises(ValueError, match="connectivity must be 4 or 8"): op(Sample(input=np.zeros((2, 2), dtype=bool))) + + +# --------------------------------------------------------------------------- +# Torch SqueezeOp +# --------------------------------------------------------------------------- + + +class TestTorchSqueezeOp: + """Tests for torch SqueezeOp.""" + + def test_squeeze_all_size1_dims(self) -> None: + tensor = torch.zeros(1, 3, 1, 4) + result = SqueezeOp()(Sample(input=tensor)) + assert result.input.shape == (3, 4) + + def test_squeeze_specific_dim(self) -> None: + tensor = torch.zeros(1, 3, 4) + result = SqueezeOp(dim=0)(Sample(input=tensor)) + assert result.input.shape == (3, 4) + + def test_squeeze_non_unit_dim_is_noop(self) -> None: + # torch.squeeze leaves non-size-1 dims unchanged + tensor = torch.zeros(2, 3) + result = SqueezeOp(dim=0)(Sample(input=tensor)) + assert result.input.shape == (2, 3) + + def test_preserves_target_and_metadata(self) -> None: + tensor = torch.zeros(1, 4) + result = SqueezeOp()(Sample(input=tensor, target=7, metadata={"k": "v"})) + assert result.target == 7 + assert result.meta == {"k": "v"} + + def test_raises_on_non_tensor(self) -> None: + with pytest.raises(TypeError, match="SqueezeOp expects a torch.Tensor"): + SqueezeOp()(Sample(input=np.zeros((1, 3)))) + + def test_zero_arg_construction(self) -> None: + op = SqueezeOp() + assert op.dim is None + + def test_negative_dim(self) -> None: + tensor = torch.zeros(3, 1) + result = SqueezeOp(dim=-1)(Sample(input=tensor)) + assert result.input.shape == (3,) + + +# --------------------------------------------------------------------------- +# Torch UnsqueezeOp +# --------------------------------------------------------------------------- + + +class TestTorchUnsqueezeOp: + """Tests for torch UnsqueezeOp.""" + + def test_unsqueeze_at_dim0(self) -> None: + tensor = torch.zeros(3, 4) + result = UnsqueezeOp(dim=0)(Sample(input=tensor)) + assert result.input.shape == (1, 3, 4) + + def test_unsqueeze_at_dim1(self) -> None: + tensor = torch.zeros(3, 4) + result = UnsqueezeOp(dim=1)(Sample(input=tensor)) + assert result.input.shape == (3, 1, 4) + + def test_unsqueeze_at_last_dim(self) -> None: + tensor = torch.zeros(3, 4) + result = UnsqueezeOp(dim=-1)(Sample(input=tensor)) + assert result.input.shape == (3, 4, 1) + + def test_default_dim_is_zero(self) -> None: + tensor = torch.zeros(5) + result = UnsqueezeOp()(Sample(input=tensor)) + assert result.input.shape == (1, 5) + + def test_preserves_target_and_metadata(self) -> None: + tensor = torch.zeros(4) + result = UnsqueezeOp()(Sample(input=tensor, target=2, metadata={"x": 1})) + assert result.target == 2 + assert result.meta == {"x": 1} + + def test_raises_on_non_tensor(self) -> None: + with pytest.raises(TypeError, match="UnsqueezeOp expects a torch.Tensor"): + UnsqueezeOp()(Sample(input=np.zeros(3))) + + def test_roundtrip_squeeze_unsqueeze(self) -> None: + tensor = torch.zeros(3, 4) + squeezed = UnsqueezeOp(dim=0)(Sample(input=tensor)) + restored = SqueezeOp(dim=0)(squeezed) + assert restored.input.shape == tensor.shape + + +# --------------------------------------------------------------------------- +# Numpy SqueezeOp +# --------------------------------------------------------------------------- + + +class TestNumpySqueezeOp: + """Tests for numpy SqueezeOp.""" + + def test_squeeze_all_size1_axes(self) -> None: + arr = np.zeros((1, 3, 1, 4)) + result = np_ops.SqueezeOp()(Sample(input=arr)) + assert result.input.shape == (3, 4) + + def test_squeeze_specific_axis(self) -> None: + arr = np.zeros((1, 3, 4)) + result = np_ops.SqueezeOp(axis=0)(Sample(input=arr)) + assert result.input.shape == (3, 4) + + def test_squeeze_non_unit_axis_raises(self) -> None: + arr = np.zeros((2, 3)) + with pytest.raises(ValueError): + np_ops.SqueezeOp(axis=0)(Sample(input=arr)) + + def test_preserves_target_and_metadata(self) -> None: + arr = np.zeros((1, 4)) + result = np_ops.SqueezeOp()(Sample(input=arr, target=7, metadata={"k": "v"})) + assert result.target == 7 + assert result.meta == {"k": "v"} + + def test_raises_on_non_ndarray(self) -> None: + with pytest.raises(TypeError, match="SqueezeOp expects an np.ndarray"): + np_ops.SqueezeOp()(Sample(input=torch.zeros(1, 3))) + + def test_zero_arg_construction(self) -> None: + op = np_ops.SqueezeOp() + assert op.axis is None + + def test_negative_axis(self) -> None: + arr = np.zeros((3, 1)) + result = np_ops.SqueezeOp(axis=-1)(Sample(input=arr)) + assert result.input.shape == (3,) + + +# --------------------------------------------------------------------------- +# Numpy UnsqueezeOp +# --------------------------------------------------------------------------- + + +class TestNumpyUnsqueezeOp: + """Tests for numpy UnsqueezeOp.""" + + def test_unsqueeze_at_axis0(self) -> None: + arr = np.zeros((3, 4)) + result = np_ops.UnsqueezeOp(axis=0)(Sample(input=arr)) + assert result.input.shape == (1, 3, 4) + + def test_unsqueeze_at_axis1(self) -> None: + arr = np.zeros((3, 4)) + result = np_ops.UnsqueezeOp(axis=1)(Sample(input=arr)) + assert result.input.shape == (3, 1, 4) + + def test_unsqueeze_at_last_axis(self) -> None: + arr = np.zeros((3, 4)) + result = np_ops.UnsqueezeOp(axis=-1)(Sample(input=arr)) + assert result.input.shape == (3, 4, 1) + + def test_default_axis_is_zero(self) -> None: + arr = np.zeros(5) + result = np_ops.UnsqueezeOp()(Sample(input=arr)) + assert result.input.shape == (1, 5) + + def test_preserves_target_and_metadata(self) -> None: + arr = np.zeros(4) + result = np_ops.UnsqueezeOp()(Sample(input=arr, target=2, metadata={"x": 1})) + assert result.target == 2 + assert result.meta == {"x": 1} + + def test_raises_on_non_ndarray(self) -> None: + with pytest.raises(TypeError, match="UnsqueezeOp expects an np.ndarray"): + np_ops.UnsqueezeOp()(Sample(input=torch.zeros(3))) + + def test_roundtrip_squeeze_unsqueeze(self) -> None: + arr = np.zeros((3, 4)) + unsqueezed = np_ops.UnsqueezeOp(axis=0)(Sample(input=arr)) + restored = np_ops.SqueezeOp(axis=0)(unsqueezed) + assert restored.input.shape == arr.shape diff --git a/tests/test_random_apply.py b/tests/test_random_apply.py index 5d64528..d3d3c8c 100644 --- a/tests/test_random_apply.py +++ b/tests/test_random_apply.py @@ -72,3 +72,43 @@ def test_sample_passes_through_unchanged_when_skipped() -> None: s = _s(42) op = RandomApply(op=_BumpOp(), probability=0.0) assert op(s) is s + + +# --------------------------------------------------------------------------- +# random_state / reproducibility +# --------------------------------------------------------------------------- + + +def test_random_state_stored_on_instance() -> None: + op = RandomApply(op=_BumpOp(), probability=0.5, random_state=42) + assert op.random_state == 42 + + +def test_random_state_none_is_default() -> None: + op = RandomApply(op=_BumpOp()) + assert op.random_state is None + + +def test_gate_reproducible_with_seed() -> None: + """Two RandomApply instances with the same seed must make identical gate decisions.""" + s = _s(0) + op_a = RandomApply(op=_BumpOp(), probability=0.5, random_state=7) + op_b = RandomApply(op=_BumpOp(), probability=0.5, random_state=7) + results_a = [op_a(s).input for _ in range(30)] + results_b = [op_b(s).input for _ in range(30)] + assert results_a == results_b + + +def test_gate_different_seeds_produce_different_sequences() -> None: + s = _s(0) + op_a = RandomApply(op=_BumpOp(), probability=0.5, random_state=1) + op_b = RandomApply(op=_BumpOp(), probability=0.5, random_state=2) + results_a = [op_a(s).input for _ in range(50)] + results_b = [op_b(s).input for _ in range(50)] + assert results_a != results_b + + +def test_zero_arg_construction_with_random_state_none() -> None: + op = RandomApply() + assert op.random_state is None + assert op._gate_rng is None # lazily initialized on first call diff --git a/tests/test_transform_chain.py b/tests/test_transform_chain.py new file mode 100644 index 0000000..e95dd0a --- /dev/null +++ b/tests/test_transform_chain.py @@ -0,0 +1,157 @@ +"""Tests for :class:`dataflux.ops.transform_chain.TransformChain`.""" + +from typing import List, Optional + +from dataflux.ops.transform_chain import TransformChain +from dataflux.sample import Sample + + +def _s(v: int = 0) -> Sample: + return Sample(input=v, target=None, metadata={}) + + +class _AddOp: + """Increment sample.input by a fixed delta.""" + + def __init__(self, delta: int = 1) -> None: + self.delta = delta + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + self.delta) + + +class _TagOp: + """Append a string tag to metadata['tags'].""" + + def __init__(self, tag: str) -> None: + self.tag = tag + + def __call__(self, sample: Sample) -> Sample: + new_meta = dict(sample.meta) + new_meta["tags"] = new_meta.get("tags", []) + [self.tag] + return sample._replace(metadata=new_meta) + + +class _DropOp: + """Always returns None — simulates a filter op.""" + + def __call__(self, sample: Sample) -> Optional[Sample]: + return None + + +class _ClosableOp: + def __init__(self, name: str, log: List[str]) -> None: + self.name = name + self.log = log + + def __call__(self, sample: Sample) -> Sample: + return sample + + def close(self) -> None: + self.log.append(self.name) + + +# --------------------------------------------------------------------------- +# Core behaviour +# --------------------------------------------------------------------------- + + +def test_zero_arg_construction() -> None: + """TransformChain() must construct with no arguments (lazy convention).""" + chain = TransformChain() + assert chain.ops == [] + + +def test_empty_chain_is_identity() -> None: + """An empty TransformChain passes the sample through unchanged.""" + chain = TransformChain() + s = _s(42) + out = chain(s) + assert out is s + + +def test_happy_path_multiple_ops_applied_in_order() -> None: + """Ops fire left-to-right; each op sees the output of the previous one.""" + chain = TransformChain(ops=[_AddOp(1), _AddOp(2), _AddOp(3)]) + out = chain(_s(0)) + assert out is not None + assert out.input == 6 # 0 + 1 + 2 + 3 + + +def test_ops_applied_in_declared_order_via_metadata_tags() -> None: + """Ordering is visible: tags accumulate in declaration order.""" + chain = TransformChain(ops=[_TagOp("a"), _TagOp("b"), _TagOp("c")]) + out = chain(_s()) + assert out is not None + assert out.meta["tags"] == ["a", "b", "c"] + + +def test_none_propagation_stops_chain_early() -> None: + """If any op returns None the chain stops and propagates None.""" + called: List[str] = [] + + class _RecordOp: + def __init__(self, tag: str) -> None: + self.tag = tag + + def __call__(self, sample: Sample) -> Sample: + called.append(self.tag) + return sample + + chain = TransformChain(ops=[_RecordOp("before"), _DropOp(), _RecordOp("after")]) + out = chain(_s()) + assert out is None + assert called == ["before"] # "after" must NOT have fired + + +def test_none_propagation_from_first_op() -> None: + """None returned by the very first op also short-circuits the chain.""" + chain = TransformChain(ops=[_DropOp(), _AddOp(99)]) + out = chain(_s(0)) + assert out is None + + +def test_fluid_resolution_lazy_and_cached() -> None: + """Confluid Fluid markers inside ops are resolved on first call and cached.""" + from confluid import configurable + from confluid.fluid import Class, Fluid + + @configurable + class _Inner: + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + 10) + + fluid_op = Class(_Inner) + chain = TransformChain(ops=[fluid_op]) + + out1 = chain(_s(5)) + assert out1 is not None + assert out1.input == 15 + + # Slot must now hold the resolved instance, not a Fluid. + assert not isinstance(chain.ops[0], Fluid) + + out2 = chain(_s(5)) + assert out2 is not None + assert out2.input == 15 + + +def test_close_propagates_to_all_inner_ops() -> None: + """close() forwards to every inner op that implements it.""" + log: List[str] = [] + chain = TransformChain(ops=[_ClosableOp("x", log), _ClosableOp("y", log)]) + chain.close() + assert log == ["x", "y"] + + +def test_close_on_empty_chain_is_safe() -> None: + """close() on an empty chain must not raise.""" + TransformChain().close() # must not raise + + +def test_close_skips_ops_without_close_method() -> None: + """close() only calls close on ops that have it — no AttributeError on plain callables.""" + log: List[str] = [] + chain = TransformChain(ops=[_AddOp(1), _ClosableOp("z", log)]) + chain.close() + assert log == ["z"] diff --git a/tests/test_typespec.py b/tests/test_typespec.py index 8fdf736..48ed442 100644 --- a/tests/test_typespec.py +++ b/tests/test_typespec.py @@ -1,6 +1,6 @@ """Exhaustive tests for the dataflux type-spec system (matching, inference, JSON, HF bridge).""" -from typing import Any, List, Tuple, cast, get_args +from typing import Any, Iterable, List, Tuple, cast, get_args import numpy as np import pytest @@ -539,3 +539,120 @@ def test_dataflux_op_spec_conformance() -> None: assert op.PRODUCES.accepts(infer_sample_type(out)), f"{name}: PRODUCES rejects its real output" # specs are JSON round-trippable assert SampleType.from_dict(op.PRODUCES.to_dict()) == op.PRODUCES + + +# -------------------------------------------------------------------------------------------------- +# Dim / ArrayType __str__ and SampleType.explain_mismatch +# -------------------------------------------------------------------------------------------------- + + +class TestDimStr: + def test_unbounded(self) -> None: + assert str(Dim.any()) == "any" + + def test_exact(self) -> None: + assert str(Dim.exact(3)) == "3" + + def test_range(self) -> None: + assert str(Dim(min=2, max=8)) == "2–8" + + def test_open_upper(self) -> None: + assert str(Dim(min=1, max=None)) == "1–∞" + + def test_open_lower(self) -> None: + assert str(Dim(min=None, max=4)) == "0–4" + + +class TestArrayTypeStr: + def test_framework_only(self) -> None: + assert str(ArrayType(frameworks={"torch"})) == "array[torch]" + + def test_framework_and_dtype(self) -> None: + s = str(ArrayType(frameworks={"numpy"}, dtype="float32")) + assert "numpy" in s and "float32" in s + + def test_shape_shown(self) -> None: + s = str(ArrayType(shape=(Dim.exact(3), Dim.any()))) + assert "shape=(3, any)" in s + + def test_rank_without_shape(self) -> None: + s = str(ArrayType(ndim=2)) + assert "rank-2" in s + + def test_empty_is_just_array(self) -> None: + assert str(ArrayType()) == "array" + + +class TestExplainMismatch: + """SampleType.explain_mismatch produces actionable human-readable reasons.""" + + def _make( + self, + frameworks: Iterable[Framework] | None = None, + dtype: Dtype | None = None, + ndim: int | None = None, + ) -> SampleType: + return SampleType( + input=ArrayType( + frameworks=frozenset(cast(Iterable[Framework], frameworks)) if frameworks is not None else None, + dtype=cast(Dtype | None, dtype), + ndim=ndim, + ) + ) + + def test_framework_mismatch_names_both_sides(self) -> None: + consumer = self._make(frameworks={"numpy"}) + producer = self._make(frameworks={"torch"}) + msg = consumer.explain_mismatch(producer) + assert "numpy" in msg and "torch" in msg + assert "framework" in msg + + def test_dtype_mismatch_names_both_dtypes(self) -> None: + consumer = self._make(dtype="float32") + producer = self._make(dtype="complex64") + msg = consumer.explain_mismatch(producer) + assert "float32" in msg and "complex64" in msg + assert "dtype" in msg + + def test_ndim_mismatch_names_both_ranks(self) -> None: + consumer = self._make(ndim=2) + producer = self._make(ndim=3) + msg = consumer.explain_mismatch(producer) + assert "2" in msg and "3" in msg + assert "rank" in msg + + def test_shape_axis_mismatch_names_axis_and_sizes(self) -> None: + consumer = SampleType(input=ArrayType(shape=(Dim.exact(1), Dim.any()))) + producer = SampleType(input=ArrayType(shape=(Dim.exact(2), Dim.any()))) + msg = consumer.explain_mismatch(producer) + assert "axis 0" in msg + assert "1" in msg and "2" in msg + + def test_no_reasons_when_accepts(self) -> None: + consumer = self._make(frameworks={"torch"}) + producer = self._make(frameworks={"torch"}) + # accepts() is True — explain_mismatch should return empty/fallback + msg = consumer.explain_mismatch(producer) + assert msg # always returns a string + + def test_python_type_mismatch(self) -> None: + consumer = SampleType(input=PythonType("PIL.Image.Image")) + producer = SampleType(input=PythonType("dict")) + msg = consumer.explain_mismatch(producer) + assert "PIL.Image.Image" in msg and "dict" in msg + + def test_original_error_scenario(self) -> None: + """The exact case from the bug report: numpy op, torch upstream.""" + consumer = SampleType(input=ArrayType(frameworks=frozenset({"numpy"}))) + producer = SampleType( + input=ArrayType( + ndim=2, + shape=(Dim.exact(1), Dim.exact(1228800)), + dtype="complex64", + frameworks=frozenset({"torch"}), + ) + ) + msg = consumer.explain_mismatch(producer) + assert "numpy" in msg + assert "torch" in msg + assert "framework" in msg From c848e1f2321fceea9d351f884a16ad12232264fc Mon Sep 17 00:00:00 2001 From: gertbehi Date: Thu, 11 Jun 2026 22:47:55 +0200 Subject: [PATCH 014/102] feat: add ConfigureOp and FormulaOp to public ops surface; fix mypy union-attr ConfigureOp (per-sample parameter injection) and FormulaOp (restricted math formula over sample.input) are now exported from dataflux.ops and covered by tests/test_ops.py + test_categories.py. ConfigureOp.__call__ now tracks current: Sample (not Optional) so mypy no longer flags union-attr on .input after the ops loop. --- .github/workflows/ci.yml | 36 ------------ AGENTS.md | 2 +- dataflux/ops/__init__.py | 6 ++ dataflux/ops/configure.py | 97 +++++++++++++++++++++++++++++++ dataflux/ops/formula.py | 47 +++++++++++++++ pyproject.toml | 4 ++ tests/test_categories.py | 4 ++ tests/test_ops.py | 116 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 275 insertions(+), 37 deletions(-) create mode 100644 dataflux/ops/configure.py create mode 100644 dataflux/ops/formula.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 642d02c..a5573dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,15 +18,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -56,15 +47,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -99,15 +81,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -137,15 +110,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: diff --git a/AGENTS.md b/AGENTS.md index 9ff65b8..386d634 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index 008ad59..af7e5a9 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -11,6 +11,8 @@ - dataflux.ops.parallel: Parallel (worker-pool sub-pipeline) - dataflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - dataflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) + - dataflux.ops.configure: ConfigureOp (per-sample parameter injection — the helios Configure pattern) + - dataflux.ops.formula: FormulaOp (math formula over sample.input — the Math node's op form) - dataflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - dataflux.ops.transform_chain: TransformChain (sequential op-chain grouping) - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp @@ -22,8 +24,10 @@ swap / stash / target utilities are field-agnostic. """ +from dataflux.ops.configure import ConfigureOp from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp from dataflux.ops.enable import Enable +from dataflux.ops.formula import FormulaOp from dataflux.ops.parallel import Parallel from dataflux.ops.random_apply import RandomApply from dataflux.ops.sink import SampleSinkOp @@ -41,12 +45,14 @@ from dataflux.ops.transform_chain import TransformChain __all__ = [ + "ConfigureOp", "CopyInputOp", "CopyMetadataOp", "CopySampleOp", "CopyTargetOp", "DecodeTargetOp", "Enable", + "FormulaOp", "EncodeTargetOp", "MetadataToTargetOp", "CocoToTorchVisionDetectionOp", diff --git a/dataflux/ops/configure.py b/dataflux/ops/configure.py new file mode 100644 index 0000000..bb6bcd6 --- /dev/null +++ b/dataflux/ops/configure.py @@ -0,0 +1,97 @@ +"""``ConfigureOp`` — per-sample parameter injection (the helios ``Configure`` pattern). + +Some op parameters are only known per sample (a threshold derived from the sample's own +max, a crop length derived from its duration). ``ConfigureOp`` is the taidal port of the +legacy helios ``Configure`` transform (``Split`` → ``ToMetadata`` → ``Config``): a +``compute`` op-chain derives the value FROM the sample, the value is written to +``metadata[key]`` (traceability) and injected as a constructor attribute of the ``target`` +op (post-construction configuration — the confluid paradigm), then ``target`` is applied +to the ORIGINAL sample. + +Modality-neutral — it threads any ``Sample`` through any ops — so it lives in core +dataflux (compose group, alongside ``Tee`` / ``Enable`` / ``RandomApply``). +""" + +from typing import Any, List, Optional, cast + +from confluid import configurable, flow +from confluid.fluid import Fluid + +from dataflux.sample import Sample + + +@configurable(category="op", group="compose") +class ConfigureOp: + """Compute a value from the sample and inject it as a parameter of a target op. + + The ``ops`` chain runs on the incoming sample as a SIDE branch — its input/target + transformations are discarded (the original sample continues), while metadata writes + survive (the shared metadata-bus convention). The final ``sample.input`` of that chain + becomes the VALUE: it is written to ``metadata[key]`` and set as the ``param`` + attribute of ``target``, then ``target`` is applied to the original sample. + + Confluid ``!class:`` / ``!lazy:`` markers in ``ops`` / ``target`` are flowed lazily at + first call (like ``Tee``), so a ``ConfigureOp()`` built from YAML costs nothing. + + YAML — a per-sample threshold (the helios ``Configure(TimeInSamples, CropToSize)`` + shape, here deriving ``ThresholdOp.low_level`` from the sample's own statistics): + + .. code-block:: yaml + + - !class:dataflux.ops.configure.ConfigureOp + ops: + - !class:dataflux.ops.numpy.MaxOp {} + target: !class:dataflux.ops.numpy.ThresholdOp + low_op: ">=" + param: low_level + + Args: + ops: Value-computing op-chain; the chain's final ``sample.input`` is injected. Empty = the incoming input. + target: The op to configure and apply; required at call time, validated lazily. + param: Target attribute name to set with the computed value (e.g. ``low_level``). + key: Metadata key the value is also written to. Blank (default) = ``param``. + """ + + def __init__( + self, + ops: Optional[List[Any]] = None, + target: Optional[object] = None, + param: str = "", + key: str = "", + ) -> None: + # Lazy / zero-arg: store config only; target/param are validated at first call. + self.ops = list(ops) if ops else [] + self.target = target + self.param = str(param) + self.key = str(key) + + def __call__(self, sample: Sample) -> Optional[Sample]: + if self.target is None: + raise ValueError("ConfigureOp: a 'target' op is required") + if not self.param: + raise ValueError("ConfigureOp: 'param' (the target attribute to set) is required") + if isinstance(self.target, Fluid): + self.target = flow(self.target) + current: Sample = sample + for i, op in enumerate(self.ops): + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + if op is None: + continue + result = op(current) + if result is None: + return None # the compute chain filtered the sample (FilterOp semantics) + current = result + value = current.input + sample.meta[self.key or self.param] = value + target = cast(Any, self.target) + setattr(target, self.param, value) + return cast(Optional[Sample], target(sample)) + + def close(self) -> None: + """Propagate close() to inner ops that own resources.""" + for op in [*self.ops, self.target]: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() diff --git a/dataflux/ops/formula.py b/dataflux/ops/formula.py new file mode 100644 index 0000000..7fa535d --- /dev/null +++ b/dataflux/ops/formula.py @@ -0,0 +1,47 @@ +"""``FormulaOp`` — evaluate a math formula over ``sample.input``. + +The op-form of FluxStudio's canvas *Math* node: a restricted Python expression over one +named variable bound to the incoming ``sample.input`` (plus the stdlib ``math`` namespace +and the scalar helpers ``abs``/``min``/``max``/``round``/``pow`` — no builtins, so +``__import__``/``open``/``exec`` are unavailable). Its main consumer is the ops-export's +value-chain compilation: an on-canvas ``… → Extract → Math → widget`` wire becomes +``ConfigureOp(ops=[…, FormulaOp(formula)], target=…, param=…)``, so the per-sample value +survives serialization. +""" + +import math as _math +from typing import Any, Dict + +from confluid import configurable + +from dataflux.sample import Sample + +# Every public ``math`` symbol + the scalar built-in helpers, mirroring the canvas Math +# node's namespace. The bound variable shadows same-named constants (e.g. ``e``). +_FORMULA_NAMESPACE: Dict[str, Any] = {k: getattr(_math, k) for k in dir(_math) if not k.startswith("_")} +_FORMULA_NAMESPACE.update({"abs": abs, "min": min, "max": max, "round": round, "pow": pow}) + + +@configurable(category="op", group="compose") +class FormulaOp: + """Replace ``sample.input`` with ``formula`` evaluated over it. + + Args: + formula: Expression over ``var`` (e.g. ``"a * 0.2"``); ``math.*`` + ``abs``/``min``/``max``/``round`` allowed. + var: Variable name the incoming ``sample.input`` binds to. Defaults to ``a``. + """ + + def __init__(self, formula: str = "a", var: str = "a") -> None: + # Lazy / zero-arg: store config only; the formula is validated at first call. + self.formula = str(formula) + self.var = str(var) + + def __call__(self, sample: Sample) -> Sample: + if not self.formula.strip(): + raise ValueError("FormulaOp: 'formula' must be a non-empty expression") + namespace = {**_FORMULA_NAMESPACE, self.var: sample.input} + try: + value = eval(self.formula, {"__builtins__": {}}, namespace) # noqa: S307 - restricted namespace + except Exception as exc: + raise ValueError(f"FormulaOp: formula {self.formula!r} failed: {exc}") from exc + return sample._replace(input=value) diff --git a/pyproject.toml b/pyproject.toml index bbdf4d2..1ee0a54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,10 @@ dataflux-ops-parallel = "dataflux.ops.parallel" dataflux-ops-tee = "dataflux.ops.tee" dataflux-ops-enable = "dataflux.ops.enable" dataflux-ops-random-apply = "dataflux.ops.random_apply" +# ConfigureOp (per-sample parameter injection — the helios Configure pattern); entry-point +# changes need an editable reinstall before FluxStudio/navigaitor discovery sees the module. +dataflux-ops-configure = "dataflux.ops.configure" +dataflux-ops-formula = "dataflux.ops.formula" dataflux-ops-transform-chain = "dataflux.ops.transform_chain" dataflux-ops-sink = "dataflux.ops.sink" dataflux-ops-stash = "dataflux.ops.stash" diff --git a/tests/test_categories.py b/tests/test_categories.py index e82fd76..2b00d17 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,8 +10,10 @@ from confluid.registry import get_registry from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from dataflux.ops.configure import ConfigureOp from dataflux.ops.copy import CopyInputOp from dataflux.ops.enable import Enable +from dataflux.ops.formula import FormulaOp from dataflux.ops.image import ConvertToImageOp, NormalizeToUint8Op from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp from dataflux.ops.parallel import Parallel @@ -102,6 +104,8 @@ def test_op_group_tags() -> None: assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" assert TransformChain.__confluid_group__ == "compose" + assert ConfigureOp.__confluid_group__ == "compose" + assert FormulaOp.__confluid_group__ == "compose" assert ConvertToImageOp.__confluid_group__ == "image" assert NormalizeToUint8Op.__confluid_group__ == "image" assert SampleSinkOp.__confluid_group__ == "sink" diff --git a/tests/test_ops.py b/tests/test_ops.py index 1c8d9ac..03a0ae6 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -8,10 +8,12 @@ from PIL import Image from dataflux.ops import ( + ConfigureOp, CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp, + FormulaOp, RescaleOp, SqueezeOp, StandardizeOp, @@ -25,6 +27,7 @@ UnstashTargetOp, ) from dataflux.ops import numpy as np_ops +from dataflux.ops.numpy import MaxOp, ThresholdOp from dataflux.sample import Sample # --------------------------------------------------------------------------- @@ -645,6 +648,119 @@ def test_stash_restore_round_trip_preserves_fork_target(self) -> None: assert restored.target == "fork-target" +# --------------------------------------------------------------------------- +# FormulaOp +# --------------------------------------------------------------------------- + + +class TestFormulaOp: + def test_evaluates_formula_over_input(self) -> None: + sample = Sample(input=10.0, target=None, metadata={}) + out = FormulaOp(formula="a * 0.2")(sample) + assert out.input == 2.0 + + def test_custom_var_binding(self) -> None: + sample = Sample(input=9.0, target=None, metadata={}) + out = FormulaOp(formula="sqrt(b)", var="b")(sample) + assert out.input == 3.0 + + def test_math_namespace_and_helpers(self) -> None: + sample = Sample(input=-4.2, target=None, metadata={}) + out = FormulaOp(formula="round(abs(a))")(sample) + assert out.input == 4 + + def test_no_builtins_in_namespace(self) -> None: + sample = Sample(input=1.0, target=None, metadata={}) + with pytest.raises(ValueError, match="failed"): + FormulaOp(formula="__import__('os').getcwd()")(sample) + + def test_bad_formula_raises_value_error(self) -> None: + sample = Sample(input=1.0, target=None, metadata={}) + with pytest.raises(ValueError, match="failed"): + FormulaOp(formula="a +")(sample) + + def test_empty_formula_raises_lazily(self) -> None: + sample = Sample(input=1.0, target=None, metadata={}) + with pytest.raises(ValueError, match="non-empty"): + FormulaOp(formula=" ")(sample) + + def test_default_is_identity(self) -> None: + sample = Sample(input=7.5, target=None, metadata={}) + assert FormulaOp()(sample).input == 7.5 + + +# --------------------------------------------------------------------------- +# ConfigureOp (the helios Configure pattern) +# --------------------------------------------------------------------------- + + +class TestConfigureOp: + def test_computes_injects_and_applies(self) -> None: + """compute-chain value → metadata + target attribute → target applied to the ORIGINAL sample.""" + sample = Sample(input=np.array([1.0, 5.0, 3.0]), target=None, metadata={}) + op = ConfigureOp( + ops=[MaxOp()], + target=ThresholdOp(low_op=">="), + param="low_level", + ) + out = op(sample) + assert out is not None + target = op.target + assert isinstance(target, ThresholdOp) and target.low_level == 5.0 # injected per sample + assert out.meta["low_level"] == 5.0 # traceability: the value rides metadata too + np.testing.assert_array_equal(out.input, [False, True, False]) # threshold on the ORIGINAL array + + def test_empty_compute_chain_uses_incoming_input(self) -> None: + class _Target: + def __init__(self) -> None: + self.level: object = None + + def __call__(self, s: Sample) -> Sample: + return s + + target = _Target() + sample = Sample(input=7.5, target=None, metadata={}) + out = ConfigureOp(target=target, param="level")(sample) + assert out is not None + assert target.level == 7.5 # no compute chain → the incoming input IS the value + assert out.meta["level"] == 7.5 + + def test_key_overrides_metadata_key(self) -> None: + sample = Sample(input=np.array([2.0]), target=None, metadata={}) + op = ConfigureOp(ops=[MaxOp()], target=ThresholdOp(low_op=">="), param="low_level", key="thr") + out = op(sample) + assert out is not None and out.meta["thr"] == 2.0 + assert "low_level" not in out.meta + + def test_missing_target_or_param_raise_lazily(self) -> None: + sample = Sample(input=np.array([1.0]), target=None, metadata={}) + with pytest.raises(ValueError, match="'target' op is required"): + ConfigureOp(param="x")(sample) + with pytest.raises(ValueError, match="'param'"): + ConfigureOp(target=ThresholdOp())(sample) + + def test_compute_chain_filtering_drops_sample(self) -> None: + """A compute op returning None propagates the drop (FilterOp semantics, like Tee).""" + sample = Sample(input=np.array([1.0]), target=None, metadata={}) + op = ConfigureOp(ops=[lambda s: None], target=ThresholdOp(), param="low_level") + assert op(sample) is None + + def test_fluid_markers_flow_lazily(self) -> None: + """!class: markers in ops/target are flowed at first call (YAML-built ConfigureOp).""" + from confluid.fluid import Class + + sample = Sample(input=np.array([1.0, 4.0]), target=None, metadata={}) + op = ConfigureOp( + ops=[Class(MaxOp)], + target=Class(ThresholdOp, low_op=">="), + param="low_level", + ) + out = op(sample) + assert out is not None + assert isinstance(op.target, ThresholdOp) and op.target.low_level == 4.0 + np.testing.assert_array_equal(out.input, [False, True]) + + # --------------------------------------------------------------------------- # numpy.resolve_expression # --------------------------------------------------------------------------- From 08c9010933dc09a6cd3316c2b580d9dcddfe39f8 Mon Sep 17 00:00:00 2001 From: gertbehi Date: Tue, 16 Jun 2026 09:49:06 +0200 Subject: [PATCH 015/102] feat: FFT/window ops, image text+histogram helpers, CaptureOutputOp; storage-sink discovery categories - 1-D FFT family (numpy+torch FourierOp/InverseFourierOp/FftShiftOp/IfftShiftOp) + WindowOp/SpectrumScalingOp, backed by the dataflux.windows calibration library. - ops/image: draw_text + array_histogram/select_channel/channel_count helpers (back FluxStudio Draw Text / Array Histogram nodes). - CaptureOutputOp (records an op @output value into metadata). - Storage sinks (HDF5Sink/ZarrGroupSink/ZarrBatchSink/DirectorySink) carry category="sink" + dataflux-storage-* entry points, so FluxStudio surfaces them as DatasetProcessor sink nodes. --- AGENTS.md | 6 +- README.md | 89 ++++- dataflux/ops/__init__.py | 31 +- dataflux/ops/capture.py | 120 +++++++ dataflux/ops/image.py | 286 ++++++++++++++- dataflux/ops/numpy.py | 331 +++++++++++++++++ dataflux/ops/torch.py | 388 +++++++++++++++++++- dataflux/storage/directory.py | 3 +- dataflux/storage/hdf5.py | 3 +- dataflux/storage/zarr.py | 6 +- dataflux/windows.py | 261 ++++++++++++++ pyproject.toml | 12 + tests/test_categories.py | 86 ++++- tests/test_fourier_ops.py | 655 ++++++++++++++++++++++++++++++++++ tests/test_image_ops.py | 203 +++++++++++ tests/test_node_docs.py | 30 +- tests/test_ops.py | 117 ++++++ tests/test_windows.py | 157 ++++++++ 18 files changed, 2768 insertions(+), 16 deletions(-) create mode 100644 dataflux/ops/capture.py create mode 100644 dataflux/windows.py create mode 100644 tests/test_fourier_ops.py create mode 100644 tests/test_windows.py diff --git a/AGENTS.md b/AGENTS.md index 386d634..5d2c426 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,14 +7,14 @@ - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. Storage classes are bare `@configurable` with **no** discovery `category` — they are YAML `!class:` nodes wired into source/sink slots, not FluxStudio canvas nodes (unlike the `category="source"`/`"op"` classes). **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `DATAFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `dataflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`dataflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `dataflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). - **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`dataflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing dataflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a dataflux dependency for this. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's *Array / Tensor Histogram* viewer (`fluxstudio.nodes.ArrayHistogramViewerNode`): `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. diff --git a/README.md b/README.md index 2a7e5c1..6da1025 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Part of the **Modular Quartet**: `LogFlow`, `Confluid`, `Liquify`, and `DataFlux ### Metadata & Discovery - **Passive Introspection:** Automatically discover available tools and ops for serialized manifests. -- **Discovery Categories:** `@configurable` classes are tagged with a confluid `category` (sources `HuggingFaceSource`/`DatasetSplit` → `source`, engines `Flux`/`JointFlux` → `engine`, concrete `Sample→Sample` ops → `op`; `FilterOp`/`WrappedOp` are deliberately UNcategorised) so tools like navigaitor's `list_configurable_classes(category=...)` enumerate them by kind. +- **Discovery Categories:** `@configurable` classes are tagged with a confluid `category` (sources `HuggingFaceSource`/`DatasetSplit` → `source`, engines `Flux`/`JointFlux` → `engine`, concrete `Sample→Sample` ops → `op`, storage **sinks** `HDF5Sink`/`ZarrGroupSink`/`ZarrBatchSink`/`DirectorySink` → `sink` (FluxStudio surfaces them as `DatasetProcessor` sink nodes; their read-back **sources** stay UNcategorised); `FilterOp`/`WrappedOp` are deliberately UNcategorised) so tools like navigaitor's `list_configurable_classes(category=...)` enumerate them by kind. - **Serialization Symmetry:** Ensure full-pipeline states are serializable and reconstructible via Confluid. ## 🛠 Quick Start @@ -87,6 +87,69 @@ class StandardizeOp: Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime against a concrete inferred type); `compatible(consumer, producer)` is permissive (used at edit time — `Any`/unknown on either side passes). A `Sample`'s own type comes from `sample.describe()` — it returns a type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict) + `__spec__` (sidecar refinements), or infers one from the live data; attach a stored type with `sample.with_type(SampleType(...))`. +## 🌀 Fourier Transform (`FourierOp` / `InverseFourierOp` / shift ops) + +A small **1-D FFT toolkit**, each op in a numpy variant (`dataflux.ops.numpy`, on `np.ndarray`) and a torch variant (`dataflux.ops.torch`, on `torch.Tensor`); the flat `from dataflux.ops import …` resolves to the torch one (the package's torch-default convention, like `RescaleOp`): + +- **`FourierOp`** — the 1-D discrete Fourier transform (`numpy.fft.fft` / `torch.fft.fft`). +- **`InverseFourierOp`** — its inverse (`…fft.ifft`), back to the time domain. +- **`FftShiftOp`** / **`IfftShiftOp`** — center the zero-frequency component, and undo it (`…fft.fftshift` / `ifftshift`). + +`FourierOp` / `InverseFourierOp` accept **real *and* complex** signals; the raw transform is always complex (take `.real` downstream if you started real). With a unit `scaling` (see **Windowing & spectral units** below) `FourierOp` may instead emit a *real* power/density spectrum, so it declares a permissive `PRODUCES` (complex **or** floating). The shift ops are pure, dtype-preserving bin rearrangements (no FFT), so they work on any array — including an already-computed 2-D spectrogram. + +```python +import numpy as np +from dataflux.sample import Sample +from dataflux.ops.numpy import FourierOp, InverseFourierOp, FftShiftOp + +x = np.array([1.0, 2.0, 3.0, 4.0]) # real signal +spectrum = FourierOp()(Sample(input=x)).input # complex128, == np.fft.fft(x) + +xc = np.array([1 + 2j, 3 - 1j, 0j, -2 + 1j]) # complex signal — also supported +FourierOp(n=8, axis=-1, norm="ortho")(Sample(input=xc)) # zero-pad to 8, orthonormal scaling + +# Round-trip (forward then inverse recovers the input): +recovered = InverseFourierOp()(FourierOp()(Sample(input=x))).input.real # ≈ x + +# Center the spectrum for display — two equivalent ways: +centered = FftShiftOp()(FourierOp()(Sample(input=x))) # explicit, composable +centered = FourierOp(shift=True)(Sample(input=x)) # the one-node convenience flag +``` + +Parameters mirror `numpy.fft.fft` / `torch.fft.fft`: `n` (output length — zero-pad/truncate), `axis` (numpy) / `dim` (torch) — the single transform axis, default the last, so a `[B, N]` batch transforms per row — and `norm`, a closed `Literal["backward", "ortho", "forward"]` (use the **same** `norm` on the inverse to round-trip). Dtype promotion follows each framework: real `float32`/`complex64` → `complex64`, `float64`/integer/`complex128` → `complex128` (numpy) or `complex64` for integer (torch); the torch FFT/IFFT ops promote half precision (`float16`/`bfloat16`) to `float32` first because torch's FFT rejects it. Both transform ops take a `shift` flag — `FourierOp(shift=True)` applies `fftshift` **after** the transform, `InverseFourierOp(shift=True)` applies `ifftshift` **before** it — so the two invert each other exactly (the standalone `FftShiftOp`/`IfftShiftOp` are the same logic, decoupled, for centering arrays that didn't come from `FourierOp`). + +### Windowing & spectral units (`WindowOp` / `SpectrumScalingOp` / `FourierOp(window=…, scaling=…)`) + +A raw FFT is **uncalibrated** — to read a spectrum in real units you must taper the signal with a *window* (to control spectral leakage) and divide out the window's gain. DataFlux ships this as two composable ops plus options on `FourierOp` (numpy **and** torch variants). The window + unit math lives in **`dataflux.windows`** (pure numpy; `get_window` / `scale_spectrum` / the `WindowName` + `SpectrumScaling` Literals). + +- **`WindowOp(window=…)`** — multiplies the signal by a taper and **stashes the correction** (`window_sum` `S1=Σw`, `window_sum_sq` `S2=Σw²`, `window_enbw_bins`, `window_coherent_gain`) into the metadata for a later scaling step. Windows: `boxcar` (rectangular/none), `bartlett`, `hann`, `hamming`, `blackman`, `blackmanharris`, `nuttall`, `flattop`, `kaiser`, `tukey`, `gaussian` — parametrized ones take `window_param` (Kaiser β / Tukey α / Gaussian σ); `periodic=True` (default) is the DFT-even form correct for FFT analysis. +- **`SpectrumScalingOp(scaling=…)`** — turns a spectrum into physical units, reading `S1`/`S2` from the metadata (rectangular `S1=S2=N` if no window was applied): + + | `scaling` | output | formula | units | + |---------------|-------------------|------------------------|----------| + | `"none"` | complex (raw) | `X` | — | + | `"amplitude"` | complex | `X / S1` | V | + | `"power"` | real | `|X|² / S1²` | V² | + | `"density"` | real | `|X|² / (Fs·S2)` | V²/Hz | + + `density` uses `sample_rate` (Hz) → falls back to `metadata["samplerate"]` → `1.0` (per normalized frequency). `one_sided=True` folds a real signal's spectrum to one side (keep `0…N/2`, double the interior bins). + +- **`FourierOp(window=…, scaling=…, sample_rate=…)`** folds all three into one node. The default (`window="boxcar"`, `scaling="none"`) is byte-for-byte the old behaviour. Calibrated `scaling` assumes the unscaled transform, so combining it with a non-`"backward"` `norm` raises. + +```python +from dataflux.ops.numpy import FourierOp, WindowOp, SpectrumScalingOp + +# one node — Hann-windowed power-spectral density in dBW/Hz-ready units: +psd = FourierOp(window="hann", scaling="density", sample_rate=122.88e6)(sample).input + +# …is exactly the explicit, composable chain: +psd = SpectrumScalingOp(scaling="density", sample_rate=122.88e6)( + FourierOp()(WindowOp(window="hann")(sample)) +).input +``` + +A unit-amplitude tone reads `amplitude` ≈ its amplitude and `power` ≈ amplitude²; `power` and `density` differ by the window's equivalent noise bandwidth in Hz (`Fs·S2/S1²`) — the calibration that makes a windowed FFT match a reference analyzer. + ## 🔎 Field Projection & Class Counting Walking a source for a single field (the classic case: counting classes from @@ -313,6 +376,30 @@ flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) The helper **materializes** the deferred `!class:` markers before attaching (via `confluid.materialize`) — necessary because `confluid.load` leaves markers nested under a mapping key deferred, and a `Flux` rejects deferred markers at iteration by design. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. +## 🎛 Per-sample op parameters (`ConfigureOp` / `CaptureOutputOp`) + +Some op parameters are only known *per sample*. Two composable ops cover this — both are what FluxStudio emits when you wire a value into an op parameter on the canvas: + +- **`ConfigureOp(ops, target, param, key)`** — runs `ops` on the sample as a side-branch; the chain's final `sample.input` is written to `metadata[key]` and injected as `target.`, then `target` is applied. Use it when the value is *derived from the sample itself* (e.g. a threshold from the sample's own max). +- **`CaptureOutputOp(op, output|captures, key)`** — applies `op`, then records one or more of its `@output` attribute values into `metadata[key]`. The value is captured from the **actual run**, so it works for *stochastic* outputs (a random draw) that can't be recomputed. It reads through a `.target` wrapper, so it composes with `ConfigureOp`. + +Together they express "feed one op's runtime `@output` into a later op's parameter" — capture the output, then unstash it into the parameter per sample: + +```yaml +ops: + # NoiseFloorOp draws an SNR each call; capture it into metadata. + - !class:dataflux.ops.capture.CaptureOutputOp + op: !class:waivefront.torchsig.processing.NoiseFloorOp {} + output: applied_snr_db + key: __captured_snr + # …then inject the captured value into a later op's `noise_power_db` per sample. + - !class:dataflux.ops.configure.ConfigureOp + ops: + - !class:dataflux.ops.stash.UnstashInputOp { key: __captured_snr } + target: !class:waivefront.torchsig.processing.NoiseFloorOp {} + param: noise_power_db +``` + ## 🔗 Paired Join (Binary ↔ Annotations) `AnnotationJoinSource` joins a data `DataSource` (e.g. raw binary samples) with a sidecar mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern — typically re-attaching a LabelStudio export back onto the raw samples for training. Three join policies cover the scenarios we actually see in ML research: diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py index af7e5a9..dce64ed 100644 --- a/dataflux/ops/__init__.py +++ b/dataflux/ops/__init__.py @@ -4,14 +4,19 @@ Submodules: - dataflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp, SqueezeOp, - UnsqueezeOp (ndarray) + UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, + WindowOp, SpectrumScalingOp (ndarray) - dataflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp, SqueezeOp, - UnsqueezeOp (tensor) + UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, + WindowOp, SpectrumScalingOp (tensor) + - dataflux.windows: get_window / scale_spectrum + the WindowName / + SpectrumScaling Literals — the window + unit-scaling math the FFT ops share - dataflux.ops.tee: Tee (fan-out branching) - dataflux.ops.parallel: Parallel (worker-pool sub-pipeline) - dataflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - dataflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - dataflux.ops.configure: ConfigureOp (per-sample parameter injection — the helios Configure pattern) + - dataflux.ops.capture: CaptureOutputOp (record an op's @output value into metadata) - dataflux.ops.formula: FormulaOp (math formula over sample.input — the Math node's op form) - dataflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - dataflux.ops.transform_chain: TransformChain (sequential op-chain grouping) @@ -24,6 +29,7 @@ swap / stash / target utilities are field-agnostic. """ +from dataflux.ops.capture import CaptureOutputOp from dataflux.ops.configure import ConfigureOp from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp from dataflux.ops.enable import Enable @@ -41,10 +47,23 @@ MetadataToTargetOp, ) from dataflux.ops.tee import Tee -from dataflux.ops.torch import RescaleOp, SqueezeOp, StandardizeOp, ToTensorOp, UnsqueezeOp +from dataflux.ops.torch import ( + FftShiftOp, + FourierOp, + IfftShiftOp, + InverseFourierOp, + RescaleOp, + SpectrumScalingOp, + SqueezeOp, + StandardizeOp, + ToTensorOp, + UnsqueezeOp, + WindowOp, +) from dataflux.ops.transform_chain import TransformChain __all__ = [ + "CaptureOutputOp", "ConfigureOp", "CopyInputOp", "CopyMetadataOp", @@ -52,7 +71,11 @@ "CopyTargetOp", "DecodeTargetOp", "Enable", + "FftShiftOp", "FormulaOp", + "FourierOp", + "IfftShiftOp", + "InverseFourierOp", "EncodeTargetOp", "MetadataToTargetOp", "CocoToTorchVisionDetectionOp", @@ -61,6 +84,7 @@ "RandomApply", "RescaleOp", "SampleSinkOp", + "SpectrumScalingOp", "SqueezeOp", "StandardizeOp", "StashInputOp", @@ -72,4 +96,5 @@ "UnstashInputOp", "UnstashTargetOp", "UnsqueezeOp", + "WindowOp", ] diff --git a/dataflux/ops/capture.py b/dataflux/ops/capture.py new file mode 100644 index 0000000..614adc9 --- /dev/null +++ b/dataflux/ops/capture.py @@ -0,0 +1,120 @@ +"""``CaptureOutputOp`` — record an op's declared ``@output`` value into sample metadata. + +Wraps a target op: applies it to the sample (so the op's ``@output`` properties take their +post-call values), then copies one or more of those ``@output`` attributes off the *live op +instance* into ``metadata[key]``, and returns the applied sample. + +This is the capture half of FluxStudio's "wire one op's runtime ``@output`` into a LATER op's +parameter" feature. A canvas wire from e.g. ``NoiseFloorOp.applied_snr_db`` into another op's +parameter compiles to a ``CaptureOutputOp`` (records the producer's ACTUAL drawn value) followed +by a ``ConfigureOp(ops=[UnstashInputOp(key)], target=…, param=…)`` that injects it per sample. +The value MUST be captured from the real run — many ``@output``\\ s are stochastic +(``applied_snr_db`` is a random SNR draw) and so cannot be re-derived by re-running the op. + +Modality-neutral — it threads any ``Sample`` through any op — so it lives in core dataflux +(``compose`` group, alongside ``ConfigureOp`` / ``FormulaOp`` / the stash family). +""" + +from typing import Any, Dict, Optional, cast + +from confluid import configurable, flow +from confluid.fluid import Fluid + +from dataflux.sample import Sample + +_MISSING = object() + + +@configurable(category="op", group="compose") +class CaptureOutputOp: + """Apply an op, then copy its ``@output`` attribute(s) into the sample metadata. + + The wrapped ``op`` is applied to the incoming sample (its input/target transformations are + KEPT — the returned sample is ``op(sample)``), then each requested ``@output`` attribute is + read off the live ``op`` instance and written to ``metadata[]``. Use ``captures`` to + record SEVERAL outputs from ONE application (so a stochastic op runs exactly once); ``output`` + / ``key`` are the single-output convenience form. + + Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call (like + ``ConfigureOp``), so a ``CaptureOutputOp()`` built from YAML costs nothing. + + YAML — capture ``NoiseFloorOp``'s drawn SNR so a later op can read it back: + + .. code-block:: yaml + + - !class:dataflux.ops.capture.CaptureOutputOp + op: !class:waivefront.torchsig.processing.NoiseFloorOp {} + output: applied_snr_db + key: __captured_snr + + Args: + op: The op to apply; its ``@output`` attributes are read after it runs. Required at call time, validated lazily. + output: A single ``@output`` attribute name to capture. Blank = capture only the ``captures`` entries. + key: Metadata key for the ``output`` value. Blank (default) = the ``output`` name itself. + captures: Mapping of ``@output`` attribute name -> metadata key, for capturing several outputs in one apply. + """ + + def __init__( + self, + op: Optional[object] = None, + output: str = "", + key: str = "", + captures: Optional[Dict[str, str]] = None, + ) -> None: + # Lazy / zero-arg: store config only; op/outputs are validated at first call. + self.op = op + self.output = str(output) + self.key = str(key) + self.captures = dict(captures) if captures else {} + + def _items(self) -> Dict[str, str]: + """The full ``{output_name: metadata_key}`` map — ``captures`` plus the single-output form.""" + items = dict(self.captures) + if self.output: + items.setdefault(self.output, self.key or self.output) + return items + + @staticmethod + def _read_output(op: Any, name: str) -> Any: + """Read the ``@output`` attribute ``name`` off ``op``, looking THROUGH a ``target`` chain. + + The op may be wrapped (e.g. by a ``ConfigureOp``, which exposes the configured op as + ``.target``) when its own params are also configured — so the ``@output`` lives on the + innermost wrapped op. Walk ``.target`` to the first level that declares ``name``; returns + ``_MISSING`` if no level has it. + """ + cur, seen = op, set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + value = getattr(cur, name, _MISSING) + if value is not _MISSING: + return value + cur = getattr(cur, "target", None) + return _MISSING + + def __call__(self, sample: Sample) -> Optional[Sample]: + if self.op is None: + raise ValueError("CaptureOutputOp: an 'op' to apply is required") + items = self._items() + if not items: + raise ValueError("CaptureOutputOp: nothing to capture — set 'output' (and 'key') or 'captures'") + if isinstance(self.op, Fluid): + self.op = flow(self.op) + op = cast(Any, self.op) + result = op(sample) + if result is None: + return None # the wrapped op filtered the sample (FilterOp semantics) + for name, meta_key in items.items(): + value = self._read_output(op, name) + if value is _MISSING: + raise AttributeError( + f"CaptureOutputOp: {type(op).__name__!r} has no @output attribute {name!r} to capture" + ) + result.meta[meta_key] = value + return cast(Optional[Sample], result) + + def close(self) -> None: + """Propagate close() to the wrapped op if it owns resources.""" + close_fn = getattr(self.op, "close", None) + if callable(close_fn): + close_fn() diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py index ebded40..9d6fdee 100644 --- a/dataflux/ops/image.py +++ b/dataflux/ops/image.py @@ -18,7 +18,7 @@ need it, so the pure-greyscale path stays matplotlib-free. """ -from typing import Any, Literal, Optional, Tuple, get_args +from typing import Any, Dict, Literal, Optional, Tuple, get_args import numpy as np import torch @@ -186,6 +186,284 @@ def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: in return value_to_image(sample.input, colormap=colormap, max_size=max_size) +# --------------------------------------------------------------------------- # +# Array introspection helpers — channel selection + histogram. +# +# These back FluxStudio's "Array / Tensor Histogram" viewer node (and are usable +# from any pipeline / notebook): a generic, modality-agnostic way to look at the +# RAW numeric values of an array/tensor — pick a channel, render it, and bin its +# values. Pure functions (NOT @configurable ops): they measure/derive, they don't +# transform a Sample, so they're library helpers like value_to_image — not canvas +# nodes. They live here (not in the FluxStudio node) so the computation is reusable +# and unit-tested, per the workspace "rendering/analysis lives in dataflux" mandate. +# --------------------------------------------------------------------------- # + + +def _coerce_to_ndarray(value: Any) -> Optional[np.ndarray]: + """Best-effort view of an arbitrary value as a numeric ``np.ndarray`` for analysis. + + PIL image → RGB array, ``torch.Tensor`` → detached numpy, complex array → + magnitude (``abs``), list/scalar → ``np.asarray``. Returns ``None`` when the + value cannot sensibly be viewed as a numeric array (string/bytes/None, or an + object-dtype array such as a list of ragged things). + """ + data: Any = value + if data is None or isinstance(data, (str, bytes)): + return None + if hasattr(data, "convert"): # PIL.Image.Image + data = np.array(data.convert("RGB")) + elif isinstance(data, torch.Tensor): + data = data.detach().cpu().numpy() + try: + arr = np.asarray(data) + except Exception: # pragma: no cover - defensive: exotic objects np can't view + return None + if arr.dtype == object: + return None + if np.iscomplexobj(arr): + arr = np.abs(arr) + return arr + + +def _squeeze_to_3d(arr: np.ndarray) -> np.ndarray: + """Squeeze size-1 axes, then drop leading axes until at most 3-D (a ``[B,C,H,W]`` → first item).""" + arr = np.squeeze(arr) + while arr.ndim > 3: + arr = arr[0] + return arr + + +def _channel_axis(shape: Tuple[int, ...]) -> int: + """Index of the channel axis of a 3-D shape: the SMALLEST axis (channels-are-fewest convention). + + Deliberately distinct from the other two channel heuristics in this workspace, each scoped to a + narrower job: :func:`_render_rgb`'s ``{1,3,4}``-membership test is RGB-render-specific (it only + recognises 1/3/4-channel *images*), and ``fluxstudio.nodes.SampleExtractorNode._as_2d`` is + mask-specific (float-only). For a general N-channel feature map (e.g. an 8-channel tensor) the + smallest-axis rule is the most defensible default; documented here so the three never look like an + accidental disagreement. + """ + return int(np.argmin(shape)) + + +def select_channel(value: Any, channel: int = -1) -> np.ndarray: + """Reduce an arbitrary array/tensor to a single 2-D ``float32`` map for the given channel. + + The view used both for rendering one channel and for the per-pixel hover readout: + + * a 2-D array passes through; a 1-D array becomes a ``(1, N)`` strip; a scalar a ``(1, 1)`` cell; + * a 3-D array selects ``channel`` along its channel axis (the smallest axis — see + :func:`_channel_axis`); ``channel < 0`` collapses that axis by **mean** (an "all channels" view); + * higher-rank arrays drop leading axes to 3-D first; complex data is magnitude (``abs``). + + Out-of-range ``channel`` is clamped into ``[0, channels-1]``. A non-numeric value yields a ``(1, 1)`` + zero map (so callers always get a real 2-D array). + + Args: + value: The array / tensor / PIL image / scalar to view. + channel: Channel index to select; ``-1`` (default) means "all" → mean across the channel axis. + """ + arr = _coerce_to_ndarray(value) + if arr is None: + return np.zeros((1, 1), dtype=np.float32) + arr = _squeeze_to_3d(np.asarray(arr, dtype=np.float32)) + if arr.ndim == 0: + return arr.reshape(1, 1) + if arr.ndim == 1: + return arr.reshape(1, -1) + if arr.ndim == 2: + return arr + # 3-D: the channel axis is the smallest axis. + caxis = _channel_axis(arr.shape) + n_channels = arr.shape[caxis] + if channel is None or channel < 0: + return np.asarray(arr.mean(axis=caxis), dtype=np.float32) + idx = min(max(int(channel), 0), n_channels - 1) + return np.asarray(np.take(arr, idx, axis=caxis), dtype=np.float32) + + +def channel_count(value: Any) -> int: + """Number of channels of an array/tensor: 1 for ≤2-D data, the smallest-axis size for 3-D, 0 for non-arrays.""" + arr = _coerce_to_ndarray(value) + if arr is None: + return 0 + sq = _squeeze_to_3d(np.asarray(arr)) + return int(sq.shape[_channel_axis(sq.shape)]) if sq.ndim == 3 else 1 + + +def array_histogram(value: Any, bins: int = 256, channel: int = -1) -> Dict[str, Any]: + """Bin the values of an array/tensor into a histogram + summary statistics. + + Counts and statistics are taken over **finite** values only (``NaN`` / ``±inf`` are dropped, so + the result is always JSON-safe — no non-finite floats leak into ``min``/``max``/``bin_edges``). + When ``channel >= 0`` the histogram is of that single channel's plane; ``channel < 0`` histograms + **every** element across all channels. + + Returns a dict with ``counts`` (length ``bins``), ``bin_edges`` (length ``bins+1``), ``min`` / + ``max`` / ``mean`` / ``std`` (``None`` when there are no finite values), ``count`` (number of + finite values) and ``channels`` (detected channel count). A degenerate all-equal array bins into + the first bin over a unit-wide range. + + Args: + value: The array / tensor / PIL image / scalar to histogram. + bins: Number of histogram bins (clamped to at least 1). + channel: Channel to histogram; ``-1`` (default) histograms all elements across channels. + """ + bins = max(1, int(bins)) + arr = _coerce_to_ndarray(value) + channels = channel_count(value) + if arr is None: + flat = np.empty((0,), dtype=np.float32) + elif channel is not None and channel >= 0: + flat = select_channel(value, channel).astype(np.float32).ravel() + else: + flat = np.asarray(arr, dtype=np.float32).ravel() + finite = flat[np.isfinite(flat)] + if finite.size == 0: + edges = np.linspace(0.0, 1.0, bins + 1) + return { + "counts": [0] * bins, + "bin_edges": edges.tolist(), + "min": None, + "max": None, + "mean": None, + "std": None, + "count": 0, + "channels": channels, + } + lo = float(finite.min()) + hi = float(finite.max()) + # A flat array (all values equal) has a zero-width range — pin a deterministic unit range so the + # single populated bin is predictable (np.histogram would otherwise auto-pad to lo±0.5). + hi_edge = hi if hi > lo else lo + 1.0 + # Pass EXPLICIT bin edges (np.linspace), NOT `bins=, range=(lo, hi)`. numpy 2.2.x's uniform + # fast path block-accumulates with `np.bincount(...)` for arrays larger than its 65536-element + # block, and on the workspace build that miscomputes the bincount length so `n += bincount(...)` + # dies with "operands could not be broadcast together with shapes (256,) (257,) (256,)" — i.e. it + # fails on any real image/spectrogram (>65536 px) while passing on the small arrays unit tests use. + # The explicit-edges path (searchsorted) sidesteps that bug and is otherwise identical: the last + # bin is closed, so values == hi are still counted (sum(counts) == finite.size). + edges = np.linspace(lo, hi_edge, bins + 1) + counts, edges = np.histogram(finite, bins=edges) + return { + "counts": counts.astype(int).tolist(), + "bin_edges": edges.astype(float).tolist(), + "min": lo, + "max": hi, + "mean": float(finite.mean()), + "std": float(finite.std()), + "count": int(finite.size), + "channels": channels, + } + + +# --------------------------------------------------------------------------- # +# Text → image rendering — draw text onto an image (or a fresh canvas). +# --------------------------------------------------------------------------- # + +# Closed 9-grid set of text anchor positions (a closed Literal per the workspace mandate, so the +# choice is a dropdown in FluxStudio / navigaitor enumerated from one source of truth). +TextPosition = Literal[ + "top-left", + "top", + "top-right", + "center-left", + "center", + "center-right", + "bottom-left", + "bottom", + "bottom-right", +] +TEXT_POSITIONS: Tuple[TextPosition, ...] = get_args(TextPosition) + + +def _text_anchor_xy(position: str, block_w: int, block_h: int, img_w: int, img_h: int, margin: int) -> Tuple[int, int]: + """Top-left ``(x, y)`` for a ``block_w × block_h`` text block per the 9-grid ``position`` + margin.""" + if "left" in position: + x: float = margin + elif "right" in position: + x = img_w - block_w - margin + else: # "top" / "bottom" / "center" (no left/right) → horizontally centered + x = (img_w - block_w) / 2 + if position.startswith("top"): + y: float = margin + elif position.startswith("bottom"): + y = img_h - block_h - margin + else: # "center-*" / left / right (no top/bottom) → vertically centered + y = (img_h - block_h) / 2 + return int(round(x)), int(round(y)) + + +def _wrap_text(draw: "ImageDraw.ImageDraw", text: str, font: Any, max_width: int) -> str: + """Greedy word-wrap so each line fits ``max_width`` px (explicit newlines preserved).""" + out: list = [] + for paragraph in text.split("\n"): + line = "" + for word in paragraph.split(" "): + trial = f"{line} {word}".strip() + if line and draw.textlength(trial, font=font) > max_width: + out.append(line) + line = word + else: + line = trial + out.append(line) + return "\n".join(out) + + +def draw_text( + text: str, + image: Optional[Any] = None, + *, + width: int = 512, + height: int = 256, + font_size: int = 24, + color: str = "white", + background: str = "black", + position: TextPosition = "top-left", + margin: int = 8, + wrap: bool = True, +) -> np.ndarray: + """Render ``text`` onto ``image`` (or a fresh ``background`` canvas) → an ``(H, W, 3)`` uint8 RGB array. + + The single, modality-agnostic "draw text on an image" renderer (FluxStudio's *Draw Text to Image* + node is thin glue over it). When ``image`` is ``None`` a blank ``(height, width)`` canvas of color + ``background`` is created; otherwise the value is coerced to an RGB image (via :func:`_render_rgb`, + so PIL / ndarray / tensor / 2-D maps all work) and drawn on a copy. The text is word-wrapped to the + image width (``wrap``; explicit newlines kept) and anchored per the 9-grid ``position`` with a + ``margin`` inset. Uses PIL's sized default bitmap font. + + Args: + text: The text to draw (multi-line allowed). + image: Background image (PIL / ndarray / tensor / 2-D map); ``None`` makes a blank canvas. + width: Blank-canvas width in pixels (used only when ``image`` is ``None``). + height: Blank-canvas height in pixels (used only when ``image`` is ``None``). + font_size: Font size in points. + color: Text color — any PIL color name or hex (``"white"`` / ``"#ffcc00"`` / ...). + background: Canvas color when ``image`` is ``None`` — any PIL color name or hex. + position: Anchor of the text block — one of the 9-grid ``TextPosition`` values. + margin: Inset in pixels from the edges for non-centered anchors. + wrap: Word-wrap long lines to fit the image width. + """ + from PIL import ImageFont + + if image is None: + img = Image.new("RGB", (max(1, int(width)), max(1, int(height))), color=background) + else: + img = Image.fromarray(_render_rgb(image, "gray")) + draw = ImageDraw.Draw(img) + try: + font = ImageFont.load_default(size=int(font_size)) + except TypeError: # Pillow < 10.1 has no sized default font — fall back to the fixed bitmap font + font = ImageFont.load_default() + rendered = _wrap_text(draw, text, font, max(1, img.width - 2 * margin)) if wrap else text + # multiline_textbbox is relative to the anchor; subtract its offset so the block's top-left lands at (x, y). + bbox = draw.multiline_textbbox((0, 0), rendered, font=font) + block_w, block_h = int(bbox[2] - bbox[0]), int(bbox[3] - bbox[1]) + x, y = _text_anchor_xy(position, block_w, block_h, img.width, img.height, margin) + draw.multiline_text((x - bbox[0], y - bbox[1]), rendered, fill=color, font=font) + return np.array(img) + + @configurable(category="op", group="image") class ConvertToImageOp: """Convert ``sample.input`` (array / tensor / 2-D map / PIL image) into a PIL image. @@ -326,4 +604,10 @@ def __call__(self, sample: Sample) -> Sample: "NormalizeToUint8Op", "value_to_image", "sample_to_image", + "select_channel", + "channel_count", + "array_histogram", + "draw_text", + "TextPosition", + "TEXT_POSITIONS", ] diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index 1182a46..4901af7 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -9,10 +9,24 @@ from dataflux.sample import Sample from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType +from dataflux.windows import ( + WINDOW_SUM_KEY, + WINDOW_SUMSQ_KEY, + SpectrumScaling, + WindowName, + get_window, + scale_spectrum, + window_metadata, + window_sums, +) # Common shorthands for the numpy ops' declared types. _NDARRAY = ArrayType(frameworks={"numpy"}) _NUMERIC_OR_PIL = UnionType((ArrayType(dtype="numeric", frameworks={"numpy"}), PythonType("PIL.Image.Image"))) +# Spectrum-scaling ops emit complex (none/amplitude) OR real (power/density) — a permissive union. +_COMPLEX_OR_FLOAT = UnionType( + (ArrayType(dtype="complex", frameworks={"numpy"}), ArrayType(dtype="floating", frameworks={"numpy"})) +) logger = get_logger(__name__) @@ -648,3 +662,320 @@ def __call__(self, sample: Sample) -> Sample: # validates min_area_bins / connectivity and raises the scipy ImportError. bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) return sample._replace(input=bboxes) + + +def _apply_window(arr: np.ndarray, window: np.ndarray, axis: int) -> np.ndarray: + """Multiply ``arr`` by the 1-D ``window`` broadcast along ``axis`` (dtype-preserving). + + The window is cast to the real dtype matching ``arr`` so a ``complex64`` / ``float32`` signal + keeps its precision (a raw ``float64`` window would otherwise upcast it). + """ + if np.issubdtype(arr.dtype, np.floating) or np.issubdtype(arr.dtype, np.complexfloating): + window = window.astype(arr.real.dtype, copy=False) + moved = np.swapaxes(arr, axis, -1) + return np.swapaxes(moved * window, axis, -1) + + +def _resolve_fft_sample_rate(sample: Sample, explicit: Optional[float]) -> Optional[float]: + """Resolve the density-scaling sample rate: explicit arg → ``metadata['samplerate']`` → ``None``.""" + if explicit is not None: + return explicit + if sample.is_batched: + return None + raw = sample.meta.get("samplerate") + return float(raw) if raw else None + + +# FFT normalization mode. Closed ``Literal`` (workspace "prefer closed Literals over bare strings" +# mandate) so FluxStudio / navigaitor render the choice as a dropdown and the allowed modes stay +# machine-introspectable via ``typing.get_args(...)``. These three strings are EXACTLY what +# ``numpy.fft.fft`` accepts for its ``norm=`` argument (the same set ``torch.fft.fft`` uses), passed +# straight through with no parallel runtime tuple to drift. +FourierNorm = Literal["backward", "ortho", "forward"] + + +@configurable(category="op", group="numpy") +class FourierOp: + """Compute the 1-D discrete Fourier transform of ``sample.input`` (``numpy.fft.fft``). + + Accepts real **and** complex arrays; the raw output is always complex — ``complex64`` for + ``float32``/``complex64`` input, ``complex128`` for ``float64``/integer/``complex128`` input + (numpy's promotion rule). This is the **1-D** transform (``numpy.fft.fft``), not the 2-D / N-D + one: for an N-D array it runs along a single ``axis`` (default the last), so a ``[B, N]`` batch + of signals transforms per row. :class:`InverseFourierOp` is the inverse; set ``shift=True`` to + center the zero-frequency bin (the standalone :class:`FftShiftOp` does the same independently). + + **Windowing & units.** ``window`` applies a :func:`dataflux.windows.get_window` taper before the + transform (default ``"boxcar"`` = no taper = unchanged behaviour) and stashes the window + correction into the metadata; ``scaling`` then returns the spectrum in real units — + ``"amplitude"`` (V), ``"power"`` (V²) or ``"density"`` (V²/Hz, using ``sample_rate``) — dividing + out the window's coherent gain / noise bandwidth. ``scaling="none"`` (default) leaves the raw + complex spectrum. The one-node ``FourierOp(window="hann", scaling="density", sample_rate=…)`` is + equivalent to the explicit chain ``WindowOp(window="hann") → FourierOp() → + SpectrumScalingOp(scaling="density", sample_rate=…)``. Calibrated ``scaling`` requires the + unscaled transform (``norm="backward"``); any other ``norm`` with ``scaling != "none"`` raises. + + Args: + n: Output length along ``axis`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. + axis: Axis to transform over. Default ``-1`` (the last axis — the natural choice for a 1-D signal). + norm: Normalization — ``"backward"`` (default, unscaled forward), ``"ortho"`` (1/sqrt(n) both ways), + or ``"forward"`` (1/n on the forward transform). Calibrated ``scaling`` requires ``"backward"``. + shift: When True, ``fftshift`` along ``axis`` after transforming (centers the zero bin). Default False. + window: Taper applied before the FFT — a ``WindowName`` (``"boxcar"`` default = no taper). + window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. + periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. + scaling: Units — ``"none"`` (complex, default), ``"amplitude"`` V, ``"power"`` V², ``"density"`` V²/Hz. + sample_rate: Hz, for ``"density"``. ``None`` reads ``metadata["samplerate"]``, else ``1.0`` (normalized). + one_sided: Fold to a one-sided spectrum (real signals). Default ``False``; exclusive with ``shift``. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_COMPLEX_OR_FLOAT) + + def __init__( + self, + n: Optional[int] = None, + axis: int = -1, + norm: FourierNorm = "backward", + shift: bool = False, + window: WindowName = "boxcar", + window_param: Optional[float] = None, + periodic: bool = True, + scaling: SpectrumScaling = "none", + sample_rate: Optional[float] = None, + one_sided: bool = False, + ) -> None: + # Lazy / zero-arg: store config only. ``n`` and window params are validated lazily in __call__. + self.n = n + self.axis = axis + self.norm = norm + self.shift = bool(shift) + self.window: WindowName = window + self.window_param = window_param + self.periodic = bool(periodic) + self.scaling: SpectrumScaling = scaling + self.sample_rate = sample_rate + self.one_sided = bool(one_sided) + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "FourierOp") + if self.scaling != "none" and self.norm != "backward": + raise ValueError( + f"FourierOp: calibrated scaling={self.scaling!r} requires norm='backward' " + f"(the unscaled transform); got norm={self.norm!r}" + ) + if self.shift and self.one_sided: + raise ValueError("FourierOp: shift and one_sided are mutually exclusive (one_sided is a half-spectrum)") + window = None + signal = arr + if self.window != "boxcar": + window = get_window( + self.window, arr.shape[self.axis], window_param=self.window_param, periodic=self.periodic + ) + signal = _apply_window(arr, window, self.axis) + out = np.fft.fft(signal, n=self.n, axis=self.axis, norm=self.norm) + if self.scaling != "none": + s1, s2 = window_sums(window) if window is not None else (float(arr.shape[self.axis]),) * 2 + out = scale_spectrum( + out, + self.scaling, + s1=s1, + s2=s2, + sample_rate=_resolve_fft_sample_rate(sample, self.sample_rate), + one_sided=self.one_sided, + axis=self.axis, + ) + if self.shift: + out = np.fft.fftshift(out, axes=self.axis) + if window is not None and not sample.is_batched: + new_meta = dict(sample.meta) + new_meta.update(window_metadata(self.window, window)) + return sample._replace(input=out, metadata=new_meta) + return sample._replace(input=out) + + +@configurable(category="op", group="numpy") +class InverseFourierOp: + """Compute the 1-D inverse discrete Fourier transform of ``sample.input`` (``numpy.fft.ifft``). + + The sibling of :class:`FourierOp`: it maps a spectrum back to the time domain. The output is + always complex (``numpy.fft.ifft`` always returns complex; take ``.real`` downstream if the + original signal was real). ``InverseFourierOp(norm=…)`` must use the **same** ``norm`` as the + forward transform to round-trip. With ``shift=True`` an ``ifftshift`` is applied to the input + **before** the inverse transform, exactly undoing a prior ``FourierOp(shift=True)`` (the correct + pairing even for odd-length axes). + + Args: + n: Output length along ``axis`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. + axis: Axis to transform over. Default ``-1`` (the last axis — the natural choice for a 1-D signal). + norm: Normalization — must match the forward transform: ``"backward"`` (default), ``"ortho"``, or ``"forward"``. + shift: When True, ``ifftshift`` along ``axis`` before inverting (undoes a prior ``fftshift``). Default False. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=ArrayType(dtype="complex", frameworks={"numpy"})) + + def __init__( + self, n: Optional[int] = None, axis: int = -1, norm: FourierNorm = "backward", shift: bool = False + ) -> None: + # Lazy / zero-arg: store config only. ``n`` (if set) is validated lazily by numpy in __call__. + self.n = n + self.axis = axis + self.norm = norm + self.shift = bool(shift) + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "InverseFourierOp") + if self.shift: + arr = np.fft.ifftshift(arr, axes=self.axis) + out = np.fft.ifft(arr, n=self.n, axis=self.axis, norm=self.norm) + return sample._replace(input=out) + + +@configurable(category="op", group="numpy") +class FftShiftOp: + """Shift the zero-frequency component to the center of the spectrum (``numpy.fft.fftshift``). + + A pure bin-rearrangement — no FFT is computed, so it is dtype- AND shape-preserving and works + on **any** array (real, complex, or integer). Chain it after :class:`FourierOp` to center a + spectrum for display (the ``FourierOp(shift=True)`` flag is the one-node convenience), or use it + standalone to center an already-computed spectrum such as a 2-D spectrogram. :class:`IfftShiftOp` + is its exact inverse (they differ only for odd-length axes). + + Args: + axis: Axis to shift. Default ``-1`` (last axis, matches :class:`FourierOp`); ``None`` shifts every axis. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: Optional[int] = -1) -> None: + self.axis = axis + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "FftShiftOp") + return sample._replace(input=np.fft.fftshift(arr, axes=self.axis)) + + +@configurable(category="op", group="numpy") +class IfftShiftOp: + """Undo an :class:`FftShiftOp` — move the center frequency back to index 0 (``numpy.fft.ifftshift``). + + The exact inverse of :class:`FftShiftOp` (the two coincide for even-length axes but differ for + odd-length ones, which is why both exist). Like its sibling it is a pure, dtype- and + shape-preserving rearrangement that accepts any array. Apply it before :class:`InverseFourierOp` + to recover the natural FFT bin order (``InverseFourierOp(shift=True)`` folds it in). + + Args: + axis: Axis to shift. Default ``-1`` (last axis, matches :class:`InverseFourierOp`); ``None`` shifts every axis. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__(self, axis: Optional[int] = -1) -> None: + self.axis = axis + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "IfftShiftOp") + return sample._replace(input=np.fft.ifftshift(arr, axes=self.axis)) + + +@configurable(category="op", group="numpy") +class WindowOp: + """Apply a window taper to ``sample.input`` and record the unit-scaling correction. + + Multiplies the signal by a :func:`dataflux.windows.get_window` taper (broadcast along ``axis``) + — the standard first step of spectral analysis, controlling FFT spectral leakage — and stashes + the window's correction factors into ``sample.metadata`` (``window`` / ``window_sum`` ``S1`` / + ``window_sum_sq`` ``S2`` / ``window_enbw_bins`` / ``window_coherent_gain``) so a later + :class:`SpectrumScalingOp` can divide them out and return the spectrum in real units. Chain + ``WindowOp → FourierOp → SpectrumScalingOp``, or fold all three into one node via + ``FourierOp(window=…, scaling=…)``. Shape-preserving; real input stays real, complex stays + complex (the taper is cast to the input's real dtype so precision is preserved). + + Args: + window: Which taper — a ``WindowName`` (default ``"hann"``; ``"boxcar"`` is the rectangular identity). + window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. + periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. + axis: Axis the window is applied along. Default ``-1`` (the last axis — the 1-D signal). + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_NDARRAY) + + def __init__( + self, + window: WindowName = "hann", + window_param: Optional[float] = None, + periodic: bool = True, + axis: int = -1, + ) -> None: + # Lazy / zero-arg: store config only; window params are validated lazily by get_window. + self.window: WindowName = window + self.window_param = window_param + self.periodic = bool(periodic) + self.axis = axis + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "WindowOp") + window = get_window(self.window, arr.shape[self.axis], window_param=self.window_param, periodic=self.periodic) + out = _apply_window(arr, window, self.axis) + if sample.is_batched: + return sample._replace(input=out) + new_meta = dict(sample.meta) + new_meta.update(window_metadata(self.window, window)) + return sample._replace(input=out, metadata=new_meta) + + +@configurable(category="op", group="numpy") +class SpectrumScalingOp: + """Scale a (complex) FFT spectrum to physical units using the window correction. + + The calibration half of the FFT chain: turns the raw :class:`FourierOp` output into an amplitude + (V), power (V²) or power-spectral-density (V²/Hz) spectrum, dividing out the window's coherent + gain ``S1`` / noise bandwidth ``S2`` — read from the ``window_*`` metadata stashed by + :class:`WindowOp` or ``FourierOp(window=…)``; if absent it assumes a rectangular/boxcar window + (``S1=S2=N``). Assumes the spectrum came from the **unscaled** forward transform + (``norm="backward"``, the FourierOp default). Output dtype follows the mode — complex for + ``"none"``/``"amplitude"`` (phase preserved), real for ``"power"``/``"density"``. + + Args: + scaling: Units — ``"none"`` (unchanged), ``"amplitude"`` V, ``"power"`` V² (default), ``"density"`` V²/Hz. + sample_rate: Hz, for ``"density"``. ``None`` (default) reads ``metadata["samplerate"]``, else ``1.0``. + one_sided: Fold to one-sided (real-signal convention: keep 0…N/2, double interior bins). Default ``False``. + axis: Spectrum axis. Default ``-1``. + """ + + ACCEPTS = SampleType(input=_NDARRAY) + PRODUCES = SampleType(input=_COMPLEX_OR_FLOAT) + + def __init__( + self, + scaling: SpectrumScaling = "power", + sample_rate: Optional[float] = None, + one_sided: bool = False, + axis: int = -1, + ) -> None: + # Lazy / zero-arg: store config only. + self.scaling: SpectrumScaling = scaling + self.sample_rate = sample_rate + self.one_sided = bool(one_sided) + self.axis = axis + + def __call__(self, sample: Sample) -> Sample: + arr = _require_ndarray(sample, "SpectrumScalingOp") + n = arr.shape[self.axis] + if sample.is_batched: + s1 = s2 = float(n) + else: + meta = sample.meta + raw_s1, raw_s2 = meta.get(WINDOW_SUM_KEY), meta.get(WINDOW_SUMSQ_KEY) + s1, s2 = ( + (float(raw_s1), float(raw_s2)) if raw_s1 is not None and raw_s2 is not None else (float(n), float(n)) + ) + fs = _resolve_fft_sample_rate(sample, self.sample_rate) + if self.scaling == "density" and not fs: + logger.debug("SpectrumScalingOp: no sample_rate for density; using normalized frequency (Fs=1.0)") + out = scale_spectrum(arr, self.scaling, s1=s1, s2=s2, sample_rate=fs, one_sided=self.one_sided, axis=self.axis) + return sample._replace(input=out) diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py index 0e2ccef..f763f56 100644 --- a/dataflux/ops/torch.py +++ b/dataflux/ops/torch.py @@ -1,4 +1,4 @@ -from typing import Optional, Sequence, Union +from typing import Literal, Optional, Sequence, Union import numpy as np import torch @@ -6,9 +6,22 @@ from dataflux.sample import Sample from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType +from dataflux.windows import ( + WINDOW_SUM_KEY, + WINDOW_SUMSQ_KEY, + SpectrumScaling, + WindowName, + get_window, + window_metadata, + window_sums, +) _TORCH = ArrayType(frameworks={"torch"}) _TORCH_FLOAT = ArrayType(dtype="floating", frameworks={"torch"}) +# Spectrum-scaling ops emit complex (none/amplitude) OR real (power/density). +_TORCH_COMPLEX_OR_FLOAT = UnionType( + (ArrayType(dtype="complex", frameworks={"torch"}), ArrayType(dtype="floating", frameworks={"torch"})) +) @configurable(category="op", group="torch") @@ -209,3 +222,376 @@ def __call__(self, sample: Sample) -> Sample: tensor = (tensor - mean_t) / std_t return sample._replace(input=tensor) + + +def _apply_window(tensor: torch.Tensor, window: np.ndarray, dim: int) -> torch.Tensor: + """Multiply ``tensor`` by the 1-D numpy ``window`` broadcast along ``dim`` (dtype/device-preserving).""" + if tensor.is_complex(): + real_dtype = tensor.real.dtype + elif tensor.is_floating_point(): + real_dtype = tensor.dtype + else: + real_dtype = torch.float32 + w = torch.as_tensor(window, dtype=real_dtype, device=tensor.device) + moved = torch.movedim(tensor, dim, -1) + return torch.movedim(moved * w, -1, dim) + + +def _fold_one_sided(spectrum: torch.Tensor, dim: int) -> torch.Tensor: + """Fold a two-sided spectrum (natural order, DC at index 0) to one-sided (mirror of windows.fold_one_sided).""" + moved = torch.movedim(spectrum, dim, -1) + n = moved.shape[-1] + out = moved[..., : n // 2 + 1].clone() + if n % 2 == 0: + out[..., 1:-1] = out[..., 1:-1] * 2 # exclude DC and Nyquist + else: + out[..., 1:] = out[..., 1:] * 2 + return torch.movedim(out, -1, dim) + + +def _scale_spectrum( + spectrum: torch.Tensor, + scaling: SpectrumScaling, + *, + s1: float, + s2: float, + sample_rate: Optional[float], + one_sided: bool, + dim: int, +) -> torch.Tensor: + """Torch mirror of :func:`dataflux.windows.scale_spectrum` (assumes ``norm="backward"``).""" + if scaling == "none": + out = spectrum + elif scaling == "amplitude": + out = spectrum / s1 + elif scaling == "power": + out = spectrum.abs().square() / (s1 * s1) + elif scaling == "density": + fs = float(sample_rate) if (sample_rate is not None and sample_rate > 0) else 1.0 + out = spectrum.abs().square() / (fs * s2) + else: + raise ValueError(f"unknown scaling {scaling!r}") + return _fold_one_sided(out, dim) if one_sided else out + + +def _resolve_fft_sample_rate(sample: Sample, explicit: Optional[float]) -> Optional[float]: + """Resolve the density rate: explicit → ``metadata['samplerate']`` → ``None`` (mirrors the numpy op).""" + if explicit is not None: + return explicit + if sample.is_batched: + return None + raw = sample.meta.get("samplerate") + return float(raw) if raw else None + + +# FFT normalization mode — see the numpy ``FourierOp`` for the rationale. Exactly the three strings +# ``torch.fft.fft`` accepts for its ``norm=`` argument; a closed ``Literal`` so GUIs enumerate the +# choice via ``typing.get_args(...)``. +FourierNorm = Literal["backward", "ortho", "forward"] + + +@configurable(category="op", group="torch") +class FourierOp: + """Compute the 1-D discrete Fourier transform of ``sample.input`` (``torch.fft.fft``). + + Accepts real **and** complex tensors; the output is always complex — ``complex64`` for + integer / ``float32`` / ``complex64`` input, ``complex128`` for ``float64`` / ``complex128``. + Half-precision (``float16`` / ``bfloat16``) tensors are promoted to ``float32`` first because + ``torch.fft.fft`` does not support them; every other dtype (including integer and bool) is handled + natively (integers auto-promote to ``complex64``). This is the **1-D** transform + (``torch.fft.fft``), not the 2-D / N-D one: for an N-D tensor it runs along a single ``dim`` + (default the last), so a ``[B, N]`` batch transforms per row. :class:`InverseFourierOp` is the + inverse; set ``shift=True`` to center the zero-frequency bin (the standalone :class:`FftShiftOp` + does the same independently). + + **Windowing & units.** Mirrors the numpy ``FourierOp``: ``window`` applies a + :func:`dataflux.windows.get_window` taper before the transform (default ``"boxcar"`` = none) and + stashes the window correction; ``scaling`` returns the spectrum in real units — ``"amplitude"`` + (V), ``"power"`` (V²) or ``"density"`` (V²/Hz, via ``sample_rate``). ``scaling="none"`` (default) + leaves the raw complex spectrum. Calibrated ``scaling`` requires ``norm="backward"`` (any other + ``norm`` with ``scaling != "none"`` raises). + + Args: + n: Output length along ``dim`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. + dim: Dimension to transform over. Default ``-1`` (the last dim — the natural choice for a 1-D signal). + norm: Normalization — ``"backward"`` (default, unscaled forward), ``"ortho"`` (1/sqrt(n) both ways), + or ``"forward"`` (1/n on the forward transform). Calibrated ``scaling`` requires ``"backward"``. + shift: When True, ``fftshift`` along ``dim`` after transforming (centers the zero bin). Default False. + window: Taper applied before the FFT — a ``WindowName`` (``"boxcar"`` default = no taper). + window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. + periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. + scaling: Units — ``"none"`` (complex, default), ``"amplitude"`` V, ``"power"`` V², ``"density"`` V²/Hz. + sample_rate: Hz, for ``"density"``. ``None`` reads ``metadata["samplerate"]``, else ``1.0`` (normalized). + one_sided: Fold to a one-sided spectrum (real signals). Default ``False``; exclusive with ``shift``. + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH_COMPLEX_OR_FLOAT) + + def __init__( + self, + n: Optional[int] = None, + dim: int = -1, + norm: FourierNorm = "backward", + shift: bool = False, + window: WindowName = "boxcar", + window_param: Optional[float] = None, + periodic: bool = True, + scaling: SpectrumScaling = "none", + sample_rate: Optional[float] = None, + one_sided: bool = False, + ) -> None: + # Lazy / zero-arg: store config only. ``n`` and window params are validated lazily in __call__. + self.n = n + self.dim = dim + self.norm = norm + self.shift = bool(shift) + self.window: WindowName = window + self.window_param = window_param + self.periodic = bool(periodic) + self.scaling: SpectrumScaling = scaling + self.sample_rate = sample_rate + self.one_sided = bool(one_sided) + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"FourierOp expects a torch.Tensor, got {type(tensor).__name__}") + if self.scaling != "none" and self.norm != "backward": + raise ValueError( + f"FourierOp: calibrated scaling={self.scaling!r} requires norm='backward' " + f"(the unscaled transform); got norm={self.norm!r}" + ) + if self.shift and self.one_sided: + raise ValueError("FourierOp: shift and one_sided are mutually exclusive (one_sided is a half-spectrum)") + # torch.fft.fft rejects half precision; promote to float32. Integer/bool/float/complex are + # all accepted natively (integers auto-promote to complex64), so leave them untouched. + if tensor.dtype in (torch.float16, torch.bfloat16): + tensor = tensor.float() + window = None + signal = tensor + if self.window != "boxcar": + window = get_window( + self.window, tensor.shape[self.dim], window_param=self.window_param, periodic=self.periodic + ) + signal = _apply_window(tensor, window, self.dim) + out = torch.fft.fft(signal, n=self.n, dim=self.dim, norm=self.norm) + if self.scaling != "none": + s1, s2 = window_sums(window) if window is not None else (float(tensor.shape[self.dim]),) * 2 + out = _scale_spectrum( + out, + self.scaling, + s1=s1, + s2=s2, + sample_rate=_resolve_fft_sample_rate(sample, self.sample_rate), + one_sided=self.one_sided, + dim=self.dim, + ) + if self.shift: + out = torch.fft.fftshift(out, dim=self.dim) + if window is not None and not sample.is_batched: + new_meta = dict(sample.meta) + new_meta.update(window_metadata(self.window, window)) + return sample._replace(input=out, metadata=new_meta) + return sample._replace(input=out) + + +@configurable(category="op", group="torch") +class InverseFourierOp: + """Compute the 1-D inverse discrete Fourier transform of ``sample.input`` (``torch.fft.ifft``). + + The sibling of :class:`FourierOp`: it maps a spectrum back to the time domain. The output is + always complex (``torch.fft.ifft`` always returns complex; take ``.real`` downstream if the + original signal was real). Half-precision (``float16``/``bfloat16``) tensors are promoted to + ``float32`` first (``torch.fft.ifft`` rejects them); other dtypes are handled natively. + ``InverseFourierOp(norm=…)`` must use the **same** ``norm`` as the forward transform to + round-trip. With ``shift=True`` an ``ifftshift`` is applied to the input **before** inverting, + exactly undoing a prior ``FourierOp(shift=True)`` (the correct pairing even for odd-length dims). + + Args: + n: Output length along ``dim`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. + dim: Dimension to transform over. Default ``-1`` (the last dim — the natural choice for a 1-D signal). + norm: Normalization — must match the forward transform: ``"backward"`` (default), ``"ortho"``, or ``"forward"``. + shift: When True, ``ifftshift`` along ``dim`` before inverting (undoes a prior ``fftshift``). Default False. + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=ArrayType(dtype="complex", frameworks={"torch"})) + + def __init__( + self, n: Optional[int] = None, dim: int = -1, norm: FourierNorm = "backward", shift: bool = False + ) -> None: + # Lazy / zero-arg: store config only. ``n`` (if set) is validated lazily by torch in __call__. + self.n = n + self.dim = dim + self.norm = norm + self.shift = bool(shift) + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"InverseFourierOp expects a torch.Tensor, got {type(tensor).__name__}") + # torch.fft.ifft rejects half precision; promote to float32 (mirrors FourierOp). + if tensor.dtype in (torch.float16, torch.bfloat16): + tensor = tensor.float() + if self.shift: + tensor = torch.fft.ifftshift(tensor, dim=self.dim) + out = torch.fft.ifft(tensor, n=self.n, dim=self.dim, norm=self.norm) + return sample._replace(input=out) + + +@configurable(category="op", group="torch") +class FftShiftOp: + """Shift the zero-frequency component to the center of the spectrum (``torch.fft.fftshift``). + + A pure bin-rearrangement — no FFT is computed, so it is dtype- AND shape-preserving and works + on **any** tensor (real, complex, or integer; half precision included). Chain it after + :class:`FourierOp` to center a spectrum for display (the ``FourierOp(shift=True)`` flag is the + one-node convenience), or use it standalone to center an already-computed spectrum such as a 2-D + spectrogram. :class:`IfftShiftOp` is its exact inverse (they differ only for odd-length dims). + + Args: + dim: Dimension to shift. Default ``-1`` (last dim, matches :class:`FourierOp`); ``None`` shifts every dim. + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH) + + def __init__(self, dim: Optional[int] = -1) -> None: + self.dim = dim + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"FftShiftOp expects a torch.Tensor, got {type(tensor).__name__}") + return sample._replace(input=torch.fft.fftshift(tensor, dim=self.dim)) + + +@configurable(category="op", group="torch") +class IfftShiftOp: + """Undo an :class:`FftShiftOp` — move the center frequency back to index 0 (``torch.fft.ifftshift``). + + The exact inverse of :class:`FftShiftOp` (the two coincide for even-length dims but differ for + odd-length ones, which is why both exist). Like its sibling it is a pure, dtype- and + shape-preserving rearrangement that accepts any tensor. Apply it before :class:`InverseFourierOp` + to recover the natural FFT bin order (``InverseFourierOp(shift=True)`` folds it in). + + Args: + dim: Dimension to shift. Default ``-1`` (last dim, matches :class:`InverseFourierOp`); ``None`` = all dims. + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH) + + def __init__(self, dim: Optional[int] = -1) -> None: + self.dim = dim + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"IfftShiftOp expects a torch.Tensor, got {type(tensor).__name__}") + return sample._replace(input=torch.fft.ifftshift(tensor, dim=self.dim)) + + +@configurable(category="op", group="torch") +class WindowOp: + """Apply a window taper to ``sample.input`` and record the unit-scaling correction (torch mirror). + + The tensor counterpart of :class:`dataflux.ops.numpy.WindowOp`: multiplies the signal by a + :func:`dataflux.windows.get_window` taper (broadcast along ``dim``) and stashes the window + correction (``window`` / ``window_sum`` ``S1`` / ``window_sum_sq`` ``S2`` / ``window_enbw_bins`` / + ``window_coherent_gain``) into ``sample.metadata`` for a later :class:`SpectrumScalingOp`. + dtype/device-preserving — real stays real, complex stays complex. + + Args: + window: Which taper — a ``WindowName`` (default ``"hann"``; ``"boxcar"`` is the rectangular identity). + window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. + periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. + dim: Dimension the window is applied along. Default ``-1`` (the last dim — the 1-D signal). + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH) + + def __init__( + self, + window: WindowName = "hann", + window_param: Optional[float] = None, + periodic: bool = True, + dim: int = -1, + ) -> None: + # Lazy / zero-arg: store config only; window params validated lazily by get_window. + self.window: WindowName = window + self.window_param = window_param + self.periodic = bool(periodic) + self.dim = dim + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"WindowOp expects a torch.Tensor, got {type(tensor).__name__}") + window = get_window(self.window, tensor.shape[self.dim], window_param=self.window_param, periodic=self.periodic) + out = _apply_window(tensor, window, self.dim) + if sample.is_batched: + return sample._replace(input=out) + new_meta = dict(sample.meta) + new_meta.update(window_metadata(self.window, window)) + return sample._replace(input=out, metadata=new_meta) + + +@configurable(category="op", group="torch") +class SpectrumScalingOp: + """Scale a (complex) FFT spectrum to physical units using the window correction (torch mirror). + + The tensor counterpart of :class:`dataflux.ops.numpy.SpectrumScalingOp`: amplitude (V) / power + (V²) / density (V²/Hz), dividing out the window ``S1``/``S2`` read from the ``window_*`` metadata + (rectangular ``S1=S2=N`` if absent). Assumes the spectrum came from the unscaled forward transform + (``norm="backward"``). Output is complex for ``"none"``/``"amplitude"``, real for + ``"power"``/``"density"``. + + Args: + scaling: Units — ``"none"``, ``"amplitude"`` V, ``"power"`` V² (default), ``"density"`` V²/Hz. + sample_rate: Hz, for ``"density"``. ``None`` (default) reads ``metadata["samplerate"]``, else ``1.0``. + one_sided: Fold to one-sided (real-signal convention: keep 0…N/2, double interior bins). Default ``False``. + dim: Spectrum dimension. Default ``-1``. + """ + + ACCEPTS = SampleType(input=_TORCH) + PRODUCES = SampleType(input=_TORCH_COMPLEX_OR_FLOAT) + + def __init__( + self, + scaling: SpectrumScaling = "power", + sample_rate: Optional[float] = None, + one_sided: bool = False, + dim: int = -1, + ) -> None: + # Lazy / zero-arg: store config only. + self.scaling: SpectrumScaling = scaling + self.sample_rate = sample_rate + self.one_sided = bool(one_sided) + self.dim = dim + + def __call__(self, sample: Sample) -> Sample: + tensor = sample.input + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"SpectrumScalingOp expects a torch.Tensor, got {type(tensor).__name__}") + n = tensor.shape[self.dim] + if sample.is_batched: + s1 = s2 = float(n) + else: + meta = sample.meta + raw_s1, raw_s2 = meta.get(WINDOW_SUM_KEY), meta.get(WINDOW_SUMSQ_KEY) + s1, s2 = ( + (float(raw_s1), float(raw_s2)) if raw_s1 is not None and raw_s2 is not None else (float(n), float(n)) + ) + out = _scale_spectrum( + tensor, + self.scaling, + s1=s1, + s2=s2, + sample_rate=_resolve_fft_sample_rate(sample, self.sample_rate), + one_sided=self.one_sided, + dim=self.dim, + ) + return sample._replace(input=out) diff --git a/dataflux/storage/directory.py b/dataflux/storage/directory.py index d760d63..87b98ad 100644 --- a/dataflux/storage/directory.py +++ b/dataflux/storage/directory.py @@ -8,7 +8,8 @@ from dataflux.storage.base import DataSink, Storage -@confluid.configurable +# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). +@confluid.configurable(category="sink") class DirectorySink(Storage, DataSink): """ High-concurrency sink that stores each Sample in its own directory. diff --git a/dataflux/storage/hdf5.py b/dataflux/storage/hdf5.py index ffe3c96..9c17bbf 100644 --- a/dataflux/storage/hdf5.py +++ b/dataflux/storage/hdf5.py @@ -67,7 +67,8 @@ def __len__(self) -> int: return len([k for k in self._file.keys() if k.endswith("_data")]) -@configurable +# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). +@configurable(category="sink") class HDF5Sink(Storage, DataSink): """High-performance HDF5 data sink focused on Sample triplets.""" diff --git a/dataflux/storage/zarr.py b/dataflux/storage/zarr.py index b8bab30..b4343f5 100644 --- a/dataflux/storage/zarr.py +++ b/dataflux/storage/zarr.py @@ -10,7 +10,8 @@ from dataflux.storage.base import DataSink, DataSource, Storage, to_numpy -@confluid.configurable +# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). +@confluid.configurable(category="sink") class ZarrGroupSink(Storage, DataSink): """ Stores each sample as a unique array within a Zarr group. @@ -110,7 +111,8 @@ def __len__(self) -> int: return len(list(self._root.group_keys())) -@confluid.configurable +# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). +@confluid.configurable(category="sink") class ZarrBatchSink(Storage, DataSink): """ Optimized for uniform data. Appends samples into a single large Zarr array. diff --git a/dataflux/windows.py b/dataflux/windows.py new file mode 100644 index 0000000..8a22b3d --- /dev/null +++ b/dataflux/windows.py @@ -0,0 +1,261 @@ +"""Window functions + spectral unit-scaling — the math home for the Fourier ops. + +A raw FFT is *uncalibrated*: to read a spectrum in real units you must (1) taper the +signal with a window to control spectral leakage and (2) divide out the window's gain. +This module is the single, framework-neutral (pure-numpy — scipy is only an optional +dataflux dependency) source of both: + +* :func:`get_window` builds the taper (``WindowName`` — Hann, Hamming, Blackman-Harris, + flat-top, Kaiser, …). +* :func:`window_sums` / :func:`coherent_gain` / :func:`enbw_bins` give the correction + factors — coherent gain ``S1 = Σw`` (amplitude) and ``S2 = Σw²`` with the equivalent + noise bandwidth (power-spectral density). +* :func:`scale_spectrum` turns a windowed FFT into the chosen ``SpectrumScaling`` units + (amplitude V, power V², density V²/Hz). + +It is a library module (like :mod:`dataflux.labels` / :mod:`dataflux.projection`), **not** +``@configurable`` and not entry-pointed. The numpy ops in :mod:`dataflux.ops.numpy` +(``WindowOp`` / ``SpectrumScalingOp`` / ``FourierOp``) and their torch mirrors in +:mod:`dataflux.ops.torch` all reuse it — the torch ops take the numpy window coefficients +and the scalar ``S1``/``S2`` corrections, then do the array arithmetic with torch. + +Calibration assumes the **unscaled forward transform** (``numpy.fft.fft`` / +``torch.fft.fft`` with ``norm="backward"`` — the default). The amplitude/power/density +formulas are only meaningful for that normalization, so the ops reject a non-``backward`` +``norm`` combined with a unit ``scaling`` rather than emit silently-wrong units. +""" + +from typing import Dict, Literal, Optional, Tuple, get_args + +import numpy as np + +# --- closed Literals (workspace "prefer closed Literals over bare strings" mandate) --- +# The supported window tapers. ``boxcar`` is the rectangular window (all ones) — i.e. *no* +# taper, the identity — and is the default for FourierOp so its behaviour is unchanged. +WindowName = Literal[ + "boxcar", + "bartlett", + "hann", + "hamming", + "blackman", + "blackmanharris", + "nuttall", + "flattop", + "kaiser", + "tukey", + "gaussian", +] +WINDOW_NAMES: Tuple[WindowName, ...] = get_args(WindowName) + +# Spectral unit-scaling modes. ``none`` = raw FFT (complex, unchanged); ``amplitude`` = +# amplitude spectrum (V, complex); ``power`` = power spectrum (V², real); ``density`` = +# power spectral density (V²/Hz, real). NB this is the GENERAL scaling set — distinct from +# the narrower matplotlib-style ``waivefront.visualizers.SpectrumScaling`` (density/spectrum), +# which is a different module modelling matplotlib's ``scale_by_freq`` toggle. +SpectrumScaling = Literal["none", "amplitude", "power", "density"] +SPECTRUM_SCALINGS: Tuple[SpectrumScaling, ...] = get_args(SpectrumScaling) + +# --- metadata keys: the window correction stashed by WindowOp / FourierOp (when a real +# window is applied) and read back by SpectrumScalingOp so a spectrum computed in one node +# can be scaled to units in another. ``window`` here is the FFT *taper* name — unrelated to +# waivefront's ``window_start_sample`` (a time-slice index). --- +WINDOW_NAME_KEY = "window" +WINDOW_SIZE_KEY = "window_size" # N (number of taps) +WINDOW_SUM_KEY = "window_sum" # S1 = Σw (coherent-gain numerator) +WINDOW_SUMSQ_KEY = "window_sum_sq" # S2 = Σw² +WINDOW_ENBW_KEY = "window_enbw_bins" # equivalent noise bandwidth, N·S2/S1² (bins) +WINDOW_CG_KEY = "window_coherent_gain" # S1/N + +# Generalized-cosine coefficients (scipy / Harris-1978 convention): w[n] = Σ_k a_k·cos(k·φ) +# with φ ∈ [-π, π] over the taps. The alternating shape is carried by cos(k·φ), so the +# coefficients are all positive and sum to 1 at the centre (coherent gain ≈ a_0). +_COSINE_COEFFS: Dict[str, Tuple[float, ...]] = { + "hann": (0.5, 0.5), + "hamming": (0.54, 0.46), + "blackman": (0.42, 0.5, 0.08), + "blackmanharris": (0.35875, 0.48829, 0.14128, 0.01168), + "nuttall": (0.3635819, 0.4891775, 0.1365995, 0.0106411), + "flattop": (0.21557895, 0.41663158, 0.277263158, 0.083578947, 0.006947368), +} + + +def _general_cosine(n: int, coeffs: Tuple[float, ...], periodic: bool) -> np.ndarray: + """Generalized-cosine window of length ``n`` (the Hann/Hamming/Blackman/… family). + + ``periodic=True`` (the DFT-even form correct for FFT spectral analysis) builds the + symmetric window of length ``n+1`` and drops the last sample; ``periodic=False`` is the + plain symmetric window (zero — or near-zero — at both endpoints). + """ + m = n + 1 if periodic else n + fac = np.linspace(-np.pi, np.pi, m) + w = np.zeros(m, dtype=np.float64) + for k, a in enumerate(coeffs): + w = w + a * np.cos(k * fac) + return w[:-1] if periodic else w + + +def _tukey(n: int, alpha: float, periodic: bool) -> np.ndarray: + """Tukey (tapered-cosine) window — ``alpha`` is the cosine-tapered fraction in [0, 1].""" + if alpha <= 0: + return np.ones(n, dtype=np.float64) + if alpha >= 1: + return _general_cosine(n, (0.5, 0.5), periodic) # full cosine taper == Hann + m = n + 1 if periodic else n + idx = np.arange(0, m) + width = int(np.floor(alpha * (m - 1) / 2.0)) + w = np.ones(m, dtype=np.float64) + n1 = idx[: width + 1] + n3 = idx[m - width - 1 :] + w[: width + 1] = 0.5 * (1 + np.cos(np.pi * (-1 + 2.0 * n1 / alpha / (m - 1)))) + w[m - width - 1 :] = 0.5 * (1 + np.cos(np.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / (m - 1)))) + return w[:-1] if periodic else w + + +def _gaussian(n: int, std: float, periodic: bool) -> np.ndarray: + """Gaussian window — ``std`` is the standard deviation in samples (must be > 0).""" + if std <= 0: + raise ValueError(f"gaussian window std must be > 0; got {std!r}") + m = n + 1 if periodic else n + k = np.arange(0, m) - (m - 1) / 2.0 + w = np.exp(-0.5 * (k / std) ** 2) + return np.asarray(w[:-1] if periodic else w, dtype=np.float64) + + +def get_window( + window: WindowName, n: int, *, window_param: Optional[float] = None, periodic: bool = True +) -> np.ndarray: + """Build a length-``n`` window taper as a ``float64`` ndarray. + + Args: + window: Which taper — one of ``WindowName`` (``boxcar`` is the rectangular identity). + n: Number of taps (must be positive); normally the signal length being transformed. + window_param: Shape parameter for the parametrized windows — Kaiser ``β`` (default 8.6), + Tukey ``α`` taper fraction in [0, 1] (default 0.5), or Gaussian ``σ`` std in samples + (required, no default). Ignored by the fixed windows. + periodic: ``True`` (default) = DFT-even window (the correct form for FFT spectral + analysis); ``False`` = symmetric window (zero at both endpoints). + + Returns: + The window coefficients, ``float64``, shape ``(n,)``. + + Raises: + ValueError: unknown ``window``, non-positive ``n``, or a Gaussian without ``window_param``. + """ + if window not in WINDOW_NAMES: + raise ValueError(f"unknown window {window!r}; valid: {WINDOW_NAMES}") + if n <= 0: + raise ValueError(f"window length n must be positive; got {n!r}") + if n == 1: + return np.ones(1, dtype=np.float64) + if window == "boxcar": + return np.ones(n, dtype=np.float64) + if window in _COSINE_COEFFS: + return _general_cosine(n, _COSINE_COEFFS[window], periodic) + if window == "bartlett": + m = n + 1 if periodic else n + w = np.bartlett(m) + return (w[:-1] if periodic else w).astype(np.float64) + if window == "kaiser": + beta = 8.6 if window_param is None else float(window_param) + m = n + 1 if periodic else n + w = np.kaiser(m, beta) + return (w[:-1] if periodic else w).astype(np.float64) + if window == "tukey": + alpha = 0.5 if window_param is None else float(window_param) + return _tukey(n, alpha, periodic) + # window == "gaussian" + if window_param is None: + raise ValueError("gaussian window requires window_param (std in samples)") + return _gaussian(n, float(window_param), periodic) + + +def window_sums(window: np.ndarray) -> Tuple[float, float]: + """Return ``(S1, S2)`` = ``(Σw, Σw²)`` — the two sums the unit corrections need.""" + w = np.asarray(window, dtype=np.float64) + return float(w.sum()), float(np.square(w).sum()) + + +def coherent_gain(window: np.ndarray) -> float: + """Coherent gain ``S1/N`` — the amplitude attenuation the window applies to a tone.""" + w = np.asarray(window, dtype=np.float64) + return float(w.sum() / w.size) + + +def enbw_bins(window: np.ndarray) -> float: + """Equivalent noise bandwidth ``N·S2/S1²`` in **bins** (e.g. ≈1.5 for Hann).""" + s1, s2 = window_sums(window) + return float(np.asarray(window).size * s2 / (s1 * s1)) + + +def window_metadata(window_name: str, window: np.ndarray) -> Dict[str, object]: + """Build the window-correction metadata dict (the keys ``SpectrumScalingOp`` reads).""" + s1, s2 = window_sums(window) + n = int(np.asarray(window).size) + return { + WINDOW_NAME_KEY: window_name, + WINDOW_SIZE_KEY: n, + WINDOW_SUM_KEY: s1, + WINDOW_SUMSQ_KEY: s2, + WINDOW_ENBW_KEY: float(n * s2 / (s1 * s1)), + WINDOW_CG_KEY: float(s1 / n), + } + + +def fold_one_sided(spectrum: np.ndarray, axis: int) -> np.ndarray: + """Fold a two-sided spectrum (natural FFT order, DC at index 0) to one-sided. + + Keeps bins ``0 … N//2`` and doubles the interior bins (everything except DC and, for + even ``N``, the Nyquist bin) so a real signal's one-sided amplitude/power reads its true + value. Meaningful only for spectra of **real** inputs in natural (un-``fftshift``ed) order. + """ + moved = np.swapaxes(np.asarray(spectrum), axis, -1) + n = moved.shape[-1] + out = moved[..., : n // 2 + 1].copy() + if n % 2 == 0: + out[..., 1:-1] = out[..., 1:-1] * 2 # exclude DC (0) and Nyquist (-1) + else: + out[..., 1:] = out[..., 1:] * 2 # no Nyquist bin for odd N + return np.swapaxes(out, axis, -1) + + +def scale_spectrum( + spectrum: np.ndarray, + scaling: SpectrumScaling, + *, + s1: float, + s2: float, + sample_rate: Optional[float] = None, + one_sided: bool = False, + axis: int = -1, +) -> np.ndarray: + """Scale a windowed FFT spectrum to the chosen units (assumes ``norm="backward"``). + + Args: + spectrum: The complex FFT output (windowed, unscaled forward transform). + scaling: ``none`` (complex, unchanged) · ``amplitude`` (V, complex, ``X/S1``) · + ``power`` (V², real, ``|X|²/S1²``) · ``density`` (V²/Hz, real, ``|X|²/(Fs·S2)``). + s1: Window coherent-gain sum ``Σw`` (use ``N`` for an unwindowed / boxcar spectrum). + s2: Window squared sum ``Σw²`` (use ``N`` for boxcar). + sample_rate: ``Fs`` in Hz for ``density`` (V²/Hz). ``None`` / ≤0 → ``1.0`` (density + per normalized frequency, V² per cycle/sample). Ignored by other modes. + one_sided: Fold to a one-sided spectrum (real-input convention) after scaling. + axis: Transform axis (for ``one_sided`` folding and the bin count). + + Returns: + The scaled spectrum — complex for ``none``/``amplitude``, real for ``power``/``density``. + """ + x = np.asarray(spectrum) + if scaling == "none": + out = x + elif scaling == "amplitude": + out = x / s1 + elif scaling == "power": + out = np.square(np.abs(x)) / (s1 * s1) + elif scaling == "density": + fs = float(sample_rate) if (sample_rate is not None and sample_rate > 0) else 1.0 + out = np.square(np.abs(x)) / (fs * s2) + else: + raise ValueError(f"unknown scaling {scaling!r}; valid: {SPECTRUM_SCALINGS}") + if one_sided: + out = fold_one_sided(out, axis) + return out diff --git a/pyproject.toml b/pyproject.toml index 1ee0a54..fbdede5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,9 @@ dataflux-ops-random-apply = "dataflux.ops.random_apply" # changes need an editable reinstall before FluxStudio/navigaitor discovery sees the module. dataflux-ops-configure = "dataflux.ops.configure" dataflux-ops-formula = "dataflux.ops.formula" +# CaptureOutputOp (records an op's @output value into metadata — the capture half of +# FluxStudio's op-@output -> param wiring, paired with ConfigureOp(UnstashInputOp)). +dataflux-ops-capture = "dataflux.ops.capture" dataflux-ops-transform-chain = "dataflux.ops.transform_chain" dataflux-ops-sink = "dataflux.ops.sink" dataflux-ops-stash = "dataflux.ops.stash" @@ -69,6 +72,15 @@ dataflux-ops-copy = "dataflux.ops.copy" dataflux-ops-swap = "dataflux.ops.swap" dataflux-ops-target = "dataflux.ops.target" dataflux-ops-image = "dataflux.ops.image" +# Storage SINKS (HDF5Sink / ZarrGroupSink / ZarrBatchSink / DirectorySink) carry +# category="sink" so FluxStudio surfaces them as DatasetProcessor sink nodes. They live under +# dataflux.storage.* (NOT re-exported from the package root), and scan_module does not recurse +# submodules, so each storage module needs its own entry point. The matching SOURCES in these +# modules stay uncategorised, so the positive {op,source,engine,sink} allowlist surfaces only the +# tagged sinks. (Entry-point changes need an editable reinstall — `aisland setup`, never --reinstall.) +dataflux-storage-hdf5 = "dataflux.storage.hdf5" +dataflux-storage-zarr = "dataflux.storage.zarr" +dataflux-storage-directory = "dataflux.storage.directory" [tool.setuptools.packages.find] where = ["."] diff --git a/tests/test_categories.py b/tests/test_categories.py index 2b00d17..0085f50 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,12 +10,23 @@ from confluid.registry import get_registry from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from dataflux.ops.capture import CaptureOutputOp from dataflux.ops.configure import ConfigureOp from dataflux.ops.copy import CopyInputOp from dataflux.ops.enable import Enable from dataflux.ops.formula import FormulaOp from dataflux.ops.image import ConvertToImageOp, NormalizeToUint8Op -from dataflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp +from dataflux.ops.numpy import ( + FftShiftOp, + FourierOp, + IfftShiftOp, + InverseFourierOp, + RescaleOp, + SpectrumScalingOp, + StandardizeOp, + ThresholdOp, + WindowOp, +) from dataflux.ops.parallel import Parallel from dataflux.ops.sink import SampleSinkOp from dataflux.ops.stash import StashTargetOp, UnstashTargetOp @@ -27,9 +38,18 @@ MetadataToTargetOp, ) from dataflux.ops.tee import Tee +from dataflux.ops.torch import FftShiftOp as TorchFftShiftOp +from dataflux.ops.torch import FourierOp as TorchFourierOp +from dataflux.ops.torch import IfftShiftOp as TorchIfftShiftOp +from dataflux.ops.torch import InverseFourierOp as TorchInverseFourierOp +from dataflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp from dataflux.ops.torch import ToTensorOp +from dataflux.ops.torch import WindowOp as TorchWindowOp from dataflux.ops.transform_chain import TransformChain from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource +from dataflux.storage.directory import DirectorySink +from dataflux.storage.hdf5 import HDF5Sink, HDF5Source +from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink def test_engine_classes_tagged() -> None: @@ -72,6 +92,18 @@ def test_op_classes_tagged() -> None: assert RescaleOp.__confluid_category__ == "op" assert StandardizeOp.__confluid_category__ == "op" assert ThresholdOp.__confluid_category__ == "op" + assert FourierOp.__confluid_category__ == "op" + assert TorchFourierOp.__confluid_category__ == "op" + assert InverseFourierOp.__confluid_category__ == "op" + assert TorchInverseFourierOp.__confluid_category__ == "op" + assert FftShiftOp.__confluid_category__ == "op" + assert TorchFftShiftOp.__confluid_category__ == "op" + assert IfftShiftOp.__confluid_category__ == "op" + assert TorchIfftShiftOp.__confluid_category__ == "op" + assert WindowOp.__confluid_category__ == "op" + assert TorchWindowOp.__confluid_category__ == "op" + assert SpectrumScalingOp.__confluid_category__ == "op" + assert TorchSpectrumScalingOp.__confluid_category__ == "op" assert Tee.__confluid_category__ == "op" assert Enable.__confluid_category__ == "op" assert TransformChain.__confluid_category__ == "op" @@ -81,6 +113,22 @@ def test_op_classes_tagged() -> None: assert DecodeTargetOp.__confluid_category__ == "op" assert CocoToTorchVisionDetectionOp.__confluid_category__ == "op" assert MasksToDetectionBoxesOp.__confluid_category__ == "op" + assert ConfigureOp.__confluid_category__ == "op" + assert FormulaOp.__confluid_category__ == "op" + assert CaptureOutputOp.__confluid_category__ == "op" + + +def test_storage_sink_classes_tagged() -> None: + """The DataFlux storage SINKS carry ``category="sink"`` so FluxStudio surfaces them as + ``DatasetProcessor`` sink nodes (``DATAFLUX_OBJECT:sink``). Their matching SOURCES stay + UNcategorised — they read a sink's layout back via YAML ``!class:``, they are not canvas nodes. + (``SampleSinkOp`` is the op-FORM sink, ``category="op"`` — a different thing, asserted above.)""" + assert HDF5Sink.__confluid_category__ == "sink" + assert ZarrGroupSink.__confluid_category__ == "sink" + assert ZarrBatchSink.__confluid_category__ == "sink" + assert DirectorySink.__confluid_category__ == "sink" + # The matching source is NOT tagged, so the positive allowlist surfaces only the sink half. + assert getattr(HDF5Source, "__confluid_category__", None) is None def test_op_group_tags() -> None: @@ -91,7 +139,19 @@ def test_op_group_tags() -> None: assert RescaleOp.__confluid_group__ == "numpy" assert StandardizeOp.__confluid_group__ == "numpy" assert ThresholdOp.__confluid_group__ == "numpy" + assert FourierOp.__confluid_group__ == "numpy" + assert InverseFourierOp.__confluid_group__ == "numpy" + assert FftShiftOp.__confluid_group__ == "numpy" + assert IfftShiftOp.__confluid_group__ == "numpy" assert ToTensorOp.__confluid_group__ == "torch" + assert TorchFourierOp.__confluid_group__ == "torch" + assert TorchInverseFourierOp.__confluid_group__ == "torch" + assert TorchFftShiftOp.__confluid_group__ == "torch" + assert TorchIfftShiftOp.__confluid_group__ == "torch" + assert WindowOp.__confluid_group__ == "numpy" + assert SpectrumScalingOp.__confluid_group__ == "numpy" + assert TorchWindowOp.__confluid_group__ == "torch" + assert TorchSpectrumScalingOp.__confluid_group__ == "torch" assert CopyInputOp.__confluid_group__ == "structure" assert StashTargetOp.__confluid_group__ == "structure" assert UnstashTargetOp.__confluid_group__ == "structure" @@ -106,6 +166,7 @@ def test_op_group_tags() -> None: assert TransformChain.__confluid_group__ == "compose" assert ConfigureOp.__confluid_group__ == "compose" assert FormulaOp.__confluid_group__ == "compose" + assert CaptureOutputOp.__confluid_group__ == "compose" assert ConvertToImageOp.__confluid_group__ == "image" assert NormalizeToUint8Op.__confluid_group__ == "image" assert SampleSinkOp.__confluid_group__ == "sink" @@ -130,6 +191,10 @@ def test_categories_enumerable_via_registry() -> None: "RescaleOp", "StandardizeOp", "ThresholdOp", + "FourierOp", + "InverseFourierOp", + "FftShiftOp", + "IfftShiftOp", "Tee", "Enable", "SampleSinkOp", @@ -140,12 +205,29 @@ def test_categories_enumerable_via_registry() -> None: "MasksToDetectionBoxesOp", "TransformChain", } <= registry.list_classes(category="op") + # The storage sinks surface under the NEW "sink" category index (FluxStudio's allowlist + the + # navigaitor sink picker). SampleSinkOp is category="op", so it is NOT here. + assert {"HDF5Sink", "ZarrGroupSink", "ZarrBatchSink", "DirectorySink"} <= registry.list_classes(category="sink") + assert "SampleSinkOp" not in registry.list_classes(category="sink") def test_groups_enumerable_via_registry() -> None: """The registry's group index must surface the tagged ops (``list_classes(group=...)``).""" registry = get_registry() - assert {"RescaleOp", "StandardizeOp", "ThresholdOp"} <= registry.list_classes(group="numpy") + assert { + "RescaleOp", + "StandardizeOp", + "ThresholdOp", + "FourierOp", + "InverseFourierOp", + "FftShiftOp", + "IfftShiftOp", + } <= registry.list_classes(group="numpy") + # The FFT ops exist in BOTH framework groups (a numpy + a torch variant under the one name, + # exactly like RescaleOp/StandardizeOp), so they surface under the torch group too. + assert {"ToTensorOp", "FourierOp", "InverseFourierOp", "FftShiftOp", "IfftShiftOp"} <= registry.list_classes( + group="torch" + ) assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") assert {"Tee", "Parallel", "Enable", "TransformChain"} <= registry.list_classes(group="compose") assert {"SampleSinkOp"} <= registry.list_classes(group="sink") diff --git a/tests/test_fourier_ops.py b/tests/test_fourier_ops.py new file mode 100644 index 0000000..9f07cda --- /dev/null +++ b/tests/test_fourier_ops.py @@ -0,0 +1,655 @@ +"""Tests for the 1-D Fourier-transform ops: ``dataflux.ops.numpy.FourierOp`` and +``dataflux.ops.torch.FourierOp``. + +Both compute the 1-D DFT (``numpy.fft.fft`` / ``torch.fft.fft``) of ``sample.input`` and +ALWAYS yield a complex result — for real and complex inputs alike. The tests pin: the +real/complex/integer dtype-promotion rules, the round-trip against the inverse transform, +the ``n`` / ``axis``-``dim`` / ``norm`` parameters, the framework type guards, the +``ACCEPTS``/``PRODUCES`` contract conformance, and the closed-``Literal`` ``norm`` validation. +""" + +import numpy as np +import pytest +import torch +from pydantic import ValidationError + +from dataflux.ops import FftShiftOp as FlatFftShiftOp +from dataflux.ops import FourierOp as FlatFourierOp +from dataflux.ops import IfftShiftOp as FlatIfftShiftOp +from dataflux.ops import InverseFourierOp as FlatInverseFourierOp +from dataflux.ops.numpy import FftShiftOp as NpFftShiftOp +from dataflux.ops.numpy import FourierNorm +from dataflux.ops.numpy import FourierOp as NpFourierOp +from dataflux.ops.numpy import IfftShiftOp as NpIfftShiftOp +from dataflux.ops.numpy import InverseFourierOp as NpInverseFourierOp +from dataflux.ops.numpy import SpectrumScalingOp as NpSpectrumScalingOp +from dataflux.ops.numpy import WindowOp as NpWindowOp +from dataflux.ops.torch import FftShiftOp as TorchFftShiftOp +from dataflux.ops.torch import FourierOp as TorchFourierOp +from dataflux.ops.torch import IfftShiftOp as TorchIfftShiftOp +from dataflux.ops.torch import InverseFourierOp as TorchInverseFourierOp +from dataflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp +from dataflux.ops.torch import WindowOp as TorchWindowOp +from dataflux.sample import Sample +from dataflux.typespec import infer_sample_type +from dataflux.windows import WINDOW_SUM_KEY + + +def test_flat_imports_are_torch_variants() -> None: + """``from dataflux.ops import …`` resolves the FFT ops to their torch variants — the package's + documented convention that flat data-op imports default to torch (mirrors RescaleOp etc.).""" + assert FlatFourierOp is TorchFourierOp + assert FlatInverseFourierOp is TorchInverseFourierOp + assert FlatFftShiftOp is TorchFftShiftOp + assert FlatIfftShiftOp is TorchIfftShiftOp + + +# --------------------------------------------------------------------------- +# numpy FourierOp +# --------------------------------------------------------------------------- + + +class TestNumpyFourierOp: + def test_real_float32_matches_numpy_and_is_complex64(self) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + out = NpFourierOp()(Sample(input=x)) + assert isinstance(out.input, np.ndarray) + assert out.input.dtype == np.complex64 + assert np.allclose(out.input, np.fft.fft(x)) + + def test_real_float64_promotes_to_complex128(self) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float64) + out = NpFourierOp()(Sample(input=x)) + assert out.input.dtype == np.complex128 + assert np.allclose(out.input, np.fft.fft(x)) + + def test_integer_input_promotes_to_complex128(self) -> None: + x = np.arange(8, dtype=np.int64) + out = NpFourierOp()(Sample(input=x)) + assert out.input.dtype == np.complex128 + assert np.allclose(out.input, np.fft.fft(x)) + + def test_complex64_input_stays_complex64(self) -> None: + x = np.array([1 + 2j, 3 - 1j, 0 + 0j, -2 + 1j], dtype=np.complex64) + out = NpFourierOp()(Sample(input=x)) + assert out.input.dtype == np.complex64 + assert np.allclose(out.input, np.fft.fft(x)) + + def test_complex128_input_stays_complex128(self) -> None: + x = np.array([1 + 2j, 3 - 1j, 0 + 0j, -2 + 1j], dtype=np.complex128) + out = NpFourierOp()(Sample(input=x)) + assert out.input.dtype == np.complex128 + + def test_constant_signal_has_only_dc_component(self) -> None: + # FFT of a length-4 constant [1,1,1,1] is [4, 0, 0, 0] (all energy in the DC bin). + out = NpFourierOp()(Sample(input=np.ones(4, dtype=np.float64))) + assert np.allclose(out.input, np.array([4, 0, 0, 0])) + + def test_roundtrip_via_ifft_recovers_input(self) -> None: + x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) + out = NpFourierOp()(Sample(input=x)) + recovered = np.fft.ifft(out.input) + assert np.allclose(recovered.real, x, atol=1e-9) + + def test_n_zero_pads(self) -> None: + x = np.arange(4, dtype=np.float64) + out = NpFourierOp(n=8)(Sample(input=x)) + assert out.input.shape == (8,) + assert np.allclose(out.input, np.fft.fft(x, n=8)) + + def test_n_truncates(self) -> None: + x = np.arange(8, dtype=np.float64) + out = NpFourierOp(n=4)(Sample(input=x)) + assert out.input.shape == (4,) + assert np.allclose(out.input, np.fft.fft(x, n=4)) + + def test_axis_transforms_per_row_of_batch(self) -> None: + x = np.random.RandomState(0).randn(3, 8) + out = NpFourierOp(axis=-1)(Sample(input=x)) + assert out.input.shape == (3, 8) + # Each row transformed independently == the per-row 1-D FFT. + for i in range(3): + assert np.allclose(out.input[i], np.fft.fft(x[i])) + + def test_axis_zero(self) -> None: + x = np.random.RandomState(1).randn(8, 3) + out = NpFourierOp(axis=0)(Sample(input=x)) + assert np.allclose(out.input, np.fft.fft(x, axis=0)) + + @pytest.mark.parametrize("norm", ["backward", "ortho", "forward"]) + def test_norm_modes_match_numpy(self, norm: FourierNorm) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float64) + out = NpFourierOp(norm=norm)(Sample(input=x)) + assert np.allclose(out.input, np.fft.fft(x, norm=norm)) + + def test_shift_flag_matches_manual_fftshift(self) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) # odd length — fftshift is non-trivial + shifted = NpFourierOp(shift=True)(Sample(input=x)).input + assert np.allclose(shifted, np.fft.fftshift(NpFourierOp()(Sample(input=x)).input)) + + def test_shift_defaults_off(self) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0]) + plain = NpFourierOp()(Sample(input=x)).input + assert np.allclose(NpFourierOp(shift=False)(Sample(input=x)).input, plain) + assert not np.allclose(NpFourierOp(shift=True)(Sample(input=x)).input, plain) + + def test_preserves_target_and_metadata(self) -> None: + out = NpFourierOp()(Sample(input=np.ones(4), target=5, metadata={"k": "v"})) + assert out.target == 5 + assert out.meta == {"k": "v"} + + def test_raises_on_non_ndarray(self) -> None: + with pytest.raises(TypeError, match="FourierOp expects an np.ndarray"): + NpFourierOp()(Sample(input=torch.zeros(4))) + + def test_zero_arg_construction(self) -> None: + op = NpFourierOp() + assert op.n is None and op.axis == -1 and op.norm == "backward" + + def test_invalid_norm_rejected_at_construction(self) -> None: + # ``norm`` is a closed ``Literal`` — confluid's pydantic schema rejects an out-of-set value. + with pytest.raises((ValueError, ValidationError)): + NpFourierOp(norm="bogus") # type: ignore[arg-type] + + def test_produces_contract_conforms_to_real_output(self) -> None: + out = NpFourierOp()(Sample(input=np.ones(4, dtype=np.float32))) + assert NpFourierOp.PRODUCES.accepts(infer_sample_type(out)) + + def test_produces_contract_conforms_to_complex_output(self) -> None: + x = np.array([1 + 2j, 3 - 1j], dtype=np.complex128) + out = NpFourierOp()(Sample(input=x)) + assert NpFourierOp.PRODUCES.accepts(infer_sample_type(out)) + + +# --------------------------------------------------------------------------- +# torch FourierOp +# --------------------------------------------------------------------------- + + +class TestTorchFourierOp: + def test_real_float32_matches_torch_and_is_complex64(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0]) + out = TorchFourierOp()(Sample(input=t)) + assert isinstance(out.input, torch.Tensor) + assert out.input.dtype == torch.complex64 + assert torch.allclose(out.input, torch.fft.fft(t)) + + def test_real_float64_promotes_to_complex128(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float64) + out = TorchFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex128 + assert torch.allclose(out.input, torch.fft.fft(t)) + + def test_integer_input_handled_natively(self) -> None: + # torch.fft.fft auto-promotes integer tensors to complex64 — no manual cast needed. + t = torch.arange(8) + out = TorchFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + assert torch.allclose(out.input, torch.fft.fft(t)) + + def test_bool_input_handled_natively(self) -> None: + t = torch.tensor([True, False, True, True]) + out = TorchFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + + def test_complex64_input_stays_complex64(self) -> None: + t = torch.tensor([1 + 2j, 3 - 1j, 0 + 0j, -2 + 1j], dtype=torch.complex64) + out = TorchFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + assert torch.allclose(out.input, torch.fft.fft(t)) + + def test_float16_promoted_to_float32_without_mutating_input(self) -> None: + # torch.fft.fft rejects half precision; the op promotes to float32 first. The promotion + # is local — the caller's tensor is untouched. + t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float16) + out = TorchFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + assert t.dtype == torch.float16 + assert torch.allclose(out.input, torch.fft.fft(t.float())) + + def test_bfloat16_promoted_to_float32(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.bfloat16) + out = TorchFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + + def test_constant_signal_has_only_dc_component(self) -> None: + out = TorchFourierOp()(Sample(input=torch.ones(4))) + assert torch.allclose(out.input, torch.tensor([4, 0, 0, 0], dtype=torch.complex64)) + + def test_roundtrip_via_ifft_recovers_input(self) -> None: + t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) + out = TorchFourierOp()(Sample(input=t)) + recovered = torch.fft.ifft(out.input) + assert torch.allclose(recovered.real, t, atol=1e-9) + + def test_n_zero_pads(self) -> None: + t = torch.arange(4, dtype=torch.float64) + out = TorchFourierOp(n=8)(Sample(input=t)) + assert out.input.shape == (8,) + assert torch.allclose(out.input, torch.fft.fft(t, n=8)) + + def test_n_truncates(self) -> None: + t = torch.arange(8, dtype=torch.float64) + out = TorchFourierOp(n=4)(Sample(input=t)) + assert out.input.shape == (4,) + + def test_dim_transforms_per_row_of_batch(self) -> None: + t = torch.randn(3, 8) + out = TorchFourierOp(dim=-1)(Sample(input=t)) + assert out.input.shape == (3, 8) + for i in range(3): + assert torch.allclose(out.input[i], torch.fft.fft(t[i])) + + def test_dim_zero(self) -> None: + t = torch.randn(8, 3) + out = TorchFourierOp(dim=0)(Sample(input=t)) + assert torch.allclose(out.input, torch.fft.fft(t, dim=0)) + + @pytest.mark.parametrize("norm", ["backward", "ortho", "forward"]) + def test_norm_modes_match_torch(self, norm: FourierNorm) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float64) + out = TorchFourierOp(norm=norm)(Sample(input=t)) + assert torch.allclose(out.input, torch.fft.fft(t, norm=norm)) + + def test_shift_flag_matches_manual_fftshift(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0]) # odd length — fftshift is non-trivial + shifted = TorchFourierOp(shift=True)(Sample(input=t)).input + assert torch.allclose(shifted, torch.fft.fftshift(TorchFourierOp()(Sample(input=t)).input)) + + def test_shift_defaults_off(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0]) + plain = TorchFourierOp()(Sample(input=t)).input + assert torch.allclose(TorchFourierOp(shift=False)(Sample(input=t)).input, plain) + assert not torch.allclose(TorchFourierOp(shift=True)(Sample(input=t)).input, plain) + + def test_preserves_target_and_metadata(self) -> None: + out = TorchFourierOp()(Sample(input=torch.ones(4), target=7, metadata={"k": "v"})) + assert out.target == 7 + assert out.meta == {"k": "v"} + + def test_raises_on_non_tensor(self) -> None: + with pytest.raises(TypeError, match="FourierOp expects a torch.Tensor"): + TorchFourierOp()(Sample(input=np.zeros(4))) + + def test_zero_arg_construction(self) -> None: + op = TorchFourierOp() + assert op.n is None and op.dim == -1 and op.norm == "backward" + + def test_invalid_norm_rejected_at_construction(self) -> None: + with pytest.raises((ValueError, ValidationError)): + TorchFourierOp(norm="bogus") # type: ignore[arg-type] + + def test_produces_contract_conforms_to_real_output(self) -> None: + out = TorchFourierOp()(Sample(input=torch.ones(4))) + assert TorchFourierOp.PRODUCES.accepts(infer_sample_type(out)) + + def test_produces_contract_conforms_to_complex_output(self) -> None: + t = torch.tensor([1 + 2j, 3 - 1j], dtype=torch.complex128) + out = TorchFourierOp()(Sample(input=t)) + assert TorchFourierOp.PRODUCES.accepts(infer_sample_type(out)) + + +# --------------------------------------------------------------------------- +# numpy InverseFourierOp +# --------------------------------------------------------------------------- + + +class TestNumpyInverseFourierOp: + def test_matches_numpy_ifft_and_is_complex(self) -> None: + x = np.array([10.0, -2.0, 0.0, 4.0]) + out = NpInverseFourierOp()(Sample(input=x)) + assert out.input.dtype == np.complex128 + assert np.allclose(out.input, np.fft.ifft(x)) + + def test_inverts_forward_transform(self) -> None: + x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) # odd length + spectrum = NpFourierOp()(Sample(input=x)) + recovered = NpInverseFourierOp()(spectrum) + assert np.allclose(recovered.input.real, x, atol=1e-9) + + def test_shift_inverts_forward_shift(self) -> None: + # InverseFourierOp(shift=True) exactly undoes FourierOp(shift=True), odd length included. + x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) + centered = NpFourierOp(shift=True)(Sample(input=x)) + recovered = NpInverseFourierOp(shift=True)(centered) + assert np.allclose(recovered.input.real, x, atol=1e-9) + + @pytest.mark.parametrize("norm", ["backward", "ortho", "forward"]) + def test_roundtrip_under_each_norm(self, norm: FourierNorm) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float64) + spectrum = NpFourierOp(norm=norm)(Sample(input=x)) + recovered = NpInverseFourierOp(norm=norm)(spectrum) + assert np.allclose(recovered.input.real, x, atol=1e-9) + + def test_n_truncates(self) -> None: + x = np.arange(8, dtype=np.float64) + out = NpInverseFourierOp(n=4)(Sample(input=x)) + assert out.input.shape == (4,) + assert np.allclose(out.input, np.fft.ifft(x, n=4)) + + def test_preserves_target_and_metadata(self) -> None: + out = NpInverseFourierOp()(Sample(input=np.ones(4), target=5, metadata={"k": "v"})) + assert out.target == 5 + assert out.meta == {"k": "v"} + + def test_raises_on_non_ndarray(self) -> None: + with pytest.raises(TypeError, match="InverseFourierOp expects an np.ndarray"): + NpInverseFourierOp()(Sample(input=torch.zeros(4))) + + def test_zero_arg_construction(self) -> None: + op = NpInverseFourierOp() + assert op.n is None and op.axis == -1 and op.norm == "backward" and op.shift is False + + def test_invalid_norm_rejected_at_construction(self) -> None: + with pytest.raises((ValueError, ValidationError)): + NpInverseFourierOp(norm="bogus") # type: ignore[arg-type] + + def test_produces_contract_conforms(self) -> None: + out = NpInverseFourierOp()(Sample(input=np.ones(4))) + assert NpInverseFourierOp.PRODUCES.accepts(infer_sample_type(out)) + + +# --------------------------------------------------------------------------- +# torch InverseFourierOp +# --------------------------------------------------------------------------- + + +class TestTorchInverseFourierOp: + def test_matches_torch_ifft_and_is_complex(self) -> None: + t = torch.tensor([10.0, -2.0, 0.0, 4.0]) + out = TorchInverseFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + assert torch.allclose(out.input, torch.fft.ifft(t)) + + def test_inverts_forward_transform(self) -> None: + t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) + spectrum = TorchFourierOp()(Sample(input=t)) + recovered = TorchInverseFourierOp()(spectrum) + assert torch.allclose(recovered.input.real, t, atol=1e-9) + + def test_shift_inverts_forward_shift(self) -> None: + t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) + centered = TorchFourierOp(shift=True)(Sample(input=t)) + recovered = TorchInverseFourierOp(shift=True)(centered) + assert torch.allclose(recovered.input.real, t, atol=1e-9) + + def test_integer_input_handled_natively(self) -> None: + out = TorchInverseFourierOp()(Sample(input=torch.arange(8))) + assert out.input.dtype == torch.complex64 + + def test_float16_promoted_without_mutating_input(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float16) + out = TorchInverseFourierOp()(Sample(input=t)) + assert out.input.dtype == torch.complex64 + assert t.dtype == torch.float16 + + def test_raises_on_non_tensor(self) -> None: + with pytest.raises(TypeError, match="InverseFourierOp expects a torch.Tensor"): + TorchInverseFourierOp()(Sample(input=np.zeros(4))) + + def test_zero_arg_construction(self) -> None: + op = TorchInverseFourierOp() + assert op.n is None and op.dim == -1 and op.norm == "backward" and op.shift is False + + def test_produces_contract_conforms(self) -> None: + out = TorchInverseFourierOp()(Sample(input=torch.ones(4))) + assert TorchInverseFourierOp.PRODUCES.accepts(infer_sample_type(out)) + + +# --------------------------------------------------------------------------- +# fftshift / ifftshift ops (numpy + torch) — pure, dtype-preserving rearrangements +# --------------------------------------------------------------------------- + + +class TestNumpyShiftOps: + def test_fftshift_matches_numpy(self) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + assert np.allclose(NpFftShiftOp()(Sample(input=x)).input, np.fft.fftshift(x)) + + def test_ifftshift_matches_numpy(self) -> None: + x = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + assert np.allclose(NpIfftShiftOp()(Sample(input=x)).input, np.fft.ifftshift(x)) + + def test_ifftshift_inverts_fftshift_odd_length(self) -> None: + x = np.arange(5) + shifted = NpFftShiftOp()(Sample(input=x)) + restored = NpIfftShiftOp()(shifted) + assert np.array_equal(restored.input, x) + + def test_preserves_dtype_integer_and_complex(self) -> None: + assert NpFftShiftOp()(Sample(input=np.arange(5))).input.dtype == np.int64 + cx = np.array([1 + 1j, 2 - 2j, 3j], dtype=np.complex64) + assert NpFftShiftOp()(Sample(input=cx)).input.dtype == np.complex64 + + def test_axis_shifts_per_row(self) -> None: + x = np.arange(15).reshape(3, 5) + assert np.allclose(NpFftShiftOp(axis=-1)(Sample(input=x)).input, np.fft.fftshift(x, axes=-1)) + + def test_axis_none_shifts_all_axes(self) -> None: + x = np.arange(15).reshape(3, 5) + assert np.allclose(NpFftShiftOp(axis=None)(Sample(input=x)).input, np.fft.fftshift(x)) + + def test_preserves_target_and_metadata(self) -> None: + out = NpFftShiftOp()(Sample(input=np.arange(4), target=9, metadata={"k": "v"})) + assert out.target == 9 + assert out.meta == {"k": "v"} + + def test_raises_on_non_ndarray(self) -> None: + with pytest.raises(TypeError, match="FftShiftOp expects an np.ndarray"): + NpFftShiftOp()(Sample(input=torch.zeros(4))) + with pytest.raises(TypeError, match="IfftShiftOp expects an np.ndarray"): + NpIfftShiftOp()(Sample(input=torch.zeros(4))) + + def test_zero_arg_construction(self) -> None: + assert NpFftShiftOp().axis == -1 + assert NpIfftShiftOp().axis == -1 + + def test_produces_contract_conforms(self) -> None: + out = NpFftShiftOp()(Sample(input=np.arange(5))) + assert NpFftShiftOp.PRODUCES.accepts(infer_sample_type(out)) + + +class TestTorchShiftOps: + def test_fftshift_matches_torch(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0]) + assert torch.allclose(TorchFftShiftOp()(Sample(input=t)).input, torch.fft.fftshift(t)) + + def test_ifftshift_matches_torch(self) -> None: + t = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0]) + assert torch.allclose(TorchIfftShiftOp()(Sample(input=t)).input, torch.fft.ifftshift(t)) + + def test_ifftshift_inverts_fftshift_odd_length(self) -> None: + t = torch.arange(5) + restored = TorchIfftShiftOp()(TorchFftShiftOp()(Sample(input=t))) + assert torch.equal(restored.input, t) + + def test_preserves_dtype_half_and_integer(self) -> None: + # Pure rearrangement — no FFT — so half precision (which the FFT ops reject) passes through. + assert TorchFftShiftOp()(Sample(input=torch.arange(5))).input.dtype == torch.int64 + half = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float16) + assert TorchFftShiftOp()(Sample(input=half)).input.dtype == torch.float16 + + def test_dim_shifts_per_row(self) -> None: + t = torch.arange(15).reshape(3, 5) + assert torch.equal(TorchFftShiftOp(dim=-1)(Sample(input=t)).input, torch.fft.fftshift(t, dim=-1)) + + def test_dim_none_shifts_all_dims(self) -> None: + t = torch.arange(15).reshape(3, 5) + assert torch.equal(TorchFftShiftOp(dim=None)(Sample(input=t)).input, torch.fft.fftshift(t)) + + def test_raises_on_non_tensor(self) -> None: + with pytest.raises(TypeError, match="FftShiftOp expects a torch.Tensor"): + TorchFftShiftOp()(Sample(input=np.zeros(4))) + with pytest.raises(TypeError, match="IfftShiftOp expects a torch.Tensor"): + TorchIfftShiftOp()(Sample(input=np.zeros(4))) + + def test_zero_arg_construction(self) -> None: + assert TorchFftShiftOp().dim == -1 + assert TorchIfftShiftOp().dim == -1 + + def test_produces_contract_conforms(self) -> None: + out = TorchFftShiftOp()(Sample(input=torch.arange(5))) + assert TorchFftShiftOp.PRODUCES.accepts(infer_sample_type(out)) + + +# --------------------------------------------------------------------------- +# Full pipeline round-trips combining the ops +# --------------------------------------------------------------------------- + + +class TestRoundTrips: + def test_numpy_fft_shift_unshift_ifft_recovers(self) -> None: + # FourierOp -> FftShiftOp -> IfftShiftOp -> InverseFourierOp == identity (real signal). + x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) + s = NpFourierOp()(Sample(input=x)) + s = NpFftShiftOp()(s) + s = NpIfftShiftOp()(s) + s = NpInverseFourierOp()(s) + assert np.allclose(s.input.real, x, atol=1e-9) + + def test_torch_fft_shift_unshift_ifft_recovers(self) -> None: + t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) + s = TorchFourierOp()(Sample(input=t)) + s = TorchFftShiftOp()(s) + s = TorchIfftShiftOp()(s) + s = TorchInverseFourierOp()(s) + assert torch.allclose(s.input.real, t, atol=1e-9) + + +# --------------------------------------------------------------------------- +# Windowing + unit scaling (WindowOp / SpectrumScalingOp + FourierOp options) +# --------------------------------------------------------------------------- + + +def _np_tone(n: int = 1024, bin_index: int = 64, amp: float = 1.0) -> np.ndarray: + return np.asarray(amp * np.exp(2j * np.pi * bin_index * np.arange(n) / n), dtype=np.complex64) + + +class TestNumpyWindowAndScaling: + def test_default_fourierop_unchanged_and_metadata_byte_identical(self) -> None: + x = _np_tone() + s = Sample(input=x, target=None, metadata={"k": 1}) + out = NpFourierOp()(s) + assert np.allclose(out.input, np.fft.fft(x)) and out.input.dtype == np.complex64 + assert out.metadata == s.metadata # boxcar/none stamps nothing + + def test_window_option_stashes_correction(self) -> None: + out = NpFourierOp(window="hann")(Sample(input=_np_tone(), target=None, metadata={})) + assert out.meta["window"] == "hann" + assert out.meta["window_sum"] == pytest.approx(512.0) + assert out.meta["window_enbw_bins"] == pytest.approx(1.5) + + def test_windowop_preserves_complex64_dtype(self) -> None: + out = NpWindowOp(window="hann")(Sample(input=_np_tone(), target=None, metadata={})) + assert out.input.dtype == np.complex64 # not upcast to complex128 by the float window + assert out.meta["window_sum"] == pytest.approx(512.0) + + def test_amplitude_and_power_recover_tone(self) -> None: + s = Sample(input=_np_tone(amp=1.0), target=None, metadata={}) + amp = NpFourierOp(window="hann", scaling="amplitude")(s).input + pw = NpFourierOp(window="hann", scaling="power")(s).input + assert np.abs(amp).max() == pytest.approx(1.0, abs=1e-4) + assert pw.max() == pytest.approx(1.0, abs=1e-4) + assert np.iscomplexobj(amp) and not np.iscomplexobj(pw) + + def test_one_node_equals_explicit_chain(self) -> None: + s = Sample(input=_np_tone(), target=None, metadata={"samplerate": 1000.0}) + one = NpFourierOp(window="hann", scaling="density", sample_rate=1000.0)(s).input + chain = NpSpectrumScalingOp(scaling="density", sample_rate=1000.0)( + NpFourierOp()(NpWindowOp(window="hann")(s)) + ).input + assert np.allclose(one, chain) + + def test_spectrumscaling_rectangular_fallback_without_metadata(self) -> None: + s = Sample(input=_np_tone(), target=None, metadata={}) + raw = NpFourierOp()(s) # boxcar default → no window correction stashed + assert WINDOW_SUM_KEY not in raw.metadata + pw = NpSpectrumScalingOp(scaling="power")(raw).input + n = raw.input.shape[-1] + assert np.allclose(pw, np.abs(raw.input) ** 2 / n**2) # rectangular S1=S2=N fallback + + def test_scaling_requires_backward_norm(self) -> None: + s = Sample(input=_np_tone(), target=None, metadata={}) + with pytest.raises(ValueError, match="norm='backward'"): + NpFourierOp(scaling="power", norm="ortho")(s) + + def test_shift_and_one_sided_mutually_exclusive(self) -> None: + s = Sample(input=_np_tone(), target=None, metadata={}) + with pytest.raises(ValueError, match="mutually exclusive"): + NpFourierOp(scaling="power", shift=True, one_sided=True)(s) + + def test_one_sided_real_signal_amplitude(self) -> None: + n = 1024 + t = np.arange(n) + x = (2.0 * np.cos(2 * np.pi * 16 * t / n)).astype(np.float64) + amp = NpFourierOp(scaling="amplitude", one_sided=True)(Sample(input=x, target=None, metadata={})).input + assert amp.shape[0] == n // 2 + 1 + assert np.abs(amp).max() == pytest.approx(2.0, abs=1e-6) + + def test_produces_accepts_complex_and_real(self) -> None: + s = Sample(input=_np_tone(), target=None, metadata={}) + assert NpFourierOp.PRODUCES.accepts(infer_sample_type(NpFourierOp()(s))) # complex + assert NpFourierOp.PRODUCES.accepts(infer_sample_type(NpFourierOp(scaling="power")(s))) # real + scaled = NpSpectrumScalingOp(scaling="power")(NpFourierOp()(s)) + assert NpSpectrumScalingOp.PRODUCES.accepts(infer_sample_type(scaled)) + + def test_sample_rate_from_metadata(self) -> None: + s = Sample(input=_np_tone(), target=None, metadata={"samplerate": 500.0}) + from_meta = NpFourierOp(window="hann", scaling="density")(s).input + explicit = NpFourierOp(window="hann", scaling="density", sample_rate=500.0)(s).input + assert np.allclose(from_meta, explicit) + + def test_spectrumscaling_density_no_rate_runs(self) -> None: + # exercises the normalized-frequency fallback (Fs=1.0) + debug log branch + s = Sample(input=_np_tone(), target=None, metadata={}) + out = NpSpectrumScalingOp(scaling="density")(NpFourierOp()(s)).input + assert out.shape == (1024,) and not np.iscomplexobj(out) + + +class TestTorchWindowAndScaling: + def test_default_unchanged(self) -> None: + x = torch.view_as_complex(torch.randn(1024, 2)) + out = TorchFourierOp()(Sample(input=x, target=None, metadata={"k": 1})) + assert torch.allclose(out.input, torch.fft.fft(x)) + assert out.metadata == {"k": 1} # boxcar/none stamps nothing + + def test_amplitude_recovers_tone(self) -> None: + n = 1024 + x = torch.exp(2j * torch.pi * 64 * torch.arange(n) / n).to(torch.complex64) + amp = TorchFourierOp(window="hann", scaling="amplitude")(Sample(input=x, target=None, metadata={})).input + assert amp.abs().max().item() == pytest.approx(1.0, abs=1e-3) + + def test_window_stashes_and_preserves_dtype(self) -> None: + x = torch.exp(2j * torch.pi * 64 * torch.arange(1024) / 1024).to(torch.complex64) + out = TorchWindowOp(window="hann")(Sample(input=x, target=None, metadata={})) + assert out.input.dtype == torch.complex64 + assert out.meta["window_sum"] == pytest.approx(512.0) + + def test_power_is_real_and_scaling_requires_backward_norm(self) -> None: + x = torch.view_as_complex(torch.randn(64, 2)) + pw = TorchFourierOp(window="hann", scaling="power")(Sample(input=x, target=None, metadata={})).input + assert not pw.is_complex() + with pytest.raises(ValueError, match="norm='backward'"): + TorchFourierOp(scaling="power", norm="ortho")(Sample(input=x, target=None, metadata={})) + + def test_spectrumscaling_standalone_matches_one_node(self) -> None: + n = 256 + x = torch.exp(2j * torch.pi * 20 * torch.arange(n) / n).to(torch.complex64) + s = Sample(input=x, target=None, metadata={}) + one = TorchFourierOp(window="hamming", scaling="power")(s).input + chain = TorchSpectrumScalingOp(scaling="power")(TorchFourierOp()(TorchWindowOp(window="hamming")(s))).input + assert torch.allclose(one, chain, atol=1e-4) + + +def test_numpy_torch_parity_window_and_scaling() -> None: + n = 512 + base = np.exp(2j * np.pi * 40 * np.arange(n) / n).astype(np.complex64) + meta = {"samplerate": 2000.0} + for scaling in ("none", "amplitude", "power", "density"): + npo = NpFourierOp(window="blackmanharris", scaling=scaling, sample_rate=2000.0)( + Sample(input=base.copy(), target=None, metadata=dict(meta)) + ).input + to = TorchFourierOp(window="blackmanharris", scaling=scaling, sample_rate=2000.0)( + Sample(input=torch.from_numpy(base.copy()), target=None, metadata=dict(meta)) + ).input + assert np.allclose(npo, to.numpy(), rtol=1e-3, atol=1e-3), scaling diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py index ca46877..1703534 100644 --- a/tests/test_image_ops.py +++ b/tests/test_image_ops.py @@ -15,11 +15,17 @@ from dataflux.ops.image import ( COLORMAPS, + TEXT_POSITIONS, Colormap, ConvertToImageOp, NormalizeToUint8Op, + TextPosition, _apply_colormap, + array_histogram, + channel_count, + draw_text, sample_to_image, + select_channel, value_to_image, ) from dataflux.sample import Sample @@ -201,3 +207,200 @@ def test_every_colormap_in_the_literal_set_renders() -> None: for cmap in COLORMAPS: img = _apply_colormap(spec_u8, cmap) assert img.mode == "RGB" and img.size == (8, 8) + + +# --------------------------------------------------------------------------- +# select_channel — reduce an arbitrary array/tensor to a 2-D map for one channel +# --------------------------------------------------------------------------- + + +def test_select_channel_2d_passthrough() -> None: + arr = np.arange(12, dtype=np.float32).reshape(3, 4) + out = select_channel(arr, channel=-1) + assert out.shape == (3, 4) + np.testing.assert_array_equal(out, arr) + + +def test_select_channel_1d_becomes_strip() -> None: + out = select_channel(np.arange(5, dtype=np.float32)) + assert out.shape == (1, 5) + + +def test_select_channel_scalar_becomes_cell() -> None: + assert select_channel(np.float32(3.0)).shape == (1, 1) + + +def test_select_channel_chw_picks_plane() -> None: + # 3 channels first (smallest axis) → channel 1 is the middle plane. + arr = np.stack([np.full((4, 5), c, dtype=np.float32) for c in range(3)], axis=0) + out = select_channel(arr, channel=1) + assert out.shape == (4, 5) + assert float(out.mean()) == 1.0 + + +def test_select_channel_hwc_picks_plane() -> None: + arr = np.stack([np.full((4, 5), c, dtype=np.float32) for c in range(3)], axis=-1) + out = select_channel(arr, channel=2) + assert out.shape == (4, 5) + assert float(out.mean()) == 2.0 + + +def test_select_channel_all_is_mean_across_channels() -> None: + arr = np.stack([np.zeros((4, 5), dtype=np.float32), np.full((4, 5), 4.0, dtype=np.float32)], axis=0) + out = select_channel(arr, channel=-1) + assert out.shape == (4, 5) + assert float(out.mean()) == 2.0 # mean of {0, 4} + + +def test_select_channel_out_of_range_clamps() -> None: + # Spatial dims (5×4) larger than the channel count (3), so the smallest-axis heuristic + # unambiguously identifies axis 0 as channels (the channels-are-fewest assumption). + arr = np.stack([np.full((5, 4), c, dtype=np.float32) for c in range(3)], axis=0) + # channel 99 clamps to the last channel (index 2). + assert float(select_channel(arr, channel=99).mean()) == 2.0 + + +def test_select_channel_complex_uses_magnitude() -> None: + arr = np.array([[3 + 4j, 0]], dtype=np.complex64) # |3+4j| = 5 + out = select_channel(arr) + assert out.shape == (1, 2) + assert float(out[0, 0]) == 5.0 + + +def test_select_channel_torch_tensor() -> None: + out = select_channel(torch.arange(6, dtype=torch.float32).reshape(2, 3)) + assert isinstance(out, np.ndarray) and out.shape == (2, 3) + + +def test_select_channel_non_array_yields_unit_map() -> None: + assert select_channel("not an array").shape == (1, 1) + + +def test_channel_count() -> None: + assert channel_count(np.zeros((4, 5), dtype=np.float32)) == 1 # 2-D → 1 + assert channel_count(np.zeros((3, 4, 5), dtype=np.float32)) == 3 # CHW + assert channel_count(np.zeros((4, 5, 3), dtype=np.float32)) == 3 # HWC + assert channel_count("not an array") == 0 + + +# --------------------------------------------------------------------------- +# array_histogram — bin an array's values + summary statistics +# --------------------------------------------------------------------------- + + +def test_array_histogram_shape_and_stats() -> None: + arr = np.linspace(0.0, 1.0, 100, dtype=np.float32) + hist = array_histogram(arr, bins=10) + assert len(hist["counts"]) == 10 + assert len(hist["bin_edges"]) == 11 # bins + 1 + assert sum(hist["counts"]) == hist["count"] == 100 + assert hist["min"] == 0.0 and hist["max"] == 1.0 + assert hist["channels"] == 1 + assert abs(hist["mean"] - 0.5) < 1e-3 + + +def test_array_histogram_excludes_non_finite() -> None: + arr = np.array([0.0, 1.0, np.nan, np.inf, -np.inf, 2.0], dtype=np.float32) + hist = array_histogram(arr, bins=4) + # Only the 3 finite values (0, 1, 2) are counted; stats are finite. + assert hist["count"] == 3 + assert sum(hist["counts"]) == 3 + assert hist["min"] == 0.0 and hist["max"] == 2.0 + assert np.isfinite(hist["mean"]) and np.isfinite(hist["std"]) + + +def test_array_histogram_all_nan_is_empty_but_well_formed() -> None: + hist = array_histogram(np.full((4,), np.nan, dtype=np.float32), bins=8) + assert hist["count"] == 0 + assert hist["counts"] == [0] * 8 + assert len(hist["bin_edges"]) == 9 + assert hist["min"] is None and hist["max"] is None and hist["mean"] is None and hist["std"] is None + + +def test_array_histogram_flat_array_bins_into_first_bin() -> None: + hist = array_histogram(np.full((10,), 5.0, dtype=np.float32), bins=4) + assert hist["count"] == 10 + assert sum(hist["counts"]) == 10 + assert hist["min"] == 5.0 and hist["max"] == 5.0 + + +def test_array_histogram_single_channel_vs_all() -> None: + # channel 0 is all zeros, channel 1 is all ones. + arr = np.stack([np.zeros((4, 4), dtype=np.float32), np.ones((4, 4), dtype=np.float32)], axis=0) + only0 = array_histogram(arr, bins=4, channel=0) + assert only0["count"] == 16 and only0["min"] == 0.0 and only0["max"] == 0.0 + allc = array_histogram(arr, bins=4, channel=-1) + assert allc["count"] == 32 and allc["min"] == 0.0 and allc["max"] == 1.0 + + +def test_array_histogram_non_array_is_empty() -> None: + hist = array_histogram("text", bins=8) + assert hist["count"] == 0 and hist["channels"] == 0 + + +@pytest.mark.parametrize( + "arr", + [ + np.tile(np.arange(256, dtype=np.uint8), (256, 1)), # 256x256 uint8 gray gradient (65536 px) + np.linspace(-120.0, 0.0, 1024 * 512, dtype=np.float32), # large float32 dB spectrogram + (np.random.RandomState(0).rand(512, 512) * 255).astype(np.float32), # 262144 px random gray + ], +) +def test_array_histogram_large_array_does_not_raise(arr: np.ndarray) -> None: + # Regression: numpy 2.2.x's uniform-bins fast path (`bins=, range=(lo,hi)`) block-accumulates + # via np.bincount for arrays >65536 elements and miscomputes the bincount length on the workspace + # build — `n += bincount(...)` raised "operands could not be broadcast together with shapes + # (256,) (257,) (256,)" on any real image. array_histogram uses explicit linspace edges to avoid it. + finite = arr[np.isfinite(arr)] + hist = array_histogram(arr, bins=256) + assert len(hist["counts"]) == 256 and len(hist["bin_edges"]) == 257 + assert sum(hist["counts"]) == hist["count"] == finite.size # every value still counted + + +# --------------------------------------------------------------------------- +# draw_text — render text onto an image / a fresh canvas +# --------------------------------------------------------------------------- + + +def test_text_positions_is_the_literal_set() -> None: + assert TEXT_POSITIONS == get_args(TextPosition) + assert "center" in TEXT_POSITIONS and "top-left" in TEXT_POSITIONS and len(TEXT_POSITIONS) == 9 + + +def test_draw_text_blank_canvas_dims_and_dtype() -> None: + img = draw_text("Hi", None, width=200, height=80, background="black", color="white") + assert img.shape == (80, 200, 3) and img.dtype == np.uint8 + assert int((img > 0).sum()) > 0 # white text drawn on the black canvas + + +def test_draw_text_onto_existing_image_preserves_dims_and_copies() -> None: + base = np.zeros((64, 128, 3), dtype=np.uint8) + out = draw_text("label", base, color="red", position="center") + assert out.shape == (64, 128, 3) + assert (out[..., 0] > 0).any() # red text pixels present + assert int(base.sum()) == 0 # the input image is not mutated (drawn on a copy) + + +def test_draw_text_positions_place_block_differently() -> None: + tl = draw_text("X", None, width=120, height=120, position="top-left") + br = draw_text("X", None, width=120, height=120, position="bottom-right") + assert tl[:60].sum() > tl[60:].sum() # top-left lights the upper half + assert br[60:].sum() > br[:60].sum() # bottom-right lights the lower half + + +def test_draw_text_wrap_uses_more_vertical_lines() -> None: + long = "alpha bravo charlie delta echo foxtrot golf hotel india juliet" + + def text_row_span(im: np.ndarray) -> int: + rows = np.where((im > 0).any(axis=(1, 2)))[0] + return int(rows.max() - rows.min() + 1) if rows.size else 0 + + wrapped = draw_text(long, None, width=120, height=240, wrap=True, position="top-left") + nowrap = draw_text(long, None, width=120, height=240, wrap=False, position="top-left") + # Wrapping spreads the text over more rows than a single (unwrapped) line. + assert text_row_span(wrapped) > text_row_span(nowrap) + + +def test_draw_text_accepts_torch_tensor_image() -> None: + out = draw_text("t", torch.zeros(3, 32, 48)) # CHW float tensor + assert isinstance(out, np.ndarray) and out.ndim == 3 and out.shape[2] == 3 diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index 2cd4456..b4c6cb5 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -13,11 +13,27 @@ from confluid import parse_param_docs # type: ignore[import-not-found] from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp -from dataflux.ops.numpy import ConnectedComponentsOp, StandardizeOp, ThresholdOp +from dataflux.ops.numpy import ( + ConnectedComponentsOp, + FftShiftOp, + FourierOp, + IfftShiftOp, + InverseFourierOp, + SpectrumScalingOp, + StandardizeOp, + ThresholdOp, + WindowOp, +) from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp from dataflux.ops.tee import Tee +from dataflux.ops.torch import FftShiftOp as TorchFftShiftOp +from dataflux.ops.torch import FourierOp as TorchFourierOp +from dataflux.ops.torch import IfftShiftOp as TorchIfftShiftOp +from dataflux.ops.torch import InverseFourierOp as TorchInverseFourierOp +from dataflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp from dataflux.ops.torch import StandardizeOp as TorchStandardizeOp from dataflux.ops.torch import ToTensorOp +from dataflux.ops.torch import WindowOp as TorchWindowOp from dataflux.ops.transform_chain import TransformChain from dataflux.sources import HuggingFaceSource @@ -31,8 +47,20 @@ StandardizeOp, ThresholdOp, ConnectedComponentsOp, + FourierOp, + InverseFourierOp, + FftShiftOp, + IfftShiftOp, ToTensorOp, TorchStandardizeOp, + TorchFourierOp, + TorchInverseFourierOp, + TorchFftShiftOp, + TorchIfftShiftOp, + WindowOp, + SpectrumScalingOp, + TorchWindowOp, + TorchSpectrumScalingOp, MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp, diff --git a/tests/test_ops.py b/tests/test_ops.py index 03a0ae6..a4090fd 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -8,6 +8,7 @@ from PIL import Image from dataflux.ops import ( + CaptureOutputOp, ConfigureOp, CopyInputOp, CopyMetadataOp, @@ -761,6 +762,122 @@ def test_fluid_markers_flow_lazily(self) -> None: np.testing.assert_array_equal(out.input, [False, True]) +class TestCaptureOutputOp: + class _DrawOp: + """Stub op: transforms the sample (+1) and exposes its drawn value as an @output-like property.""" + + def __init__(self, value: float = 0.0) -> None: + self._value = value + self._last: object = None + self.calls = 0 + + def __call__(self, sample: Sample) -> Sample: + self.calls += 1 + self._last = self._value + return sample._replace(input=sample.input + 1) + + @property + def drawn(self) -> object: + return self._last + + def test_records_output_into_metadata_and_keeps_transform(self) -> None: + op = self._DrawOp(value=42.0) + sample = Sample(input=np.array([1.0]), target=None, metadata={}) + out = CaptureOutputOp(op=op, output="drawn", key="captured")(sample) + assert out is not None + assert out.meta["captured"] == 42.0 # the @output value rides metadata + np.testing.assert_array_equal(out.input, [2.0]) # the wrapped op's transform is kept + assert op.calls == 1 + + def test_default_key_is_output_name(self) -> None: + out = CaptureOutputOp(op=self._DrawOp(value=7.0), output="drawn")(Sample(input=0, target=None, metadata={})) + assert out is not None and out.meta["drawn"] == 7.0 + + def test_multi_capture_applies_op_once(self) -> None: + class _Multi: + def __init__(self) -> None: + self.calls = 0 + + def __call__(self, s: Sample) -> Sample: + self.calls += 1 + return s + + @property + def a(self) -> int: + return 1 + + @property + def b(self) -> int: + return 2 + + op = _Multi() + out = CaptureOutputOp(op=op, captures={"a": "ka", "b": "kb"})(Sample(input=0, target=None, metadata={})) + assert out is not None and out.meta["ka"] == 1 and out.meta["kb"] == 2 + assert op.calls == 1 # one application, several captures + + def test_captures_actual_drawn_value_not_recompute(self) -> None: + """A stochastic @output must be captured from the SAME application — never a fresh re-draw.""" + seq = iter([11.0, 22.0, 33.0]) + + class _Stochastic: + def __init__(self) -> None: + self._last: object = None + + def __call__(self, s: Sample) -> Sample: + self._last = next(seq) + return s + + @property + def drawn(self) -> object: + return self._last + + out = CaptureOutputOp(op=_Stochastic(), output="drawn", key="v")(Sample(input=0, target=None, metadata={})) + assert out is not None and out.meta["v"] == 11.0 # the first (and only) draw + + def test_requires_op(self) -> None: + with pytest.raises(ValueError, match="'op'"): + CaptureOutputOp(output="x")(Sample(input=0, target=None, metadata={})) + + def test_requires_something_to_capture(self) -> None: + with pytest.raises(ValueError, match="nothing to capture"): + CaptureOutputOp(op=self._DrawOp())(Sample(input=0, target=None, metadata={})) + + def test_missing_attribute_raises(self) -> None: + with pytest.raises(AttributeError, match="no @output attribute"): + CaptureOutputOp(op=self._DrawOp(), output="nope")(Sample(input=0, target=None, metadata={})) + + def test_filtering_op_propagates_none(self) -> None: + assert ( + CaptureOutputOp(op=lambda s: None, output="x", key="k")(Sample(input=0, target=None, metadata={})) is None + ) + + def test_reads_output_through_target_wrapper(self) -> None: + """The @output is read THROUGH a ``.target`` wrapper (e.g. a ConfigureOp), so a node that is + both a capture-consumer (its param configured) AND a capture-producer composes.""" + from typing import Callable + + class _Wrapper: # mimics ConfigureOp: applies .target and exposes it as .target + def __init__(self, target: Callable[[Sample], Sample]) -> None: + self.target = target + + def __call__(self, s: Sample) -> Sample: + return self.target(s) + + out = CaptureOutputOp(op=_Wrapper(self._DrawOp(value=99.0)), output="drawn", key="v")( + Sample(input=np.array([1.0]), target=None, metadata={}) + ) + assert out is not None and out.meta["v"] == 99.0 + np.testing.assert_array_equal(out.input, [2.0]) + + def test_fluid_marker_op_flows_lazily(self) -> None: + from confluid.fluid import Class + + op = CaptureOutputOp(op=Class(self._DrawOp, value=5.0), output="drawn", key="v") + out = op(Sample(input=np.array([1.0]), target=None, metadata={})) + assert out is not None and out.meta["v"] == 5.0 + np.testing.assert_array_equal(out.input, [2.0]) + + # --------------------------------------------------------------------------- # numpy.resolve_expression # --------------------------------------------------------------------------- diff --git a/tests/test_windows.py b/tests/test_windows.py new file mode 100644 index 0000000..cd90efe --- /dev/null +++ b/tests/test_windows.py @@ -0,0 +1,157 @@ +"""Tests for :mod:`dataflux.windows` — the window functions + spectral unit-scaling math. + +Pins (1) the closed Literals match their runtime tuples, (2) the pure-numpy windows match +scipy (when available) and have the right correction constants (Hann coherent gain 0.5 / ENBW +1.5 bins, flat-top CG 0.2156), and (3) ``scale_spectrum`` returns calibrated units — a +unit-amplitude tone reads amplitude 1.0 / power 1.0, and the power-to-density ratio is the +window's equivalent noise bandwidth in Hz. +""" + +from typing import cast, get_args + +import numpy as np +import pytest + +from dataflux import windows as W +from dataflux.windows import WindowName + + +def test_literals_match_runtime_tuples() -> None: + assert W.WINDOW_NAMES == get_args(W.WindowName) + assert W.SPECTRUM_SCALINGS == get_args(W.SpectrumScaling) + assert "boxcar" in W.WINDOW_NAMES and "hann" in W.WINDOW_NAMES + assert W.SPECTRUM_SCALINGS == ("none", "amplitude", "power", "density") + + +@pytest.mark.parametrize("name", W.WINDOW_NAMES) +def test_get_window_builds_each(name: WindowName) -> None: + wp = {"kaiser": 8.6, "tukey": 0.5, "gaussian": 7.0}.get(name) + w = W.get_window(name, 64, window_param=wp) + assert w.shape == (64,) and w.dtype == np.float64 + assert np.all(np.isfinite(w)) + # boxcar is the rectangular identity; every other taper has a sub-unity mean (it attenuates) + if name == "boxcar": + assert np.allclose(w, 1.0) + else: + assert w.mean() < 1.0 + + +def test_get_window_periodic_differs_from_symmetric() -> None: + p = W.get_window("hann", 64, periodic=True) + s = W.get_window("hann", 64, periodic=False) + assert not np.allclose(p, s) + # symmetric Hann is zero at both endpoints; periodic is zero only at index 0 + assert s[0] == pytest.approx(0.0) and s[-1] == pytest.approx(0.0) + assert p[0] == pytest.approx(0.0) + + +def test_get_window_matches_scipy() -> None: + sw = pytest.importorskip("scipy.signal") + specs = { + "boxcar": "boxcar", + "bartlett": "bartlett", + "hann": "hann", + "hamming": "hamming", + "blackman": "blackman", + "blackmanharris": "blackmanharris", + "nuttall": "nuttall", + "flattop": "flattop", + "kaiser": ("kaiser", 8.6), + "tukey": ("tukey", 0.5), + "gaussian": ("gaussian", 7.0), + } + for name, spec in specs.items(): + wp = {"kaiser": 8.6, "tukey": 0.5, "gaussian": 7.0}.get(name) + for periodic in (True, False): + mine = W.get_window(cast(WindowName, name), 128, window_param=wp, periodic=periodic) + ref = sw.get_window(spec, 128, fftbins=periodic) + assert np.allclose(mine, ref, atol=1e-12), f"{name} periodic={periodic}" + + +def test_get_window_errors() -> None: + with pytest.raises(ValueError, match="unknown window"): + W.get_window("nope", 16) # type: ignore[arg-type] + with pytest.raises(ValueError, match="must be positive"): + W.get_window("hann", 0) + with pytest.raises(ValueError, match="gaussian window requires"): + W.get_window("gaussian", 16) # no window_param + + +def test_correction_constants() -> None: + # asymptotic (large-N) coherent gain + ENBW for the textbook windows + cases = {"boxcar": (1.0, 1.0), "hann": (0.5, 1.5), "hamming": (0.54, 1.363), "flattop": (0.2156, 3.77)} + for name, (cg, enbw) in cases.items(): + w = W.get_window(cast(WindowName, name), 8192) + assert W.coherent_gain(w) == pytest.approx(cg, abs=2e-3) + assert W.enbw_bins(w) == pytest.approx(enbw, abs=2e-2) + + +def test_window_sums_and_metadata() -> None: + w = W.get_window("hann", 1024) + s1, s2 = W.window_sums(w) + assert s1 == pytest.approx(512.0) and s2 == pytest.approx(384.0) + meta = W.window_metadata("hann", w) + assert meta[W.WINDOW_NAME_KEY] == "hann" + assert meta[W.WINDOW_SIZE_KEY] == 1024 + assert meta[W.WINDOW_SUM_KEY] == pytest.approx(512.0) + assert meta[W.WINDOW_SUMSQ_KEY] == pytest.approx(384.0) + assert meta[W.WINDOW_ENBW_KEY] == pytest.approx(1.5) + assert meta[W.WINDOW_CG_KEY] == pytest.approx(0.5) + + +def _tone(n: int, bin_index: int, amp: float = 1.0) -> np.ndarray: + return np.asarray(amp * np.exp(2j * np.pi * bin_index * np.arange(n) / n), dtype=np.complex64) + + +def test_scale_spectrum_none_is_passthrough() -> None: + x = _tone(256, 10) + X = np.fft.fft(x) + out = W.scale_spectrum(X, "none", s1=256.0, s2=256.0) + assert np.array_equal(out, X) + + +def test_scale_spectrum_amplitude_and_power_recover_tone() -> None: + n = 1024 + X = np.fft.fft(_tone(n, 64, amp=1.0)) + amp = W.scale_spectrum(X, "amplitude", s1=float(n), s2=float(n)) + pw = W.scale_spectrum(X, "power", s1=float(n), s2=float(n)) + assert np.abs(amp).max() == pytest.approx(1.0, abs=1e-4) # amplitude V + assert pw.max() == pytest.approx(1.0, abs=1e-4) # power V² = amplitude² + assert np.iscomplexobj(amp) and not np.iscomplexobj(pw) # power is real + + +def test_scale_spectrum_density_is_power_over_enbw_hz() -> None: + n, fs = 1024, 1000.0 + rng = np.random.default_rng(0) + x = (rng.standard_normal(n) + 1j * rng.standard_normal(n)).astype(np.complex64) + w = W.get_window("hann", n) + Xw = np.fft.fft(x * w) + s1, s2 = W.window_sums(w) + pw = W.scale_spectrum(Xw, "power", s1=s1, s2=s2) + den = W.scale_spectrum(Xw, "density", s1=s1, s2=s2, sample_rate=fs) + enbw_hz = fs * s2 / (s1 * s1) # = Fs · ENBW_bins / N + assert np.allclose(pw, den * enbw_hz) # power = density × ENBW_Hz, bin-for-bin + + +def test_scale_spectrum_density_normalized_without_rate() -> None: + X = np.fft.fft(_tone(256, 8)) + den = W.scale_spectrum(X, "density", s1=256.0, s2=256.0) # Fs defaults to 1.0 + den_fs1 = W.scale_spectrum(X, "density", s1=256.0, s2=256.0, sample_rate=1.0) + assert np.allclose(den, den_fs1) + + +def test_fold_one_sided_even_and_odd() -> None: + # real cosine, amplitude 2 at bin 4 → one-sided amplitude reads 2 at that bin + for n in (64, 65): + t = np.arange(n) + x = 2.0 * np.cos(2 * np.pi * 4 * t / n) + amp = W.scale_spectrum(np.fft.fft(x), "amplitude", s1=float(n), s2=float(n), one_sided=True) + assert amp.shape[0] == n // 2 + 1 + assert np.abs(amp).max() == pytest.approx(2.0, abs=1e-6) + + +def test_fold_one_sided_preserves_dc_and_nyquist() -> None: + n = 64 + x = np.ones(n) * 3.0 # pure DC + one = W.fold_one_sided(np.fft.fft(x), axis=-1) + assert one[0] == pytest.approx(3.0 * n) # DC not doubled From 43f703db9db1587a399bc9ddc77ab06749da0417 Mon Sep 17 00:00:00 2001 From: gertbehi Date: Thu, 18 Jun 2026 14:06:39 +0200 Subject: [PATCH 016/102] feat: add DropMetadataOp + PrintSampleOp; UnstashOps gain remove param; fix ThresholdOp numeric bound handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New `dataflux.ops.metadata.DropMetadataOp`: strips metadata keys matching fnmatch globs (include-wins model like rsync); entry-pointed as `dataflux-ops-metadata` - New `dataflux.ops.debug.PrintSampleOp`: pass-through probe logging/printing per-sample summary (shape+dtype+metadata); level restricted to trace/debug; entry-pointed as `dataflux-ops-debug` - `UnstashInputOp`/`UnstashTargetOp` gain `remove: bool = True` — deletes the stash key after restoring so snapshots never leak into downstream sinks; only the final unstash of a fan-out key removes it - `ThresholdOp._resolve` now accepts any float()-able value (NumPy scalars, 0-d arrays) not just Python int/float, enabling the ConfigureOp per-sample value-chain path - AGENTS.md and pyproject.toml updated to document new ops and entry points --- AGENTS.md | 2 +- dataflux/ops/debug.py | 112 ++++++++++++++++++++++ dataflux/ops/metadata.py | 51 ++++++++++ dataflux/ops/numpy.py | 24 +++-- dataflux/ops/stash.py | 22 ++++- pyproject.toml | 5 + tests/test_categories.py | 4 + tests/test_ops.py | 194 ++++++++++++++++++++++++++++++++++++++- 8 files changed, 399 insertions(+), 15 deletions(-) create mode 100644 dataflux/ops/debug.py create mode 100644 dataflux/ops/metadata.py diff --git a/AGENTS.md b/AGENTS.md index 5d2c426..50e2b3a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `dataflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `dataflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the LogFlow logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's *Array / Tensor Histogram* viewer (`fluxstudio.nodes.ArrayHistogramViewerNode`): `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/dataflux/ops/debug.py b/dataflux/ops/debug.py new file mode 100644 index 0000000..d611581 --- /dev/null +++ b/dataflux/ops/debug.py @@ -0,0 +1,112 @@ +"""Sample inspection / debug ops.""" + +from typing import Any, Literal, Optional + +from confluid import configurable +from logflow import get_logger + +from dataflux.sample import Sample + +logger = get_logger(__name__) + +# Per-sample output is DIAGNOSTIC, so the logger level is restricted to trace/debug (the workspace +# "Diagnostic Log Levels" mandate — never info/warning for per-iteration events). Console visibility +# comes from ``to_console`` (a plain ``print``), independent of the log level. +LogLevel = Literal["trace", "debug"] + +_MAX_VALUE_REPR = 200 + + +def _summarize(value: Any) -> str: + """A compact one-line description: array shape/dtype + a length-capped value preview (large + arrays elided by numpy), else a length-capped ``repr``.""" + if value is None: + return "None" + shape = getattr(value, "shape", None) + dtype = getattr(value, "dtype", None) + if shape is not None and dtype is not None: + values = _cap(_array_values_repr(value)) + return f"{type(value).__name__}(shape={tuple(shape)}, dtype={dtype}, values={values})" + return _cap(repr(value)) + + +def _cap(text: str) -> str: + """Truncate a repr to ``_MAX_VALUE_REPR`` chars with an ellipsis marker.""" + return text if len(text) <= _MAX_VALUE_REPR else text[:_MAX_VALUE_REPR] + "…" + + +def _array_values_repr(value: Any) -> str: + """Compact one-line repr of an array's values (numpy / torch / anything array-like).""" + try: + import numpy as np + + return np.array2string(np.asarray(value), threshold=20, separator=", ").replace("\n", " ") + except Exception: # noqa: BLE001 - best-effort preview; fall back to plain repr + return repr(value) + + +def _summarize_metadata(metadata: Any) -> str: + """Summarise a metadata payload — each VALUE compacted so a large array shows shape/dtype.""" + if isinstance(metadata, dict): + return "{" + ", ".join(f"{key!r}: {_summarize(value)}" for key, value in metadata.items()) + "}" + if isinstance(metadata, list): + return f"[batch of {len(metadata)} metadata dicts]" + return _summarize(metadata) + + +@configurable(category="op", group="debug") +class PrintSampleOp: + """Log / print a summary of each sample passing through (a pass-through ``Sample -> Sample`` op). + + A pipeline probe: emits a compact description of the sample — ``input`` / ``target`` shape+dtype + plus a length-capped value preview (large arrays elided), and the ``metadata`` (values + summarised the same way) — to the LogFlow logger (the LOG file + console) and, by default, to stdout via + ``print`` (so it shows in a terminal / the FluxStudio node output panel regardless of log + level). The sample is returned UNCHANGED. + + Args: + label: A prefix identifying this probe in the output (e.g. "after-impairments"). + level: LogFlow level for the logged line — "trace" or "debug" (per-sample output is + diagnostic, so info/warning are deliberately not offered; use ``to_console`` to see it). + include_data: Include an ``input`` / ``target`` shape+dtype + value preview. + include_metadata: Include the sample's metadata (values summarised). + to_console: Also ``print`` the line to stdout — guaranteed console / node-panel visibility, + independent of the log level. Set False to log only. + limit: Stop emitting after this many samples (None = every sample) — avoids flooding on a + large dataset; the op still passes EVERY sample through unchanged. + """ + + def __init__( + self, + label: str = "sample", + level: LogLevel = "debug", + include_data: bool = True, + include_metadata: bool = True, + to_console: bool = True, + limit: Optional[int] = None, + ) -> None: + self.label = label + self.level = level + self.include_data = include_data + self.include_metadata = include_metadata + self.to_console = to_console + self.limit = limit + self._count = 0 # runtime probe counter — NOT config (per-instance, per-process) + + def __call__(self, sample: Sample) -> Sample: + if self.limit is None or self._count < self.limit: + message = self._format(sample) + getattr(logger, self.level)(message) # level is a closed Literal, so this method exists + if self.to_console: + print(message) + self._count += 1 + return sample + + def _format(self, sample: Sample) -> str: + parts = [f"[{self.label} #{self._count}]"] + if self.include_data: + parts.append(f"input={_summarize(sample.input)}") + parts.append(f"target={_summarize(sample.target)}") + if self.include_metadata: + parts.append(f"metadata={_summarize_metadata(sample.metadata)}") + return " ".join(parts) diff --git a/dataflux/ops/metadata.py b/dataflux/ops/metadata.py new file mode 100644 index 0000000..db96916 --- /dev/null +++ b/dataflux/ops/metadata.py @@ -0,0 +1,51 @@ +"""Metadata-manipulation ops.""" + +import fnmatch +from typing import List, Optional, Tuple + +from confluid import configurable + +from dataflux.sample import Sample + + +def _matches_any(key: str, patterns: Tuple[str, ...]) -> bool: + """True if ``key`` matches ANY ``fnmatch`` glob in ``patterns`` (case-sensitive).""" + return any(fnmatch.fnmatchcase(key, pattern) for pattern in patterns) + + +@configurable(category="op", group="structure") +class DropMetadataOp: + """Remove metadata keys matching glob patterns (a pass-through ``Sample -> Sample`` op). + + A key is DROPPED when it matches an ``exclude`` pattern AND does NOT match any ``include`` + pattern — so ``include`` PROTECTS keys and takes priority over ``exclude`` (the rsync / + gitignore include-wins model). Strips bookkeeping you don't want a downstream sink to + serialise — e.g. the internal ``__taidal_stash_*`` snapshots FluxStudio's DAG -> sequential + export leaves on the metadata bus (a stashed complex signal). The replacement metadata is a + fresh dict (copy-on-write); ``input`` / ``target`` are untouched. Single-sample only (reads + ``sample.meta``), like ``CopyMetadataOp`` — drop keys before collation. + + Args: + exclude: Glob patterns (``fnmatch``: ``*`` = any run, ``?`` = one char, ``[seq]`` = a set) + for keys to REMOVE. A pattern with NO wildcards matches that key exactly. Case-sensitive. + E.g. ``__taidal_stash*`` removes every auto-stash snapshot; with no ``exclude`` nothing + is dropped. + include: Glob patterns for keys to KEEP even when they match ``exclude`` — higher priority, + so it carves exceptions out of ``exclude``. E.g. ``exclude=["__taidal_stash*"]`` + + ``include=["__taidal_stash_456:*"]`` drops every stash key EXCEPT node 456's. ``include`` + only ever protects against ``exclude`` (with no ``exclude`` it has no effect). + """ + + def __init__(self, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> None: + self.exclude = exclude + self.include = include + + def __call__(self, sample: Sample) -> Sample: + exclude = tuple(self.exclude or ()) + include = tuple(self.include or ()) + kept = { + key: value + for key, value in sample.meta.items() + if not (_matches_any(key, exclude) and not _matches_any(key, include)) + } + return sample._replace(metadata=kept) diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index 4901af7..5efccc9 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -386,16 +386,22 @@ def __init__( def _resolve(self, bound: Optional[Union[float, int, str]], sample: Sample) -> float: if bound is None: raise ValueError("ThresholdOp._resolve called with None — bound was not filtered by __call__") - if isinstance(bound, (int, float)): - return float(bound) - if not isinstance(bound, str): - raise TypeError(f"ThresholdOp bounds must be a number or expression string; got {type(bound).__name__}") - resolved = resolve_expression(bound, sample) + if isinstance(bound, str): + resolved = resolve_expression(bound, sample) + try: + return float(resolved) + except (TypeError, ValueError) as exc: + raise ValueError( + f"ThresholdOp: expression {bound!r} resolved to {resolved!r}, which is not a number" + ) from exc + # Any non-string numeric: a Python int/float, a NumPy scalar (e.g. the float32 a value-chain + # MaxOp → FormulaOp → ConfigureOp injects into low_level per sample), or a 0-d array — anything + # float() accepts. A list / multi-D array / complex value fails float() and raises the TypeError. try: - return float(resolved) - except ValueError as exc: - raise ValueError( - f"ThresholdOp: expression {bound!r} resolved to {resolved!r}, " f"which is not a number" + return float(bound) + except (TypeError, ValueError) as exc: + raise TypeError( + f"ThresholdOp bounds must be a number or expression string; got {type(bound).__name__}" ) from exc def __call__(self, sample: Sample) -> Sample: diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py index bbd032f..a38f74a 100644 --- a/dataflux/ops/stash.py +++ b/dataflux/ops/stash.py @@ -56,17 +56,26 @@ class UnstashInputOp: key from corrupting each other through downstream in-place mutations. Set ``False`` only when the caller has audited that no downstream op mutates the array in place. + remove: When ``True`` (default), DELETE the key from metadata after + restoring it — so the snapshot doesn't linger on the bus and + leak into a downstream sink. Set ``False`` to keep it (required + when the SAME key is unstashed again later, e.g. a fan-out that + restores the fork before several branches — only the LAST + unstash of a key may remove it). """ - def __init__(self, key: str = "", copy: bool = True) -> None: + def __init__(self, key: str = "", copy: bool = True, remove: bool = True) -> None: # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. self.key = key self.copy = copy + self.remove = remove def __call__(self, sample: Sample) -> Sample: value = sample.meta[self.key] if self.copy: value = _copy.deepcopy(value) + if self.remove: + del sample.meta[self.key] # key exists (just read above) return sample._replace(input=value) @@ -103,15 +112,24 @@ class UnstashTargetOp: key from corrupting each other through downstream in-place mutations. Set ``False`` only when the caller has audited that no downstream op mutates the value in place. + remove: When ``True`` (default), DELETE the key from metadata after + restoring it — so the snapshot doesn't linger on the bus and + leak into a downstream sink. Set ``False`` to keep it (required + when the SAME key is unstashed again later, e.g. a fan-out that + restores the fork before several branches — only the LAST + unstash of a key may remove it). """ - def __init__(self, key: str = "", copy: bool = True) -> None: + def __init__(self, key: str = "", copy: bool = True, remove: bool = True) -> None: # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. self.key = key self.copy = copy + self.remove = remove def __call__(self, sample: Sample) -> Sample: value = sample.meta[self.key] if self.copy: value = _copy.deepcopy(value) + if self.remove: + del sample.meta[self.key] # key exists (just read above) return sample._replace(target=value) diff --git a/pyproject.toml b/pyproject.toml index fbdede5..962f473 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,11 @@ dataflux-ops-copy = "dataflux.ops.copy" dataflux-ops-swap = "dataflux.ops.swap" dataflux-ops-target = "dataflux.ops.target" dataflux-ops-image = "dataflux.ops.image" +# DropMetadataOp (strip metadata keys — e.g. the __taidal_stash_* snapshots) + PrintSampleOp +# (log/print a per-sample summary). Entry-point changes need an editable reinstall before +# FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). +dataflux-ops-metadata = "dataflux.ops.metadata" +dataflux-ops-debug = "dataflux.ops.debug" # Storage SINKS (HDF5Sink / ZarrGroupSink / ZarrBatchSink / DirectorySink) carry # category="sink" so FluxStudio surfaces them as DatasetProcessor sink nodes. They live under # dataflux.storage.* (NOT re-exported from the package root), and scan_module does not recurse diff --git a/tests/test_categories.py b/tests/test_categories.py index 0085f50..971c7f6 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -13,9 +13,11 @@ from dataflux.ops.capture import CaptureOutputOp from dataflux.ops.configure import ConfigureOp from dataflux.ops.copy import CopyInputOp +from dataflux.ops.debug import PrintSampleOp from dataflux.ops.enable import Enable from dataflux.ops.formula import FormulaOp from dataflux.ops.image import ConvertToImageOp, NormalizeToUint8Op +from dataflux.ops.metadata import DropMetadataOp from dataflux.ops.numpy import ( FftShiftOp, FourierOp, @@ -153,6 +155,8 @@ def test_op_group_tags() -> None: assert TorchWindowOp.__confluid_group__ == "torch" assert TorchSpectrumScalingOp.__confluid_group__ == "torch" assert CopyInputOp.__confluid_group__ == "structure" + assert DropMetadataOp.__confluid_group__ == "structure" + assert PrintSampleOp.__confluid_group__ == "debug" assert StashTargetOp.__confluid_group__ == "structure" assert UnstashTargetOp.__confluid_group__ == "structure" assert MetadataToTargetOp.__confluid_group__ == "structure" diff --git a/tests/test_ops.py b/tests/test_ops.py index a4090fd..85b6689 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -593,13 +593,31 @@ def test_unstash_no_copy_aliases(self) -> None: assert out.input is arr def test_two_unstashes_with_in_place_mutation_dont_corrupt(self) -> None: - """Default copy=True prevents branch-A's in-place write from leaking into branch-B.""" + """Default copy=True prevents branch-A's in-place write from leaking into branch-B. + + A multi-unstash of the SAME key needs remove=False on the NON-final unstashes so the + snapshot survives (the compiler emits exactly this for a fan-out); the LAST unstash + cleans it up. + """ arr = np.array([1.0, 2.0, 3.0]) sample = Sample(input=None, target=None, metadata={"snap": arr}) - a = UnstashInputOp(key="snap")(sample) + a = UnstashInputOp(key="snap", remove=False)(sample) # keep the key for branch B a.input.fill(99.0) # in-place mutation on branch A's restored array - b = UnstashInputOp(key="snap")(sample) + b = UnstashInputOp(key="snap")(sample) # final unstash → removes the key np.testing.assert_array_equal(b.input, [1.0, 2.0, 3.0]) + assert "snap" not in sample.meta # cleaned up by the final unstash + + def test_unstash_removes_key_by_default(self) -> None: + sample = Sample(input=None, target=None, metadata={"snap": np.array([1.0, 2.0]), "keep": 1}) + out = UnstashInputOp(key="snap")(sample) + np.testing.assert_array_equal(out.input, [1.0, 2.0]) + assert "snap" not in out.meta # removed by default + assert out.meta["keep"] == 1 # other keys untouched + + def test_unstash_keeps_key_when_remove_false(self) -> None: + sample = Sample(input=None, target=None, metadata={"snap": np.array([1.0])}) + out = UnstashInputOp(key="snap", remove=False)(sample) + assert "snap" in out.meta # --------------------------------------------------------------------------- @@ -647,6 +665,12 @@ def test_stash_restore_round_trip_preserves_fork_target(self) -> None: branched = stashed._replace(target="branch-target") restored = UnstashTargetOp(key="fork")(branched) assert restored.target == "fork-target" + assert "fork" not in restored.meta # removed by default after restore + + def test_unstash_target_keeps_key_when_remove_false(self) -> None: + sample = Sample(input=None, target=None, metadata={"snap": np.array([1.0])}) + out = UnstashTargetOp(key="snap", remove=False)(sample) + assert "snap" in out.meta # --------------------------------------------------------------------------- @@ -1030,6 +1054,20 @@ def test_raises_on_bad_value_type(self) -> None: with pytest.raises((TypeError, ValidationError)): np_ops.ThresholdOp(low_level=[1, 2])(Sample(input=np.array([0.0]))) # type: ignore[arg-type] + def test_numpy_scalar_and_zero_d_array_bounds_accepted(self) -> None: + # A value chain (MaxOp → FormulaOp → ConfigureOp) injects a NumPy scalar / 0-d array into a + # bound via setattr, bypassing the pydantic ctor. np.float64 SUBCLASSES Python float (so it + # slipped through the old `isinstance(bound, (int, float))`), but np.float32 does NOT — + # _resolve must accept anything float() accepts. Live regression for a float32 spectrogram: + # "ThresholdOp bounds must be a number or expression string; got float32". + arr = np.array([0.0, 1.0, 2.0], dtype=np.float32) + for bound in (np.float32(2.0), np.array(2.0)): + op = np_ops.ThresholdOp(low_op=">=") + op.low_level = bound # type: ignore[assignment] # post-construction injection (ConfigureOp does this) + out = op(Sample(input=arr)) + np.testing.assert_array_equal(out.input, [False, False, True]) + assert out.meta["threshold_low"] == 2.0 + def test_threshold_comparison_maps_match_literals() -> None: # The operator-dispatch dicts must stay in lockstep with their closed @@ -1277,3 +1315,153 @@ def test_roundtrip_squeeze_unsqueeze(self) -> None: unsqueezed = np_ops.UnsqueezeOp(axis=0)(Sample(input=arr)) restored = np_ops.SqueezeOp(axis=0)(unsqueezed) assert restored.input.shape == arr.shape + + +# --------------------------------------------------------------------------- +# DropMetadataOp +# --------------------------------------------------------------------------- +class TestDropMetadataOp: + def test_literal_exclude_drops_exact_keys(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + # A pattern with no wildcards is an EXACT key match; a missing key is ignored. + sample = Sample(input=np.zeros(2), target=None, metadata={"keep": 1, "drop_me": 2, "also": 3}) + out = DropMetadataOp(exclude=["drop_me", "also", "nope"])(sample) + assert out.meta == {"keep": 1} + + def test_glob_star_drops_all_matching(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + meta = {"real": 1, "__taidal_stash_456:input": [1j], "__taidal_stash_456:target": [2j]} + out = DropMetadataOp(exclude=["__taidal_stash*"])(Sample(input=np.zeros(2), metadata=meta)) + assert out.meta == {"real": 1} + + def test_glob_mid_wildcard_is_specific(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + # `__taidal_stash_456:*input` drops ONLY node 456's input stash — keeps its target and + # other nodes' inputs. + meta = { + "__taidal_stash_456:input": 1, + "__taidal_stash_456:target": 2, + "__taidal_stash_99:input": 3, + } + out = DropMetadataOp(exclude=["__taidal_stash_456:*input"])(Sample(input=np.zeros(2), metadata=meta)) + assert out.meta == {"__taidal_stash_456:target": 2, "__taidal_stash_99:input": 3} + + def test_multiple_exclude_patterns_any_match(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + meta = {"a": 1, "b": 2, "__t_x": 3, "__t_y": 4} + out = DropMetadataOp(exclude=["a", "__t_*"])(Sample(input=np.zeros(2), metadata=meta)) + assert out.meta == {"b": 2} + + def test_question_mark_and_set_globs(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + meta = {"img0": 1, "img1": 2, "imgX": 3, "image": 4} + out = DropMetadataOp(exclude=["img[0-9]"])(Sample(input=np.zeros(2), metadata=meta)) + assert out.meta == {"imgX": 3, "image": 4} # only single-digit img0/img1 dropped + + def test_matching_is_case_sensitive(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + out = DropMetadataOp(exclude=["key"])(Sample(input=np.zeros(2), metadata={"Key": 1, "key": 2})) + assert out.meta == {"Key": 1} + + def test_include_protects_keys_from_exclude(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + # include WINS: drop every stash key EXCEPT node 456's (carved out by include). + meta = { + "real": 1, + "__taidal_stash_456:input": 2, + "__taidal_stash_456:target": 3, + "__taidal_stash_99:input": 4, + } + out = DropMetadataOp(exclude=["__taidal_stash*"], include=["__taidal_stash_456:*"])( + Sample(input=np.zeros(2), metadata=meta) + ) + assert out.meta == {"real": 1, "__taidal_stash_456:input": 2, "__taidal_stash_456:target": 3} + + def test_include_without_exclude_drops_nothing(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + meta = {"a": 1, "b": 2} + out = DropMetadataOp(include=["a"])(Sample(input=np.zeros(2), metadata=meta)) + assert out.meta == {"a": 1, "b": 2} # include only protects against exclude + + def test_zero_arg_is_identity_metadata(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + meta = {"a": 1, "b": 2} + out = DropMetadataOp()(Sample(input=np.zeros(2), metadata=meta)) + assert out.meta == {"a": 1, "b": 2} + + def test_copy_on_write_does_not_mutate_original(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + original = {"a": 1, "drop": 2} + out = DropMetadataOp(exclude=["drop"])(Sample(input=np.zeros(2), metadata=original)) + assert original == {"a": 1, "drop": 2} # untouched + assert out.meta == {"a": 1} + + def test_input_and_target_untouched(self) -> None: + from dataflux.ops.metadata import DropMetadataOp + + arr = np.arange(3) + out = DropMetadataOp(exclude=["x"])(Sample(input=arr, target=7, metadata={"x": 1, "y": 2})) + np.testing.assert_array_equal(out.input, arr) + assert out.target == 7 + + +# --------------------------------------------------------------------------- +# PrintSampleOp +# --------------------------------------------------------------------------- +class TestPrintSampleOp: + def test_returns_sample_unchanged(self) -> None: + from dataflux.ops.debug import PrintSampleOp + + sample = Sample(input=np.zeros(3), target=1, metadata={"a": 1}) + out = PrintSampleOp(to_console=False)(sample) + assert out is sample + + def test_prints_to_console(self, capsys: pytest.CaptureFixture) -> None: + from dataflux.ops.debug import PrintSampleOp + + PrintSampleOp(label="probe")(Sample(input=np.zeros((2, 3)), target=None, metadata={"k": 1})) + captured = capsys.readouterr().out + assert "[probe #0]" in captured + assert "shape=(2, 3)" in captured # input summary + assert "'k'" in captured # metadata key + + def test_summarizes_large_array_metadata_without_dumping(self, capsys: pytest.CaptureFixture) -> None: + from dataflux.ops.debug import PrintSampleOp + + big = np.arange(100000, dtype=np.complex64) # would flood / not be reprable in full + PrintSampleOp(label="p")(Sample(input=np.zeros(2), metadata={"iq": big})) + out = capsys.readouterr().out + assert "shape=(100000,)" in out and "complex64" in out + assert "..." in out and "50000" not in out # values elided, not dumped in full + + def test_prints_small_array_values(self, capsys: pytest.CaptureFixture) -> None: + from dataflux.ops.debug import PrintSampleOp + + PrintSampleOp(label="p")(Sample(input=np.array([1, 2, 3]), target=None, metadata={})) + out = capsys.readouterr().out + assert "values=[1, 2, 3]" in out # actual values shown for a small array + + def test_limit_caps_emissions_but_passes_all(self, capsys: pytest.CaptureFixture) -> None: + from dataflux.ops.debug import PrintSampleOp + + op = PrintSampleOp(label="p", limit=2) + for _ in range(5): + assert op(Sample(input=np.zeros(1), metadata={})) is not None # all pass through + lines = [ln for ln in capsys.readouterr().out.splitlines() if ln.startswith("[p #")] + assert len(lines) == 2 # only the first 2 printed + + def test_to_console_false_is_silent_on_stdout(self, capsys: pytest.CaptureFixture) -> None: + from dataflux.ops.debug import PrintSampleOp + + PrintSampleOp(to_console=False)(Sample(input=np.zeros(1), metadata={})) + assert capsys.readouterr().out == "" From e1fc41992e2c3fa4226280f76beeb4f4c099f8f8 Mon Sep 17 00:00:00 2001 From: gertbehi Date: Sat, 20 Jun 2026 16:07:41 +0200 Subject: [PATCH 017/102] feat: add confusion matrix payload functions for FluxStudio viewer; include normalization and extraction utilities --- AGENTS.md | 2 +- dataflux/ops/image.py | 111 +++++++++++++++++++++++++++++++++++++++- tests/test_image_ops.py | 93 +++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 50e2b3a..db3480d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `dataflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `dataflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the LogFlow logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's *Array / Tensor Histogram* viewer (`fluxstudio.nodes.ArrayHistogramViewerNode`): `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. +- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py index 9d6fdee..ba21ac9 100644 --- a/dataflux/ops/image.py +++ b/dataflux/ops/image.py @@ -18,7 +18,7 @@ need it, so the pure-greyscale path stays matplotlib-free. """ -from typing import Any, Dict, Literal, Optional, Tuple, get_args +from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, get_args import numpy as np import torch @@ -357,6 +357,113 @@ def array_histogram(value: Any, bins: int = 256, channel: int = -1) -> Dict[str, } +def confusion_matrix_payload( + matrix: Any, + class_names: Optional[Sequence[Any]] = None, +) -> Dict[str, Any]: + """Structure a confusion matrix + class names into a JSON-safe payload for a GUI viewer. + + Backs FluxStudio's *Confusion Matrix* viewer node (``fluxstudio.nodes.ConfusionMatrixViewerNode``). + The MATH that lives here is the three normalizations (the viewer toggles between them WITHOUT a + re-run — the JS only colours + labels + hovers): ``true`` (each row / actual-class sums to 1), + ``pred`` (each column / predicted-class sums to 1) and ``all`` (the whole matrix sums to 1). Every + float is finite-checked (``NaN``/``±inf`` → ``None``, never a misleading substitute) so the payload + survives ComfyUI's ``json.dumps`` websocket encoding — mirroring :func:`array_histogram`. A row / + column whose count-sum is ``0`` normalises to ``None`` (undefined, not ``0``). + + Args: + matrix: A square ``N×N`` confusion matrix (integer counts) as an array / tensor / nested list. + class_names: Optional length-``N`` class labels; defaults to ``["0", "1", …, "N-1"]``. + + Returns a dict with ``counts`` (``N×N`` ints), ``normalized`` (``{"true","pred","all"}``, each + ``N×N`` floats or ``None``), ``class_names`` (length ``N``), ``n_classes``, and ``total``. A + non-square / empty / non-2-D input yields ``{"n_classes": 0, ...}`` + a ``message``. + """ + arr = _coerce_to_ndarray(matrix) + if arr is None or arr.ndim != 2 or arr.shape[0] != arr.shape[1] or arr.shape[0] == 0: + shape = None if arr is None else tuple(int(d) for d in arr.shape) + return { + "counts": [], + "normalized": {"true": [], "pred": [], "all": []}, + "class_names": [], + "n_classes": 0, + "total": 0, + "message": f"not a square 2-D confusion matrix (shape {shape})", + } + + counts = np.asarray(arr).astype(np.int64) + n = int(counts.shape[0]) + total = int(counts.sum()) + row_sums = counts.sum(axis=1) # per true class + col_sums = counts.sum(axis=0) # per predicted class + + def _normed(divisor: np.ndarray) -> list: + # Element-wise count / divisor; a 0 divisor (empty row/col/matrix) -> None (undefined). + out: list = [] + for i in range(n): + row: list = [] + for j in range(n): + d = float(divisor[i, j]) + row.append(_sanitize_finite(counts[i, j] / d) if d != 0.0 else None) + out.append(row) + return out + + names = [str(c) for c in class_names] if class_names is not None else [str(i) for i in range(n)] + # Pad / trim to exactly N so the viewer always has one label per row/column. + names = (names + [str(i) for i in range(len(names), n)])[:n] + + return { + "counts": [[int(c) for c in row] for row in counts.tolist()], + "normalized": { + "true": _normed(np.broadcast_to(row_sums.reshape(n, 1), (n, n))), + "pred": _normed(np.broadcast_to(col_sums.reshape(1, n), (n, n))), + "all": _normed(np.full((n, n), float(total))), + }, + "class_names": names, + "n_classes": n, + "total": total, + } + + +def _is_square_2d(value: Any) -> bool: + """True when ``value`` views as a square ``N×N`` (``N>=1``) numeric array — confusion-matrix shape.""" + arr = _coerce_to_ndarray(value) + return arr is not None and arr.ndim == 2 and arr.shape[0] == arr.shape[1] and arr.shape[0] >= 1 + + +def confusion_matrices_payload(metrics: Any, class_names: Optional[Sequence[Any]] = None) -> List[Dict[str, Any]]: + """Extract EVERY confusion matrix from a metrics result and build a render payload for each. + + The generic counterpart to :func:`confusion_matrix_payload`: a model evaluator emits its FULL + metric results (``name -> value``; scalars, vectors, AND `N×N` matrices) with NO knowledge of which + is a confusion matrix — this scans them and renders all CONFUSION-MATRIX-SHAPED entries (square 2-D, + ``_is_square_2d``, by SHAPE not name), returning one :func:`confusion_matrix_payload` per match + (each tagged with its metric ``name``) in dict order, or ``[]`` when none. A bare square-2D + ``metrics`` (not a dict) is treated as a single matrix named ``"confusion_matrix"``. This is what + lets FluxStudio's *Confusion Matrix* viewer render ALL matrices from one all-metrics output (there + can be several). ``class_names`` labels every matrix the same way (they share the class set). + + Args: + metrics: An evaluator's metric results — a ``dict`` of ``name -> value`` (the usual form), or a + single ``N×N`` matrix. + class_names: Optional length-``N`` class labels applied to each matrix; defaults to indices. + """ + items = list(metrics.items()) if isinstance(metrics, dict) else [("confusion_matrix", metrics)] + out: List[Dict[str, Any]] = [] + for name, value in items: + if _is_square_2d(value): + payload = confusion_matrix_payload(value, class_names=class_names) + payload["name"] = str(name) + out.append(payload) + return out + + +def _sanitize_finite(x: float) -> Optional[float]: + """A finite float rounded for compactness, or ``None`` for ``NaN``/``±inf`` (JSON-safe).""" + v = float(x) + return round(v, 6) if np.isfinite(v) else None + + # --------------------------------------------------------------------------- # # Text → image rendering — draw text onto an image (or a fresh canvas). # --------------------------------------------------------------------------- # @@ -607,6 +714,8 @@ def __call__(self, sample: Sample) -> Sample: "select_channel", "channel_count", "array_histogram", + "confusion_matrix_payload", + "confusion_matrices_payload", "draw_text", "TextPosition", "TEXT_POSITIONS", diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py index 1703534..4cfd0a2 100644 --- a/tests/test_image_ops.py +++ b/tests/test_image_ops.py @@ -6,6 +6,7 @@ waivefront (``RenderOverlaysOp``) and is tested there. """ +import json from typing import get_args import numpy as np @@ -23,6 +24,8 @@ _apply_colormap, array_histogram, channel_count, + confusion_matrices_payload, + confusion_matrix_payload, draw_text, sample_to_image, select_channel, @@ -357,6 +360,96 @@ def test_array_histogram_large_array_does_not_raise(arr: np.ndarray) -> None: assert sum(hist["counts"]) == hist["count"] == finite.size # every value still counted +# --------------------------------------------------------------------------- # +# confusion_matrix_payload — the math behind FluxStudio's Confusion Matrix viewer. +# --------------------------------------------------------------------------- # + + +def test_confusion_matrix_payload_counts_and_class_names() -> None: + m = np.array([[50, 2, 1], [3, 47, 0], [0, 1, 49]]) + p = confusion_matrix_payload(m, class_names=["cat", "dog", "fox"]) + assert p["n_classes"] == 3 and p["total"] == 153 + assert p["counts"] == [[50, 2, 1], [3, 47, 0], [0, 1, 49]] + assert p["class_names"] == ["cat", "dog", "fox"] + + +def test_confusion_matrix_payload_normalizations() -> None: + p = confusion_matrix_payload([[8, 2], [0, 10]]) + # true = row-normalized (each true-class row sums to 1) + assert p["normalized"]["true"] == [[0.8, 0.2], [0.0, 1.0]] + # pred = column-normalized (each predicted-class column sums to 1) + assert p["normalized"]["pred"] == [ + [1.0, pytest.approx(0.166667, abs=1e-5)], + [0.0, pytest.approx(0.833333, abs=1e-5)], + ] + # all = total-normalized + assert p["normalized"]["all"][0][0] == pytest.approx(8 / 20) + + +def test_confusion_matrix_payload_zero_row_is_none_not_nan() -> None: + # A class with no samples (empty row) normalizes to None (undefined), never 0/0 = NaN. + p = confusion_matrix_payload([[0, 0], [1, 3]]) + assert p["normalized"]["true"][0] == [None, None] + assert json.dumps(p, allow_nan=False) # JSON-safe: no bare NaN tokens + + +def test_confusion_matrix_payload_defaults_to_index_labels() -> None: + p = confusion_matrix_payload([[1, 0], [0, 1]]) + assert p["class_names"] == ["0", "1"] + + +def test_confusion_matrix_payload_pads_or_trims_class_names_to_n() -> None: + assert confusion_matrix_payload([[1, 0], [0, 1]], class_names=["only"])["class_names"] == ["only", "1"] + assert confusion_matrix_payload([[1, 0], [0, 1]], class_names=["a", "b", "c"])["class_names"] == ["a", "b"] + + +def test_confusion_matrix_payload_non_square_is_well_formed() -> None: + p = confusion_matrix_payload(np.zeros((2, 3))) + assert p["n_classes"] == 0 and "message" in p + p2 = confusion_matrix_payload("not a matrix") + assert p2["n_classes"] == 0 + + +def test_confusion_matrix_payload_accepts_torch_tensor() -> None: + p = confusion_matrix_payload(torch.tensor([[5, 1], [0, 4]])) + assert p["counts"] == [[5, 1], [0, 4]] and p["n_classes"] == 2 + + +# --------------------------------------------------------------------------- # +# confusion_matrices_payload — extract EVERY confusion matrix from a metrics result. +# --------------------------------------------------------------------------- # + + +def test_confusion_matrices_payload_extracts_all_square_2d_entries() -> None: + # A full metrics dict: scalars + a 1-D vector + TWO confusion matrices. Only the square-2D + # entries are extracted, in dict order, each tagged with its metric name. + metrics = { + "test/acc": 0.93, + "test/cm_a": [[5, 1], [0, 4]], + "test/per_class": [0.9, 0.8], # 1-D vector — NOT a confusion matrix + "test/cm_b": [[10, 2, 1], [0, 9, 1], [1, 0, 8]], + } + payloads = confusion_matrices_payload(metrics, class_names=["a", "b", "c"]) + assert [(p["name"], p["n_classes"]) for p in payloads] == [("test/cm_a", 2), ("test/cm_b", 3)] + # class_names are trimmed per matrix (cm_a has only 2 classes). + assert payloads[0]["class_names"] == ["a", "b"] + assert json.dumps(payloads, allow_nan=False) # JSON-safe + + +def test_confusion_matrices_payload_none_when_no_square_metric() -> None: + assert confusion_matrices_payload({"acc": 0.9, "vec": [1, 2, 3]}) == [] + + +def test_confusion_matrices_payload_bare_matrix_is_one_named_default() -> None: + payloads = confusion_matrices_payload([[1, 0], [0, 1]]) + assert [p["name"] for p in payloads] == ["confusion_matrix"] + + +def test_confusion_matrices_payload_accepts_tensor_values() -> None: + payloads = confusion_matrices_payload({"cm": torch.tensor([[5, 1], [0, 4]])}) + assert payloads[0]["name"] == "cm" and payloads[0]["counts"] == [[5, 1], [0, 4]] + + # --------------------------------------------------------------------------- # draw_text — render text onto an image / a fresh canvas # --------------------------------------------------------------------------- From 9a568b48d4ca7da26c6dc9b29470cf589f57ca07 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 5 Jul 2026 13:31:03 +0200 Subject: [PATCH 018/102] chore: depend on logflow-ml>=0.2.0 (log-flow dist renamed for PyPI) The logflow distribution was renamed log-flow -> logflow-ml (PyPI ultranormalization similarity check rejects log-flow vs the existing abandoned logflow project). Import package unchanged (import logflow). --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 962f473..1b765f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Clean, functional data pipelines for ML research and production." authors = [{ name = "Taidal", email = "info@gearlux.ai" }] dependencies = [ "confluid>=0.1.0", - "log-flow>=0.1.0", + "logflow-ml>=0.2.0", "numpy", "Pillow", "h5py", From 06ebae50d443c34721705d9cd945fbfbb1b207df Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 6 Jul 2026 18:46:33 +0200 Subject: [PATCH 019/102] chore: migrate logflow -> loggair (complete rename) The logging library was renamed and republished as loggair (github.com/Gearlux/loggair, dist == import == loggair, v0.1.0): imports `logflow` -> `loggair`, dependency logflow-ml>=0.2.0 -> loggair>=0.1.0, LOGFLOW_* env vars -> LOGGAIR_*, docs/CI references updated. --- .github/workflows/ci.yml | 8 ++++---- AGENTS.md | 2 +- Jenkinsfile | 2 +- Jenkinsfile.local | 2 +- README.md | 2 +- dataflux/core.py | 4 ++-- dataflux/ops/debug.py | 6 +++--- dataflux/ops/enable.py | 2 +- dataflux/ops/image.py | 2 +- dataflux/ops/numpy.py | 2 +- dataflux/ops/random_apply.py | 2 +- dataflux/ops/sink.py | 2 +- dataflux/ops/transform_chain.py | 2 +- dataflux/paired.py | 2 +- dataflux/sources.py | 2 +- dataflux/storage/cache.py | 2 +- dataflux/storage/hdf5.py | 2 +- pyproject.toml | 2 +- 18 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5573dc..68314f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" - name: Run Isort @@ -58,7 +58,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" - name: Run Tests @@ -92,7 +92,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" - name: Run Examples @@ -121,7 +121,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" # Notebook-only dependencies live in the optional `[notebook]` extra diff --git a/AGENTS.md b/AGENTS.md index db3480d..03c8b98 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `dataflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `dataflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the LogFlow logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `dataflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `dataflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/Jenkinsfile b/Jenkinsfile index aed8f21..4bd9b2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,7 +33,7 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/log-flow.git@main" + sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live diff --git a/Jenkinsfile.local b/Jenkinsfile.local index fe3f31e..cc534d7 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,7 +42,7 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/logflow" + sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/loggair" sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/confluid" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live diff --git a/README.md b/README.md index 6da1025..d7995b6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **DataFlux** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. -Part of the **Modular Quartet**: `LogFlow`, `Confluid`, `Liquify`, and `DataFlux`. +Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquify`, and `DataFlux`. ## 🚀 Key Features diff --git a/dataflux/core.py b/dataflux/core.py index dd13460..567c1b7 100644 --- a/dataflux/core.py +++ b/dataflux/core.py @@ -23,7 +23,7 @@ from confluid import load as _confluid_load from confluid import materialize as _confluid_materialize from confluid.fluid import Fluid as _ConfluidFluid -from logflow import get_logger +from loggair import get_logger from dataflux.projection import ProjectionField from dataflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, Sample @@ -481,7 +481,7 @@ def _iter_parallel(self) -> Iterator[Sample]: return _check_ops_materialized(self.ops) - # We use 'spawn' to be consistent with LogFlow and prevent CI deadlocks + # We use 'spawn' to be consistent with Loggair and prevent CI deadlocks ctx = multiprocessing.get_context("spawn") with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: diff --git a/dataflux/ops/debug.py b/dataflux/ops/debug.py index d611581..0bb6a73 100644 --- a/dataflux/ops/debug.py +++ b/dataflux/ops/debug.py @@ -3,7 +3,7 @@ from typing import Any, Literal, Optional from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample @@ -60,13 +60,13 @@ class PrintSampleOp: A pipeline probe: emits a compact description of the sample — ``input`` / ``target`` shape+dtype plus a length-capped value preview (large arrays elided), and the ``metadata`` (values - summarised the same way) — to the LogFlow logger (the LOG file + console) and, by default, to stdout via + summarised the same way) — to the Loggair logger (the LOG file + console) and, by default, to stdout via ``print`` (so it shows in a terminal / the FluxStudio node output panel regardless of log level). The sample is returned UNCHANGED. Args: label: A prefix identifying this probe in the output (e.g. "after-impairments"). - level: LogFlow level for the logged line — "trace" or "debug" (per-sample output is + level: Loggair level for the logged line — "trace" or "debug" (per-sample output is diagnostic, so info/warning are deliberately not offered; use ``to_console`` to see it). include_data: Include an ``input`` / ``target`` shape+dtype + value preview. include_metadata: Include the sample's metadata (values summarised). diff --git a/dataflux/ops/enable.py b/dataflux/ops/enable.py index a91b773..dbdbef2 100644 --- a/dataflux/ops/enable.py +++ b/dataflux/ops/enable.py @@ -9,7 +9,7 @@ from typing import List, Optional, Tuple from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample diff --git a/dataflux/ops/image.py b/dataflux/ops/image.py index ba21ac9..cc8b5ba 100644 --- a/dataflux/ops/image.py +++ b/dataflux/ops/image.py @@ -23,7 +23,7 @@ import numpy as np import torch from confluid import configurable -from logflow import get_logger +from loggair import get_logger from PIL import Image, ImageDraw from dataflux.sample import Sample diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py index 5efccc9..b97a54c 100644 --- a/dataflux/ops/numpy.py +++ b/dataflux/ops/numpy.py @@ -5,7 +5,7 @@ import numpy as np from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType diff --git a/dataflux/ops/random_apply.py b/dataflux/ops/random_apply.py index 958ed43..df2dce4 100644 --- a/dataflux/ops/random_apply.py +++ b/dataflux/ops/random_apply.py @@ -12,7 +12,7 @@ from typing import Optional, cast from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample diff --git a/dataflux/ops/sink.py b/dataflux/ops/sink.py index 32e0ae5..711a07a 100644 --- a/dataflux/ops/sink.py +++ b/dataflux/ops/sink.py @@ -10,7 +10,7 @@ from typing import Any from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample diff --git a/dataflux/ops/transform_chain.py b/dataflux/ops/transform_chain.py index 81e21d5..c177a2b 100644 --- a/dataflux/ops/transform_chain.py +++ b/dataflux/ops/transform_chain.py @@ -15,7 +15,7 @@ from typing import List, Optional from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample diff --git a/dataflux/paired.py b/dataflux/paired.py index 940ba57..8bf06a4 100644 --- a/dataflux/paired.py +++ b/dataflux/paired.py @@ -27,7 +27,7 @@ ) from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.discovery import get_callable_path, resolve_callable from dataflux.sample import Sample diff --git a/dataflux/sources.py b/dataflux/sources.py index 7237832..ca2193a 100644 --- a/dataflux/sources.py +++ b/dataflux/sources.py @@ -3,7 +3,7 @@ from typing import Any, Dict, Iterator, List, Literal, Optional, get_args from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample diff --git a/dataflux/storage/cache.py b/dataflux/storage/cache.py index 036888c..4c5b36b 100644 --- a/dataflux/storage/cache.py +++ b/dataflux/storage/cache.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Callable, List, Tuple, Union -from logflow import get_logger +from loggair import get_logger logger = get_logger(__name__) diff --git a/dataflux/storage/hdf5.py b/dataflux/storage/hdf5.py index 9c17bbf..0ac3ed6 100644 --- a/dataflux/storage/hdf5.py +++ b/dataflux/storage/hdf5.py @@ -5,7 +5,7 @@ import numpy as np import torch from confluid import configurable -from logflow import get_logger +from loggair import get_logger from dataflux.sample import Sample from dataflux.storage.base import DataSink, DataSource, Storage, to_numpy diff --git a/pyproject.toml b/pyproject.toml index 1b765f7..aa491b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Clean, functional data pipelines for ML research and production." authors = [{ name = "Taidal", email = "info@gearlux.ai" }] dependencies = [ "confluid>=0.1.0", - "logflow-ml>=0.2.0", + "loggair>=0.1.0", "numpy", "Pillow", "h5py", From 6ed14b30aa870637ef62962846d375836ce9d601 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 14 Jul 2026 13:42:19 +0200 Subject: [PATCH 020/102] chore: run GitHub CI push trigger on main only; replace committed .vscode extraPaths with generated pyrightconfig.json --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + .vscode/settings.json | 26 -------------------------- 3 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68314f0..a56b9cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ name: Dataflux CI on: push: - branches: [ main, dev/main ] + branches: [ main ] pull_request: branches: [ main ] diff --git a/.gitignore b/.gitignore index dae5208..2d29224 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ mypy.txt coverage.xml test-report.xml coverage/ +/pyrightconfig.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 8c67431..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "python.testing.pytestEnabled": true, - "python.testing.pytestArgs": [ - "tests" - ], - "python-envs.pythonProjects": [ - { - "path": ".", - "envManager": "ms-python.python:venv", - "packageManager": "ms-python.python:pip" - } - ], - "python.analysis.extraPaths": [ - "${workspaceFolder}/../logflow", - "${workspaceFolder}/../confluid", - "${workspaceFolder}/../liquify", - "${workspaceFolder}/../waivefront", - "${workspaceFolder}/../torpedo", - "${workspaceFolder}/../navigaitor", - "${workspaceFolder}/../aisland", - "${workspaceFolder}/../annotaide", - "${workspaceFolder}/../marainer", - "${workspaceFolder}/../waivefront-helios", - "${workspaceFolder}/../waivefront-rfuav" - ] -} From 91455700c7483d94aebfad7f70a9cbc1fb40550b Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 18 Jul 2026 05:05:37 +0200 Subject: [PATCH 021/102] feat: rename to sampleflux + graph execution model, multi-type engine, 1->N ops, SigMF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete identity switch from dataflux (taken on PyPI): package dir, dist name, entry points, docs. On top of the rename, the FlowGraph plan lands: - context.py: per-sample named-cell Context (the graph data plane, never sample.metadata) activated via a ContextVar around the plain op loop. - ops/context.py: Save/Use/Drop/Apply/Capture/Mix — the flat-list building blocks that let the serial Flux engine execute fan-out/fan-in graphs. - flow.py: the readable named-step flow: document, the FlowGraph engine, and BIDIRECTIONAL converters to_ops/from_ops with pinned execution parity both ways (Flux.from_flow_yaml / FlowGraph.from_yaml/from_ops_yaml). - kinds.py: op-contract introspection (sample/pair/value/any taxonomy, EXPANDS detection, class-attr overrides); Flux(native=True) carries metadata-free pairs/values with per-op adaptation. - collate.py: pluggable collate registry (sample/pair/value defaults, additive task aliases). - 1->N expanding ops: generator-returning ops flatten depth-first across all routes; expanding pipelines are iterable-only (len/getitem raise). - storage/sigmf.py: SigMFSink<->SigMFSource recording pair (hand-rolled JSON, dtype<->core:datatype, meta_encoder/decoder hooks). - storage/query.py: SupportsMetadataScan protocol (HDF5/Zarr/SigMF scan metadata without array loads) + MetadataFilterSource (where expressions). 951 tests; examples/flow_graph.py runs the full flow<->ops round-trip. --- .coveragerc | 2 +- .github/workflows/ci.yml | 8 +- AGENTS.md | 29 +- Jenkinsfile | 38 +- Jenkinsfile.local | 42 +- README.md | 245 ++++-- dataflux/__init__.py | 67 -- dataflux/ops/__init__.py | 100 --- examples/advanced_storage_demo.py | 8 +- examples/basic_pipeline.py | 4 +- examples/cache_pipeline.py | 6 +- examples/dataset_split.yaml | 24 +- examples/discovery_demo.py | 2 +- examples/flow_graph.py | 84 ++ examples/hdf5_pipeline.py | 6 +- examples/paired_annotations.py | 6 +- examples/parallel_hdf5_stream.py | 4 +- examples/parallel_pipeline.py | 2 +- examples/storage_roundtrip.py | 10 +- pyproject.toml | 65 +- sampleflux/__init__.py | 82 ++ sampleflux/collate.py | 127 +++ sampleflux/context.py | 120 +++ {dataflux => sampleflux}/core.py | 288 +++++-- {dataflux => sampleflux}/discovery.py | 4 +- sampleflux/flow.py | 781 ++++++++++++++++++ sampleflux/kinds.py | 151 ++++ {dataflux => sampleflux}/labels.py | 16 +- sampleflux/ops/__init__.py | 109 +++ {dataflux => sampleflux}/ops/capture.py | 6 +- {dataflux => sampleflux}/ops/configure.py | 10 +- sampleflux/ops/context.py | 316 +++++++ {dataflux => sampleflux}/ops/copy.py | 2 +- {dataflux => sampleflux}/ops/debug.py | 2 +- {dataflux => sampleflux}/ops/enable.py | 14 +- {dataflux => sampleflux}/ops/formula.py | 2 +- {dataflux => sampleflux}/ops/image.py | 20 +- {dataflux => sampleflux}/ops/metadata.py | 2 +- {dataflux => sampleflux}/ops/numpy.py | 20 +- {dataflux => sampleflux}/ops/parallel.py | 6 +- {dataflux => sampleflux}/ops/random_apply.py | 8 +- {dataflux => sampleflux}/ops/sink.py | 12 +- {dataflux => sampleflux}/ops/stash.py | 2 +- {dataflux => sampleflux}/ops/swap.py | 2 +- {dataflux => sampleflux}/ops/target.py | 24 +- {dataflux => sampleflux}/ops/tee.py | 2 +- {dataflux => sampleflux}/ops/torch.py | 16 +- .../ops/transform_chain.py | 16 +- {dataflux => sampleflux}/paired.py | 6 +- {dataflux => sampleflux}/projection.py | 12 +- {dataflux => sampleflux}/py.typed | 0 {dataflux => sampleflux}/sample.py | 12 +- {dataflux => sampleflux}/sources.py | 16 +- {dataflux => sampleflux}/storage/base.py | 6 +- {dataflux => sampleflux}/storage/cache.py | 0 {dataflux => sampleflux}/storage/directory.py | 6 +- {dataflux => sampleflux}/storage/hdf5.py | 18 +- sampleflux/storage/query.py | 172 ++++ sampleflux/storage/sigmf.py | 270 ++++++ {dataflux => sampleflux}/storage/zarr.py | 16 +- {dataflux => sampleflux}/typespec.py | 6 +- {dataflux => sampleflux}/windows.py | 8 +- tests/test_cache.py | 2 +- tests/test_categories.py | 62 +- tests/test_context.py | 442 ++++++++++ tests/test_coverage_gap.py | 16 +- tests/test_discovery.py | 6 +- tests/test_enable.py | 12 +- tests/test_expanding_ops.py | 182 ++++ tests/test_flow.py | 396 +++++++++ tests/test_flux.py | 8 +- tests/test_fourier_ops.py | 46 +- tests/test_from_ops_yaml.py | 8 +- tests/test_image_ops.py | 6 +- tests/test_joint.py | 4 +- tests/test_kinds.py | 262 ++++++ tests/test_labels.py | 10 +- tests/test_lazy_construction.py | 22 +- tests/test_node_docs.py | 30 +- tests/test_ops.py | 54 +- tests/test_paired.py | 8 +- tests/test_parallel.py | 2 +- tests/test_parallel_op.py | 8 +- tests/test_projection.py | 6 +- tests/test_random_apply.py | 6 +- tests/test_sample.py | 4 +- tests/test_sigmf.py | 175 ++++ tests/test_sources.py | 30 +- tests/test_storage.py | 12 +- tests/test_target_ops.py | 6 +- tests/test_transform_chain.py | 6 +- tests/test_typespec.py | 14 +- tests/test_windows.py | 6 +- 93 files changed, 4595 insertions(+), 716 deletions(-) delete mode 100644 dataflux/__init__.py delete mode 100644 dataflux/ops/__init__.py create mode 100644 examples/flow_graph.py create mode 100644 sampleflux/__init__.py create mode 100644 sampleflux/collate.py create mode 100644 sampleflux/context.py rename {dataflux => sampleflux}/core.py (61%) rename {dataflux => sampleflux}/discovery.py (98%) create mode 100644 sampleflux/flow.py create mode 100644 sampleflux/kinds.py rename {dataflux => sampleflux}/labels.py (90%) create mode 100644 sampleflux/ops/__init__.py rename {dataflux => sampleflux}/ops/capture.py (97%) rename {dataflux => sampleflux}/ops/configure.py (93%) create mode 100644 sampleflux/ops/context.py rename {dataflux => sampleflux}/ops/copy.py (97%) rename {dataflux => sampleflux}/ops/debug.py (99%) rename {dataflux => sampleflux}/ops/enable.py (94%) rename {dataflux => sampleflux}/ops/formula.py (98%) rename {dataflux => sampleflux}/ops/image.py (98%) rename {dataflux => sampleflux}/ops/metadata.py (98%) rename {dataflux => sampleflux}/ops/numpy.py (98%) rename {dataflux => sampleflux}/ops/parallel.py (95%) rename {dataflux => sampleflux}/ops/random_apply.py (92%) rename {dataflux => sampleflux}/ops/sink.py (86%) rename {dataflux => sampleflux}/ops/stash.py (99%) rename {dataflux => sampleflux}/ops/swap.py (93%) rename {dataflux => sampleflux}/ops/target.py (94%) rename {dataflux => sampleflux}/ops/tee.py (98%) rename {dataflux => sampleflux}/ops/torch.py (97%) rename {dataflux => sampleflux}/ops/transform_chain.py (83%) rename {dataflux => sampleflux}/paired.py (98%) rename {dataflux => sampleflux}/projection.py (93%) rename {dataflux => sampleflux}/py.typed (100%) rename {dataflux => sampleflux}/sample.py (92%) rename {dataflux => sampleflux}/sources.py (97%) rename {dataflux => sampleflux}/storage/base.py (89%) rename {dataflux => sampleflux}/storage/cache.py (100%) rename {dataflux => sampleflux}/storage/directory.py (90%) rename {dataflux => sampleflux}/storage/hdf5.py (88%) create mode 100644 sampleflux/storage/query.py create mode 100644 sampleflux/storage/sigmf.py rename {dataflux => sampleflux}/storage/zarr.py (91%) rename {dataflux => sampleflux}/typespec.py (99%) rename {dataflux => sampleflux}/windows.py (97%) create mode 100644 tests/test_context.py create mode 100644 tests/test_expanding_ops.py create mode 100644 tests/test_flow.py create mode 100644 tests/test_kinds.py create mode 100644 tests/test_sigmf.py diff --git a/.coveragerc b/.coveragerc index 87cdc8e..f034c22 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -source = dataflux +source = sampleflux omit = */tests/* */examples/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a56b9cf..d6ad4bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,10 @@ # ========================================================================= # AUTO-GENERATED FILE — DO NOT EDIT BY HAND -# Generated by: aisland jenkins scaffold --project dataflux +# Generated by: aisland jenkins scaffold --project sampleflux # Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -# To regenerate: aisland jenkins scaffold --project dataflux --force +# To regenerate: aisland jenkins scaffold --project sampleflux --force # ========================================================================= -name: Dataflux CI +name: Sampleflux CI on: push: @@ -64,7 +64,7 @@ jobs: - name: Run Tests run: | if [ -d tests ] && find tests -name '*.py' | grep -q .; then - pytest tests --junitxml=test-report.xml --cov=dataflux --cov-report=xml --cov-report=term + pytest tests --junitxml=test-report.xml --cov=sampleflux --cov-report=xml --cov-report=term else echo "No tests found. Skipping." fi diff --git a/AGENTS.md b/AGENTS.md index 03c8b98..bd8542a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,22 +1,27 @@ -# DataFlux Mandates +# SampleFlux Mandates - **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. - **`Sample.metadata` Is `dict` (single) OR `list[dict]` (batch) — Narrow via `.meta` / `.batch_meta`:** The `metadata` field is `Metadata = Union[Dict[str, Any], List[Dict[str, Any]]]`. A **single** item carries one `dict` (the normal pipeline form every source/op produces and consumes); a **batch** carries a `list` of per-item dicts (one per stacked item), produced by the collate functions (`marainer.collate.collate_fn_with_metadata`, `sonair.classification.classification_collate_fn`) when N samples are stacked into one Sample for the model/loss/predictions-sinks. `Sample.is_batched` (= `isinstance(metadata, list)`) is the single source of truth for telling them apart. Per-sample code MUST read/mutate metadata through the narrowing accessor **`sample.meta`** (returns the dict, raises `TypeError` on a batch) — `sample.meta[key]` / `sample.meta[key] = v`; batch consumers use **`sample.batch_meta`** (returns the list, raises on a single). NEVER index the raw `sample.metadata` Union directly (mypy rejects `Union[...][str]`). NOTE the batch convention is per-collate: marainer/sonair stack into the **list** form (`is_batched` True); deltaid's `segmentation_collate_fn` instead nests under a **dict** `metadata={"per_sample": [...]}` (so `is_batched` is False there — use `.meta["per_sample"]`). `describe()`/`with_type()` operate on single samples only (a batch infers / raises). Pins: `tests/test_sample.py` (batch vs single, `.meta`/`.batch_meta` guards). - **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY dataflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **The Context Is the Graph Data Plane — Never `sample.metadata` (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its `input`, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `Mix` (fan-in; named slots read cells, empty slots keep the incoming sample; metadata merges incoming-first then slot order, `metadata_from` wins last). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches `sample.metadata` — a linear run's metadata is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`), mirroring `UnstashInputOp(copy=True, remove=True)` — the stash family remains the METADATA-bus twin for hand-written configs, the context ops are what `flow:` documents/FluxStudio lower to; (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. +- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `target_from`/`metadata_from` (fan-in slots, Mix field semantics), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Op Kind Is INTROSPECTED, Never Declared in the Engine (`sampleflux.kinds`, 2026-07-17):** The native multi-type engine (`Flux(native=True)`, OPT-IN — the `native=False` default coerces to `Sample` exactly as before, so all consumers are untouched) carries `Sample` triplets, metadata-free **pairs** (2-tuples), and bare **values** through one pipeline, adapting each op via `op_contract(op)` → `OpContract(accepts, produces, expands)` cached per type: `__call__`'s first-param annotation (`Sample`→`sample`, `tuple[...]`→`pair`, missing/`Any`→`any`) and return annotation (`Iterator[...]`/`Iterable[...]`/`List[...]` → `expands=True` — a `Tuple` return is a PAIR, never an expansion). ANY introspection failure (lazy imports, unresolvable forward refs) degrades to `any` so an untyped/exotic op behaves exactly as today; the class attrs `SAMPLE_KIND_IN`/`SAMPLE_KIND_OUT`/`EXPANDS` are the explicit escape hatch and ALWAYS win. Adaptation rules (`core._apply_op_native`): a pair-op on a Sample gets `sample.to_pair()` and its returned pair merges back via `_replace` (METADATA PRESERVED); a sample-op on a pair/value gets a PROMOTED `Sample.from_any` view — promotion is one-way and STICKY (op-written metadata is never dropped); an any-op gets the carrier verbatim. `SampleKind` is a closed Literal (`sample`/`pair`/`value`/`any`; runtime tuple `SAMPLE_KINDS = get_args(...)` — one source of truth); `classify_carrier` is the runtime classifier (ONLY a 2-tuple is a pair). **Collation is the pluggable registry `sampleflux.collate`** (`register_collate(key)` / `get_collate` / `collate(items, key=None)` — key defaults to the detected kind): sampleflux registers `"sample"` (list-form batched metadata — the `is_batched` convention) / `"pair"` / `"value"` defaults; consumers register task aliases ADDITIVELY and their divergent conventions (deltaid/raidar `{"per_sample": …}`) are deliberately NOT unified (TASKS.md follow-up). Pins: `tests/test_kinds.py`. +- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op whose return annotation is `Iterator[...]`/`Iterable[...]`/`List[...]` (or that carries `EXPANDS = True`) is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op's contract expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `HDF5WindowSource`/`RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. `_refresh_type` is applied per CHILD (`core._expand`). Pins: `tests/test_expanding_ops.py`. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. -- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `dataflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `DATAFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `dataflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`dataflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. -- **Field Projection (`dataflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `dataflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). -- **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`dataflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing dataflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a dataflux dependency for this. -- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The dataflux buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `DATAFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `DATAFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `DATAFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `DATAFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `dataflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `dataflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `dataflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `dataflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`dataflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`dataflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`dataflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`dataflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `DATAFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`dataflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`dataflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`dataflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `dataflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/DataFlux/Op/`): dataflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `dataflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `dataflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`dataflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in dataflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. +- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `sampleflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. +- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `SAMPLEFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **SigMF Is the Waveform Recording Carrier; Metadata Is QUERYABLE Without Array Loads (2026-07-17):** `sampleflux.storage.sigmf` holds the `SigMFSink`↔`SigMFSource` pair (one `.sigmf-data` + `.sigmf-meta` recording per sample; hand-rolled JSON — no `sigmf` dependency; dtype↔`core:datatype` via `_DTYPE_TO_SIGMF`, unsupported dtypes raise; optional `core:sha512`). sampleflux stays DOMAIN-NEUTRAL: the default encoder/decoder pass `core:*` keys verbatim and namespace everything else `sampleflux:`; the WAVEFORM vocabulary (`samplerate`↔`core:sample_rate` incl. the torchsig `sample_rate` spelling, `center_freq`→the capture's `core:frequency` — SCOPE resolves the torchsig per-signal collision, string `snr`→`waivefront:snr_raw`/`snr_db`, `{role}_regions`/`labels`↔annotations with `waivefront:role`) lives in **`waivefront.vocab`** and plugs in via the `meta_encoder`/`meta_decoder` hooks (dotted `module.attr` or `module:attr` paths, lazily resolved). A JSON-able target rides `sampleflux:target` (SigMF is input-centric; array targets skip with a debug note). **Queryability**: `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; implemented on `HDF5Source` (attrs + array-metadata shape/dtype STUBS), `ZarrGroupSource` (`.zattrs`), `SigMFSource` (meta JSON) — existing files queryable with NO rewrite) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND/composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration (the projection-module pattern). Entry points `sampleflux-storage-sigmf`/`-query`. No index sidecar in v1 (TASKS.md). Pins: `tests/test_sigmf.py`, `waivefront/tests/test_vocab.py`. +- **Field Projection (`sampleflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `sampleflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). +- **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`sampleflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing sampleflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a sampleflux dependency for this. +- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The sampleflux buckets are deliberately split by ROLE, not lumped under "dataset": + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `sampleflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `sampleflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `sampleflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `sampleflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`sampleflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `sampleflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). -- **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). +- **Type Specs Live in `sampleflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). - **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. ## Testing & Validation diff --git a/Jenkinsfile b/Jenkinsfile index 4bd9b2b..7bd75d0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,8 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project dataflux +// Generated by: aisland jenkins scaffold --project sampleflux // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project dataflux --force +// To regenerate: aisland jenkins scaffold --project sampleflux --force // ========================================================================= pipeline { agent any @@ -51,7 +51,7 @@ pipeline { steps { script { sh "rm -f black-diff.txt black-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/black --check --diff ${targets} 2>&1 | tee black-diff.txt'", returnStatus: true) @@ -87,8 +87,8 @@ with open('black-checkstyle.xml', 'w') as f: script { if (fileExists('black-checkstyle.xml')) { recordIssues( - id: 'black-dataflux', - name: 'Black Formatting (Dataflux)', + id: 'black-sampleflux', + name: 'Black Formatting (Sampleflux)', tools: [checkStyle(pattern: 'black-checkstyle.xml')] ) } @@ -100,7 +100,7 @@ with open('black-checkstyle.xml', 'w') as f: steps { script { sh "rm -f isort-diff.txt isort-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/isort --check-only --diff ${targets} 2>&1 | tee isort-diff.txt'", returnStatus: true) @@ -136,8 +136,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('isort-checkstyle.xml')) { recordIssues( - id: 'isort-dataflux', - name: 'Isort Import Order (Dataflux)', + id: 'isort-sampleflux', + name: 'Isort Import Order (Sampleflux)', tools: [checkStyle(pattern: 'isort-checkstyle.xml')] ) } @@ -149,7 +149,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { sh "rm -f flake8.txt" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { sh "${VENV_BIN}/flake8 ${targets} --tee --output-file=flake8.txt" } else { @@ -162,8 +162,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('flake8.txt') && readFile('flake8.txt').trim()) { recordIssues( - id: 'flake8-dataflux', - name: 'Flake8 (Dataflux)', + id: 'flake8-sampleflux', + name: 'Flake8 (Sampleflux)', tools: [flake8(pattern: 'flake8.txt')] ) } @@ -192,8 +192,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('mypy.txt') && readFile('mypy.txt').trim()) { recordIssues( - id: 'mypy-dataflux', - name: 'Mypy (Dataflux)', + id: 'mypy-sampleflux', + name: 'Mypy (Sampleflux)', tools: [myPy(pattern: 'mypy.txt')] ) } @@ -208,7 +208,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { if (fileExists('tests') && sh(script: "find tests -name '*.py' | grep -q .", returnStatus: true) == 0) { - sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=dataflux --cov-report=xml:coverage.xml --cov-report=term" + sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=sampleflux --cov-report=xml:coverage.xml --cov-report=term" } else { echo "No tests found in 'tests' directory. Skipping." } @@ -222,8 +222,8 @@ with open('isort-checkstyle.xml', 'w') as f: } if (fileExists('coverage.xml')) { recordCoverage( - id: 'coverage-dataflux', - name: 'Dataflux Coverage', + id: 'coverage-sampleflux', + name: 'Sampleflux Coverage', tools: [[parser: 'COBERTURA', pattern: 'coverage.xml']] ) } @@ -289,13 +289,13 @@ with open('isort-checkstyle.xml', 'w') as f: post { always { - echo 'Dataflux Pipeline Complete.' + echo 'Sampleflux Pipeline Complete.' } success { - echo 'Dataflux is healthy.' + echo 'Sampleflux is healthy.' } failure { - echo 'Dataflux build failed. Please check linting or test failures.' + echo 'Sampleflux build failed. Please check linting or test failures.' } } } diff --git a/Jenkinsfile.local b/Jenkinsfile.local index cc534d7..f02e8f1 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -1,14 +1,14 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project dataflux +// Generated by: aisland jenkins scaffold --project sampleflux // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project dataflux --force +// To regenerate: aisland jenkins scaffold --project sampleflux --force // ========================================================================= pipeline { agent { node { label 'built-in' - customWorkspace "${env.WORKSPACE_ROOT}/dataflux" + customWorkspace "${env.WORKSPACE_ROOT}/sampleflux" } } @@ -60,7 +60,7 @@ pipeline { steps { script { sh "rm -f black-diff.txt black-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/black --check --diff ${targets} 2>&1 | tee black-diff.txt'", returnStatus: true) @@ -96,8 +96,8 @@ with open('black-checkstyle.xml', 'w') as f: script { if (fileExists('black-checkstyle.xml')) { recordIssues( - id: 'black-dataflux', - name: 'Black Formatting (Dataflux)', + id: 'black-sampleflux', + name: 'Black Formatting (Sampleflux)', tools: [checkStyle(pattern: 'black-checkstyle.xml')] ) } @@ -109,7 +109,7 @@ with open('black-checkstyle.xml', 'w') as f: steps { script { sh "rm -f isort-diff.txt isort-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/isort --check-only --diff ${targets} 2>&1 | tee isort-diff.txt'", returnStatus: true) @@ -145,8 +145,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('isort-checkstyle.xml')) { recordIssues( - id: 'isort-dataflux', - name: 'Isort Import Order (Dataflux)', + id: 'isort-sampleflux', + name: 'Isort Import Order (Sampleflux)', tools: [checkStyle(pattern: 'isort-checkstyle.xml')] ) } @@ -158,7 +158,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { sh "rm -f flake8.txt" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { sh "${VENV_BIN}/flake8 ${targets} --tee --output-file=flake8.txt" } else { @@ -171,8 +171,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('flake8.txt') && readFile('flake8.txt').trim()) { recordIssues( - id: 'flake8-dataflux', - name: 'Flake8 (Dataflux)', + id: 'flake8-sampleflux', + name: 'Flake8 (Sampleflux)', tools: [flake8(pattern: 'flake8.txt')] ) } @@ -187,7 +187,7 @@ with open('isort-checkstyle.xml', 'w') as f: // Run from workspace root so root mypy.ini is picked up and cross-project imports resolve. // tee streams errors to the Jenkins console; pipefail ensures mypy's exit code (not tee's) propagates // so the stage fails visibly when types break. - def exitCode = sh(script: "bash -c 'set -o pipefail; cd ${env.WORKSPACE_ROOT} && ${VENV_BIN}/mypy dataflux 2>&1 | tee ${env.WORKSPACE_ROOT}/dataflux/mypy.txt'", returnStatus: true) + def exitCode = sh(script: "bash -c 'set -o pipefail; cd ${env.WORKSPACE_ROOT} && ${VENV_BIN}/mypy sampleflux 2>&1 | tee ${env.WORKSPACE_ROOT}/sampleflux/mypy.txt'", returnStatus: true) if (exitCode != 0) { if (params.REPORT_ALL_WARNINGS) { unstable("Mypy found type errors. See console output above and Mypy report.") @@ -202,8 +202,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('mypy.txt') && readFile('mypy.txt').trim()) { recordIssues( - id: 'mypy-dataflux', - name: 'Mypy (Dataflux)', + id: 'mypy-sampleflux', + name: 'Mypy (Sampleflux)', tools: [myPy(pattern: 'mypy.txt')] ) } @@ -218,7 +218,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { if (fileExists('tests') && sh(script: "find tests -name '*.py' | grep -q .", returnStatus: true) == 0) { - sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=dataflux --cov-report=xml:coverage.xml --cov-report=term" + sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=sampleflux --cov-report=xml:coverage.xml --cov-report=term" } else { echo "No tests found in 'tests' directory. Skipping." } @@ -232,8 +232,8 @@ with open('isort-checkstyle.xml', 'w') as f: } if (fileExists('coverage.xml')) { recordCoverage( - id: 'coverage-dataflux', - name: 'Dataflux Coverage', + id: 'coverage-sampleflux', + name: 'Sampleflux Coverage', tools: [[parser: 'COBERTURA', pattern: 'coverage.xml']] ) } @@ -299,13 +299,13 @@ with open('isort-checkstyle.xml', 'w') as f: post { always { - echo 'Dataflux Pipeline Complete.' + echo 'Sampleflux Pipeline Complete.' } success { - echo 'Dataflux is healthy.' + echo 'Sampleflux is healthy.' } failure { - echo 'Dataflux build failed. Please check linting or test failures.' + echo 'Sampleflux build failed. Please check linting or test failures.' } } } diff --git a/README.md b/README.md index d7995b6..1fb2f4a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# DataFlux +# SampleFlux -**DataFlux** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. +**SampleFlux** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. -Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquify`, and `DataFlux`. +Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquify`, and `SampleFlux`. ## 🚀 Key Features @@ -36,7 +36,7 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquify`, and `DataFlux ```python import numpy as np -from dataflux.core import Flux +from sampleflux.core import Flux # 1. Define a simple transformation def normalize(data: np.ndarray, mean: float = 0.0): @@ -57,10 +57,10 @@ for sample in flux: ## 🏷 Type Specs -`dataflux.typespec` describes *what flows through a `Sample`* and lets ops declare what they accept/produce, so tools like FluxStudio can filter which nodes may connect. It is flexible by design — N-dimensional arrays across numpy/torch/tensorflow, **per-axis bounded ranges**, dtype families, images, and arbitrary Python types — and anything left unspecified defaults to `Any`. +`sampleflux.typespec` describes *what flows through a `Sample`* and lets ops declare what they accept/produce, so tools like FluxStudio can filter which nodes may connect. It is flexible by design — N-dimensional arrays across numpy/torch/tensorflow, **per-axis bounded ranges**, dtype families, images, and arbitrary Python types — and anything left unspecified defaults to `Any`. ```python -from dataflux.typespec import SampleType, ArrayType, Dim, PythonType, UnionType +from sampleflux.typespec import SampleType, ArrayType, Dim, PythonType, UnionType # "a 2-D float array whose first axis is 1–10, second axis any size" ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="floating") @@ -89,7 +89,7 @@ Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime ## 🌀 Fourier Transform (`FourierOp` / `InverseFourierOp` / shift ops) -A small **1-D FFT toolkit**, each op in a numpy variant (`dataflux.ops.numpy`, on `np.ndarray`) and a torch variant (`dataflux.ops.torch`, on `torch.Tensor`); the flat `from dataflux.ops import …` resolves to the torch one (the package's torch-default convention, like `RescaleOp`): +A small **1-D FFT toolkit**, each op in a numpy variant (`sampleflux.ops.numpy`, on `np.ndarray`) and a torch variant (`sampleflux.ops.torch`, on `torch.Tensor`); the flat `from sampleflux.ops import …` resolves to the torch one (the package's torch-default convention, like `RescaleOp`): - **`FourierOp`** — the 1-D discrete Fourier transform (`numpy.fft.fft` / `torch.fft.fft`). - **`InverseFourierOp`** — its inverse (`…fft.ifft`), back to the time domain. @@ -99,8 +99,8 @@ A small **1-D FFT toolkit**, each op in a numpy variant (`dataflux.ops.numpy`, o ```python import numpy as np -from dataflux.sample import Sample -from dataflux.ops.numpy import FourierOp, InverseFourierOp, FftShiftOp +from sampleflux.sample import Sample +from sampleflux.ops.numpy import FourierOp, InverseFourierOp, FftShiftOp x = np.array([1.0, 2.0, 3.0, 4.0]) # real signal spectrum = FourierOp()(Sample(input=x)).input # complex128, == np.fft.fft(x) @@ -120,7 +120,7 @@ Parameters mirror `numpy.fft.fft` / `torch.fft.fft`: `n` (output length — zero ### Windowing & spectral units (`WindowOp` / `SpectrumScalingOp` / `FourierOp(window=…, scaling=…)`) -A raw FFT is **uncalibrated** — to read a spectrum in real units you must taper the signal with a *window* (to control spectral leakage) and divide out the window's gain. DataFlux ships this as two composable ops plus options on `FourierOp` (numpy **and** torch variants). The window + unit math lives in **`dataflux.windows`** (pure numpy; `get_window` / `scale_spectrum` / the `WindowName` + `SpectrumScaling` Literals). +A raw FFT is **uncalibrated** — to read a spectrum in real units you must taper the signal with a *window* (to control spectral leakage) and divide out the window's gain. SampleFlux ships this as two composable ops plus options on `FourierOp` (numpy **and** torch variants). The window + unit math lives in **`sampleflux.windows`** (pure numpy; `get_window` / `scale_spectrum` / the `WindowName` + `SpectrumScaling` Literals). - **`WindowOp(window=…)`** — multiplies the signal by a taper and **stashes the correction** (`window_sum` `S1=Σw`, `window_sum_sq` `S2=Σw²`, `window_enbw_bins`, `window_coherent_gain`) into the metadata for a later scaling step. Windows: `boxcar` (rectangular/none), `bartlett`, `hann`, `hamming`, `blackman`, `blackmanharris`, `nuttall`, `flattop`, `kaiser`, `tukey`, `gaussian` — parametrized ones take `window_param` (Kaiser β / Tukey α / Gaussian σ); `periodic=True` (default) is the DFT-even form correct for FFT analysis. - **`SpectrumScalingOp(scaling=…)`** — turns a spectrum into physical units, reading `S1`/`S2` from the metadata (rectangular `S1=S2=N` if no window was applied): @@ -137,7 +137,7 @@ A raw FFT is **uncalibrated** — to read a spectrum in real units you must tape - **`FourierOp(window=…, scaling=…, sample_rate=…)`** folds all three into one node. The default (`window="boxcar"`, `scaling="none"`) is byte-for-byte the old behaviour. Calibrated `scaling` assumes the unscaled transform, so combining it with a non-`"backward"` `norm` raises. ```python -from dataflux.ops.numpy import FourierOp, WindowOp, SpectrumScalingOp +from sampleflux.ops.numpy import FourierOp, WindowOp, SpectrumScalingOp # one node — Hann-windowed power-spectral density in dBW/Hz-ready units: psd = FourierOp(window="hann", scaling="density", sample_rate=122.88e6)(sample).input @@ -153,12 +153,12 @@ A unit-amplitude tone reads `amplitude` ≈ its amplitude and `power` ≈ amplit ## 🔎 Field Projection & Class Counting Walking a source for a single field (the classic case: counting classes from -*targets*) shouldn't pay to build the fields you don't need. `dataflux.projection` +*targets*) shouldn't pay to build the fields you don't need. `sampleflux.projection` adds an opt-in protocol plus lazy helpers: ```python -from dataflux import project, iter_targets, num_classes -from dataflux import ProjectionField # Literal["input", "target", "metadata"] +from sampleflux import project, iter_targets, num_classes +from sampleflux import ProjectionField # Literal["input", "target", "metadata"] # A source MAY implement SupportsProjection (`project(fields)`) to skip building # unrequested fields — e.g. an image dataset reads only the label column for a @@ -192,7 +192,7 @@ those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: ```python -from dataflux import LabelMap, Flux +from sampleflux import LabelMap, Flux lm = LabelMap.fit(iter_targets(train_source)) # {"bird": 0, "cat": 1, "dog": 2} lm.num_classes # 3 @@ -208,7 +208,7 @@ lm2 = LabelMap.load("class_names.json") `LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the mapping is pinned, so train / eval / predict never disagree. `scikit-learn` backs `fit` (lazy-imported). -## 🖼 Image Conversion (`dataflux.ops.image`) +## 🖼 Image Conversion (`sampleflux.ops.image`) The single, modality-agnostic "any value → image" layer — generic so every project (waivefront spectrograms, any dataset preview, FluxStudio) reuses one @@ -216,7 +216,7 @@ implementation. Domain-specific rendering (overlays, signal plots) stays in the consuming package. ```python -from dataflux.ops.image import ConvertToImageOp, value_to_image +from sampleflux.ops.image import ConvertToImageOp, value_to_image # Op: sample.input (2-D map / CHW tensor / PIL / bool mask) -> PIL image. op = ConvertToImageOp( @@ -232,7 +232,7 @@ rgb = value_to_image(some_value, colormap="magma", max_size=512) # NormalizeToUint8Op: the standalone min-max value -> uint8 quantization step # (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; # set them to pin a fixed scale across samples (out-of-range values clamp). -from dataflux.ops.image import NormalizeToUint8Op +from sampleflux.ops.image import NormalizeToUint8Op sample = NormalizeToUint8Op()(sample) # auto per-array min/max sample = NormalizeToUint8Op(vmin=-80.0, vmax=0.0)(sample) # fixed dB window across a dataset @@ -245,11 +245,11 @@ dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). ## 📦 Storage Integration -DataFlux makes it easy to move data between different formats: +SampleFlux makes it easy to move data between different formats: ```python -from dataflux.storage.hdf5 import HDF5Source -from dataflux.storage.zarr import ZarrGroupSink +from sampleflux.storage.hdf5 import HDF5Source +from sampleflux.storage.zarr import ZarrGroupSink # Stream from HDF5 to Zarr in parallel Flux.from_source(HDF5Source("input.h5")) \ @@ -270,7 +270,7 @@ Every sink has a source that reads its layout back into `Sample` triplets: | Directory (one dir / sample) | `DirectorySink` | — | — | ```python -from dataflux.storage.zarr import ZarrGroupSink, ZarrGroupSource +from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource Flux(samples).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) for sample in ZarrGroupSource("ds.zarr"): # input/target as before, metadata from .zattrs @@ -302,7 +302,7 @@ loaded.metadata["snr"] # scalar, via attributes as before **Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: ```python -from dataflux import DatasetSplit +from sampleflux import DatasetSplit split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% ``` @@ -310,27 +310,27 @@ split.train # ≈80% — the remainder split.val # ≈10% split.te The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: ```yaml -hf_train: !class:dataflux.sources.HuggingFaceSource() +hf_train: !class:sampleflux.sources.HuggingFaceSource() path: mnist split: train -my_split: !class:dataflux.sources.DatasetSplit() +my_split: !class:sampleflux.sources.DatasetSplit() source: !ref:hf_train val_fraction: 0.1 test_fraction: 0.1 seed: 42 -train_set: !class:dataflux.core.Flux() { source: !ref:my_split.train } -val_set: !class:dataflux.core.Flux() { source: !ref:my_split.val } -test_set: !class:dataflux.core.Flux() { source: !ref:my_split.test } +train_set: !class:sampleflux.core.Flux() { source: !ref:my_split.train } +val_set: !class:sampleflux.core.Flux() { source: !ref:my_split.val } +test_set: !class:sampleflux.core.Flux() { source: !ref:my_split.test } ``` Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). -**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `dataflux.SplitName`. +**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `sampleflux.SplitName`. ```yaml -val_set: !class:dataflux.sources.DatasetSplit() +val_set: !class:sampleflux.sources.DatasetSplit() source: !ref:hf_train split: val val_fraction: 0.1 @@ -342,7 +342,7 @@ val_set: !class:dataflux.sources.DatasetSplit() - **`RangeSource(source, start, end)`** — a contiguous index slice `[start:end)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. ```yaml - first_half: !class:dataflux.sources.RangeSource() + first_half: !class:sampleflux.sources.RangeSource() source: !ref:hf_train start: 0 end: 5000 @@ -351,13 +351,13 @@ val_set: !class:dataflux.sources.DatasetSplit() - **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointFlux`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. ```yaml - combined: !class:dataflux.sources.ConcatSource() + combined: !class:sampleflux.sources.ConcatSource() sources: - !ref:train_main - !ref:extra_shard ``` -**HuggingFace native slicing** (alternative, no DataFlux split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. +**HuggingFace native slicing** (alternative, no SampleFlux split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. > **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. @@ -368,8 +368,8 @@ val_set: !class:dataflux.sources.DatasetSplit() A `{ops: [!class:…()]}` document — e.g. one exported from a FluxStudio canvas (`fluxstudio export …`) — can be attached to any source: ```python -from dataflux import Flux -from dataflux.sources import HuggingFaceSource +from sampleflux import Flux +from sampleflux.sources import HuggingFaceSource flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) ``` @@ -388,18 +388,171 @@ Together they express "feed one op's runtime `@output` into a later op's paramet ```yaml ops: # NoiseFloorOp draws an SNR each call; capture it into metadata. - - !class:dataflux.ops.capture.CaptureOutputOp + - !class:sampleflux.ops.capture.CaptureOutputOp op: !class:waivefront.torchsig.processing.NoiseFloorOp {} output: applied_snr_db key: __captured_snr # …then inject the captured value into a later op's `noise_power_db` per sample. - - !class:dataflux.ops.configure.ConfigureOp + - !class:sampleflux.ops.configure.ConfigureOp ops: - - !class:dataflux.ops.stash.UnstashInputOp { key: __captured_snr } + - !class:sampleflux.ops.stash.UnstashInputOp { key: __captured_snr } target: !class:waivefront.torchsig.processing.NoiseFloorOp {} param: noise_power_db ``` +## 🗺 Flow documents & the FlowGraph engine (`sampleflux.flow`) + +The **readable authoring form** of a graph pipeline is a `flow:` document — named steps where a step's name is how later steps reference its result: + +```yaml +flow: + spec: !class:waivefront.SpectrogramOp() # input: the source sample + rescaled: !class:sampleflux.ops.numpy.RescaleOp() # input: previous step + masked: !class:waivefront.SegmentOp() {from: spec} # 2nd reader of spec = fan-out + thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: masked} + denoised: !class:waivefront.torchsig.processing.NoiseFloorOp() + from: rescaled + bind: {low_level: thresh} # per-sample param := thresh's result + out: {from: denoised, target_from: masked} # pure fan-in (no op) +outputs: out +``` + +Step grammar (four reserved keys, stripped before the op is built): **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document order is the schedule and cycles are inexpressible); **`target_from:`/`metadata_from:`** — fan-in slots (a step result contributes its corresponding field; metadata merges last-write-wins); **`bind:`** — `{param: step}` per-sample parameters (a step name = its result's `input`; `step.attr` = the step op's live `@output`, stochastic-correct). A plain-mapping step with no op (`out: {from: a, target_from: b}`) is a pure fan-in; `{}` is the identity (names the source). Cell lifetimes are **automatic** in both forms. + +Two engines, one contract — **bidirectional conversion with execution parity**: + +```python +from sampleflux import Flux, FlowGraph, to_ops, from_ops + +graph = FlowGraph.from_yaml("graph.yaml", source=src) # native named-step engine +flux = Flux.from_flow_yaml("graph.yaml", source=src) # same graph, LOWERED to the + # flat context-ops list (serial) +ops = to_ops(graph.steps, graph.output_step) # flow -> flat ops +flow2 = from_ops(ops) # flat ops -> flow (lifting) +``` + +`FlowGraph` is a `torch.utils.data.Dataset` like `Flux` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Flux's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. See `examples/flow_graph.py` for the full round-trip. + +## 🕸 Graph pipelines on a flat op list (Context ops) + +A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-sample **`Context`** (a named-cell store, `sampleflux.context`) around each sample's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never touches `sample.metadata` — the metadata bus stays byte-identical to a linear run. + +| Op | Semantics | +|---|---| +| `Save(name)` | snapshot the stream sample into a cell (pass-through) — the fork point | +| `Use(name, drop=False)` | stream := the cell's value; deep-copies unless `drop` frees the cell (move) | +| `Drop(names)` | free cells explicitly | +| `Apply(op, param, source, drop=False)` | set `op.` from a cell's value, then apply `op` | +| `Capture(op, output, name)` | apply `op`, record its live `@output` into a cell (stochastic-correct) | +| `Mix(input_from, target_from, metadata_from, drop)` | fan-in: compose a sample from cells + the incoming sample | + +```yaml +ops: + - !class:sampleflux.ops.context.Save(name=fork) # fork the stream + - !class:waivefront.SpectrogramOp() # branch A rides the stream + - !class:sampleflux.ops.context.Save(name=branch_a) + - !class:sampleflux.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork + - !class:waivefront.SegmentOp() + - !class:sampleflux.ops.context.Mix(target_from=branch_a) # fan-in + drop: [branch_a] +``` + +A straight sequence needs none of this — a bare `ops:` list stays exactly as before. Outside an engine (a hand-rolled loop), activate a Context explicitly: + +```python +from sampleflux.context import Context, activate + +with activate(Context()): + for op in ops: + sample = op(sample) +``` + +Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's `input`, `Mix` reads each cell's corresponding field. Copy discipline mirrors the stash family: stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to (see `sampleflux.flow`). + +## 🎭 Multi-type carriers & the collate registry (`sampleflux.kinds` / `sampleflux.collate`) + +Pipelines can carry more than `Sample` triplets: **`Flux(native=True)`** (opt-in) keeps each carrier's own kind — a metadata-free **pair** (`(image, label)`, `(tensor, mask)`, `(tensor, coco_dict)`) or a bare **value** — and adapts every op via its introspected contract: + +```python +from confluid import configurable +from sampleflux import Flux, Sample, op_contract + +@configurable +class NormalizePair: # a pair-native op — no metadata anywhere + def __call__(self, pair: tuple) -> tuple: + img, label = pair + return img / 255.0, label + +@configurable +class StampOp: # a classic Sample op — unchanged + def __call__(self, sample: Sample) -> Sample: ... + +flux = Flux(source=[(img_a, 3), (img_b, 7)], ops=[NormalizePair(), StampOp()], native=True) +# NormalizePair receives the raw pair; StampOp receives a PROMOTED Sample view +# (promotion is one-way and sticky, so op-written metadata is never dropped). + +op_contract(NormalizePair()) # OpContract(accepts='pair', produces='pair', expands=False) +``` + +Detection reads the `__call__` annotations (`Sample` → sample-op, `tuple[...]` → pair-op, untyped → works-on-anything — **untyped ops behave exactly as today**); the class attrs `SAMPLE_KIND_IN` / `SAMPLE_KIND_OUT` / `EXPANDS` override detection where introspection can't see. `native=False` (the default) coerces everything to `Sample` exactly as before — no consumer changes. + +**Collation** is a pluggable registry keyed by representation: + +```python +from sampleflux import collate, get_collate, register_collate + +batch = collate(list(flux)) # dispatches on the detected kind +@register_collate("yolo") # task aliases are additive +def yolo_collate(items): ... +loader = DataLoader(flux, collate_fn=get_collate("yolo")) +``` + +Defaults: `"sample"` (stacked input/target + list-form batched metadata — the `is_batched` convention), `"pair"` (`(stacked_inputs, stacked_targets)`), `"value"`. Consumer collates (classification/segmentation/detection) register additively and keep their own conventions. + +## 🌱 1→N expanding ops (iterable-only pipelines) + +An op may return **several** carriers — a windowing op splitting one capture into N windows is just a generator-returning op: + +```python +from typing import Iterator + +@configurable +class WindowOp: + def __call__(self, sample: Sample) -> Iterator[Sample]: + for w in sliding_windows(sample.input, self.size, self.stride): + yield sample._replace(input=w) +``` + +Expansion is detected from the return annotation (`Iterator[...]` / `Iterable[...]` / `List[...]`; or the explicit `EXPANDS = True` marker) and flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. + +A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(flux)` / `flux[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(flux)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Flux` engine. + +## 📡 SigMF recordings & queryable metadata (`sampleflux.storage.sigmf` / `.query`) + +**SigMF** ([sigmf.org](https://sigmf.org)) is the open Signal Metadata Format — a raw binary sample file (`.sigmf-data`) plus a JSON metadata file (`.sigmf-meta`) with `global`/`captures`/`annotations` sections. `SigMFSink` ↔ `SigMFSource` are the sampleflux carrier pair (siblings of HDF5/Zarr — additive, no migration): + +```python +from sampleflux.storage.sigmf import SigMFSink, SigMFSource + +sink = SigMFSink(path="recordings/", meta_encoder="waivefront.vocab.to_sigmf", checksum=True) +sink.write(sample) # complex64 IQ -> cf32_le + JSON metadata +source = SigMFSource(path="recordings/", meta_decoder="waivefront.vocab.from_sigmf") +``` + +sampleflux stays domain-neutral (unrecognised keys ride the namespaced `sampleflux:` extension); the *waveform vocabulary* — `samplerate` ↔ `core:sample_rate`, `center_freq` ↔ the capture's `core:frequency`, `{role}_regions`/`{role}_labels` ↔ SigMF annotations, the torchsig naming collisions — plugs in from the domain package via the `meta_encoder`/`meta_decoder` hooks. + +**Queryable metadata** — filter stored samples by metadata predicates *without loading arrays*: sources implementing the `SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs/`.zattrs`/meta-JSON — `HDF5Source`, `ZarrGroupSource`, and `SigMFSource` all do, so **existing HDF5/Zarr files are queryable with no rewrite**: + +```python +from sampleflux.storage.query import MetadataFilterSource + +view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="snr_db > 10 and drone == 'DJI'") +len(view) # matches counted from a metadata-only scan +flux = Flux(source=view, ops=[...]) # arrays load ONLY for matching samples +``` + +`where` uses the FormulaOp restricted namespace with metadata keys as variables (a missing key = non-matching, a malformed expression fails loudly); a programmatic `predicate=` composes with AND; sources without the protocol fall back to full-iteration filtering. + ## 🔗 Paired Join (Binary ↔ Annotations) `AnnotationJoinSource` joins a data `DataSource` (e.g. raw binary samples) with a sidecar mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern — typically re-attaching a LabelStudio export back onto the raw samples for training. Three join policies cover the scenarios we actually see in ML research: @@ -418,7 +571,7 @@ data: !class:waivefront.rfuav.data.source.RFUAVSource() labels: !class:annotaide.store.JSONFileAnnotationStore() path: /Volumes/Data/RFUAV-labels -paired: !class:dataflux.paired.AnnotationJoinSource() +paired: !class:sampleflux.paired.AnnotationJoinSource() data: !ref:data annotations: !ref:labels key_fn: "waivefront.rfuav.keys:sample_window_key" @@ -438,34 +591,34 @@ Multi-granularity joins (e.g. pack-level + window-level annotations merged toget ### Callable resolution -`key_fn`, `extract_fn`, and `data_resolver` all accept either a callable **or** a `"module:function"` string path resolved through `dataflux.discovery.resolve_callable`. The string form is what survives YAML round-trip via Confluid. +`key_fn`, `extract_fn`, and `data_resolver` all accept either a callable **or** a `"module:function"` string path resolved through `sampleflux.discovery.resolve_callable`. The string form is what survives YAML round-trip via Confluid. See [`examples/paired_annotations.py`](examples/paired_annotations.py) for a runnable end-to-end walkthrough of all four scenarios. ## 🌐 Ecosystem Integration -DataFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines. +SampleFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines. ### Hugging Face (Community & Standardized Datasets) - **Use Hugging Face for:** Accessing community datasets and leveraging the `datasets` library for efficient Arrow/Parquet loading. -- **Integration:** Use DataFlux to transform `datasets.Dataset` objects into standardized `Sample` triplets, ensuring metadata traceability that often goes missing in simple dictionary-based records. +- **Integration:** Use SampleFlux to transform `datasets.Dataset` objects into standardized `Sample` triplets, ensuring metadata traceability that often goes missing in simple dictionary-based records. - **`metadata_features` (which columns ride along on `Sample.metadata`):** `None` / `[]` keep none (the default); an explicit list keeps exactly those columns; and the sentinel **`"*"`** (or `["*"]`) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load. It stays opt-in so existing configs are unchanged. ```yaml -hf_train: !class:dataflux.sources.HuggingFaceSource() +hf_train: !class:sampleflux.sources.HuggingFaceSource() path: mnist input_feature: image target_feature: label metadata_features: ["*"] # keep every other column as metadata (here: none extra beyond hf_path/hf_split) ``` -### DataFlux (The Functional Engine) -- **Use DataFlux for:** The "inner loop" of your experiment. When you need high-performance multiprocess streaming, per-sample metadata preservation, and 100% reproducible pipelines via **Confluid** serialization. +### SampleFlux (The Functional Engine) +- **Use SampleFlux for:** The "inner loop" of your experiment. When you need high-performance multiprocess streaming, per-sample metadata preservation, and 100% reproducible pipelines via **Confluid** serialization. ## 🔧 Installation ```bash -pip install git+https://github.com/Gearlux/dataflux.git@main +pip install git+https://github.com/Gearlux/sampleflux.git@main ``` ## 📄 License diff --git a/dataflux/__init__.py b/dataflux/__init__.py deleted file mode 100644 index cce1872..0000000 --- a/dataflux/__init__.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -DataFlux: Modular, functional data pipelines. -""" - -from dataflux.core import Flux, JointFlux, WrappedOp -from dataflux.labels import LabelMap -from dataflux.ops import RescaleOp, StandardizeOp, ToTensorOp -from dataflux.paired import AnnotationJoinSource, AnnotationStore -from dataflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project -from dataflux.sample import Sample -from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName -from dataflux.typespec import ( - AnyType, - ArrayType, - Dim, - Dtype, - DtypeFamily, - DtypeSpec, - Framework, - ListType, - MappingType, - PythonType, - SampleType, - UnionType, - infer_sample_type, - infer_type, - typed, -) - -__all__ = [ - "AnnotationJoinSource", - "AnnotationStore", - "AnyType", - "ArrayType", - "ConcatSource", - "DatasetSplit", - "Dim", - "Dtype", - "DtypeFamily", - "DtypeSpec", - "Flux", - "Framework", - "HuggingFaceSource", - "JointFlux", - "LabelMap", - "ListType", - "MappingType", - "ProjectionField", - "PythonType", - "RangeSource", - "RescaleOp", - "Sample", - "SampleType", - "SplitName", - "StandardizeOp", - "SupportsProjection", - "ToTensorOp", - "UnionType", - "WrappedOp", - "infer_sample_type", - "infer_type", - "iter_inputs", - "iter_targets", - "num_classes", - "project", - "typed", -] diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py deleted file mode 100644 index dce64ed..0000000 --- a/dataflux/ops/__init__.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -DataFlux operations. - -Submodules: - - dataflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, - ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp, SqueezeOp, - UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, - WindowOp, SpectrumScalingOp (ndarray) - - dataflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp, SqueezeOp, - UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, - WindowOp, SpectrumScalingOp (tensor) - - dataflux.windows: get_window / scale_spectrum + the WindowName / - SpectrumScaling Literals — the window + unit-scaling math the FFT ops share - - dataflux.ops.tee: Tee (fan-out branching) - - dataflux.ops.parallel: Parallel (worker-pool sub-pipeline) - - dataflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - - dataflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - - dataflux.ops.configure: ConfigureOp (per-sample parameter injection — the helios Configure pattern) - - dataflux.ops.capture: CaptureOutputOp (record an op's @output value into metadata) - - dataflux.ops.formula: FormulaOp (math formula over sample.input — the Math node's op form) - - dataflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - - dataflux.ops.transform_chain: TransformChain (sequential op-chain grouping) - - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - - dataflux.ops.swap: SwapInputTargetOp - - dataflux.ops.stash: StashInputOp, UnstashInputOp, StashTargetOp, UnstashTargetOp - - dataflux.ops.target: MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp (target field) - -Flat imports default to torch variants for the data ops; flow / copy / -swap / stash / target utilities are field-agnostic. -""" - -from dataflux.ops.capture import CaptureOutputOp -from dataflux.ops.configure import ConfigureOp -from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp -from dataflux.ops.enable import Enable -from dataflux.ops.formula import FormulaOp -from dataflux.ops.parallel import Parallel -from dataflux.ops.random_apply import RandomApply -from dataflux.ops.sink import SampleSinkOp -from dataflux.ops.stash import StashInputOp, StashTargetOp, UnstashInputOp, UnstashTargetOp -from dataflux.ops.swap import SwapInputTargetOp -from dataflux.ops.target import ( - CocoToTorchVisionDetectionOp, - DecodeTargetOp, - EncodeTargetOp, - MasksToDetectionBoxesOp, - MetadataToTargetOp, -) -from dataflux.ops.tee import Tee -from dataflux.ops.torch import ( - FftShiftOp, - FourierOp, - IfftShiftOp, - InverseFourierOp, - RescaleOp, - SpectrumScalingOp, - SqueezeOp, - StandardizeOp, - ToTensorOp, - UnsqueezeOp, - WindowOp, -) -from dataflux.ops.transform_chain import TransformChain - -__all__ = [ - "CaptureOutputOp", - "ConfigureOp", - "CopyInputOp", - "CopyMetadataOp", - "CopySampleOp", - "CopyTargetOp", - "DecodeTargetOp", - "Enable", - "FftShiftOp", - "FormulaOp", - "FourierOp", - "IfftShiftOp", - "InverseFourierOp", - "EncodeTargetOp", - "MetadataToTargetOp", - "CocoToTorchVisionDetectionOp", - "MasksToDetectionBoxesOp", - "Parallel", - "RandomApply", - "RescaleOp", - "SampleSinkOp", - "SpectrumScalingOp", - "SqueezeOp", - "StandardizeOp", - "StashInputOp", - "StashTargetOp", - "SwapInputTargetOp", - "Tee", - "TransformChain", - "ToTensorOp", - "UnstashInputOp", - "UnstashTargetOp", - "UnsqueezeOp", - "WindowOp", -] diff --git a/examples/advanced_storage_demo.py b/examples/advanced_storage_demo.py index 845a31f..e0df7fa 100644 --- a/examples/advanced_storage_demo.py +++ b/examples/advanced_storage_demo.py @@ -2,10 +2,10 @@ import numpy as np -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.storage.directory import DirectorySink -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink +from sampleflux.core import Flux +from sampleflux.sample import Sample +from sampleflux.storage.directory import DirectorySink +from sampleflux.storage.zarr import ZarrBatchSink, ZarrGroupSink def main() -> None: diff --git a/examples/basic_pipeline.py b/examples/basic_pipeline.py index 4e1313f..41c3d21 100644 --- a/examples/basic_pipeline.py +++ b/examples/basic_pipeline.py @@ -1,7 +1,7 @@ import confluid # type: ignore[import-not-found] import numpy as np -from dataflux.core import Flux +from sampleflux.core import Flux # 1. Define simple functional transformations @@ -26,7 +26,7 @@ def main() -> None: # 4. Serialize the Pipeline # We set source=None before serialization to only serialize the "recipe" # and avoid serializing raw numpy data which is not YAML-safe. - print("\n--- Serialized DataFlux Pipeline ---") + print("\n--- Serialized SampleFlux Pipeline ---") yaml_state = "" try: pipeline.source = None diff --git a/examples/cache_pipeline.py b/examples/cache_pipeline.py index 884fdcc..5f5c8ed 100644 --- a/examples/cache_pipeline.py +++ b/examples/cache_pipeline.py @@ -6,7 +6,7 @@ 3. The cache enforces an LRU budget — the oldest entry is evicted when a new one would push total usage past ``max_bytes``. -Runs end-to-end with no external data and no DataFlux pipeline. +Runs end-to-end with no external data and no SampleFlux pipeline. """ import tempfile @@ -14,11 +14,11 @@ from pathlib import Path from typing import Callable -from dataflux.storage.cache import CacheBudgetExceeded, DiskCache +from sampleflux.storage.cache import CacheBudgetExceeded, DiskCache def main() -> None: - with tempfile.TemporaryDirectory(prefix="dataflux-cache-demo-") as tmp: + with tempfile.TemporaryDirectory(prefix="sampleflux-cache-demo-") as tmp: cache = DiskCache(Path(tmp), max_bytes=300) print(f"Cache root: {cache.root} (max_bytes={cache.max_bytes})") diff --git a/examples/dataset_split.yaml b/examples/dataset_split.yaml index 1b284e7..13f08c8 100644 --- a/examples/dataset_split.yaml +++ b/examples/dataset_split.yaml @@ -1,10 +1,10 @@ -# DataFlux DatasetSplit / RangeSource / ConcatSource Example +# SampleFlux DatasetSplit / RangeSource / ConcatSource Example # # DatasetSplit is a `category="source"` that partitions an indexable source into # reproducible train/val/test views. `split` is the closed Literal["train","val","test"] -# (dataflux.SplitName). Reload with `confluid.load(...)` and iterate the Fluxes. +# (sampleflux.SplitName). Reload with `confluid.load(...)` and iterate the Fluxes. -hf_train: !class:dataflux.sources.HuggingFaceSource() +hf_train: !class:sampleflux.sources.HuggingFaceSource() path: mnist split: train input_feature: image @@ -14,30 +14,30 @@ hf_train: !class:dataflux.sources.HuggingFaceSource() # Configure the split once (80/10/10 here) and reference its cached views by # attribute. All three `!ref:my_split.` resolve to the SAME materialized # DatasetSplit, so `hf_train` is loaded exactly once and the partition is shared. -my_split: !class:dataflux.sources.DatasetSplit() +my_split: !class:sampleflux.sources.DatasetSplit() source: !ref:hf_train val_fraction: 0.1 test_fraction: 0.1 seed: 42 -train_set: !class:dataflux.core.Flux() +train_set: !class:sampleflux.core.Flux() source: !ref:my_split.train -val_set: !class:dataflux.core.Flux() +val_set: !class:sampleflux.core.Flux() source: !ref:my_split.val -test_set: !class:dataflux.core.Flux() +test_set: !class:sampleflux.core.Flux() source: !ref:my_split.test # === RangeSource: a contiguous [start:end) slice over any indexable source === -first_1000: !class:dataflux.core.Flux() - source: !class:dataflux.sources.RangeSource() +first_1000: !class:sampleflux.core.Flux() + source: !class:sampleflux.sources.RangeSource() source: !ref:hf_train start: 0 end: 1000 # === ConcatSource: join several indexable sources into one (then optionally split) === -hf_test: !class:dataflux.sources.HuggingFaceSource() +hf_test: !class:sampleflux.sources.HuggingFaceSource() path: mnist split: test input_feature: image @@ -45,10 +45,10 @@ hf_test: !class:dataflux.sources.HuggingFaceSource() # Concatenate train + test into one source; ConcatSource is indexable, so it can # itself feed a DatasetSplit (e.g. to re-partition the combined pool). -combined: !class:dataflux.sources.ConcatSource() +combined: !class:sampleflux.sources.ConcatSource() sources: - !ref:hf_train - !ref:hf_test -combined_pool: !class:dataflux.core.Flux() +combined_pool: !class:sampleflux.core.Flux() source: !ref:combined diff --git a/examples/discovery_demo.py b/examples/discovery_demo.py index 42e4169..7541e46 100644 --- a/examples/discovery_demo.py +++ b/examples/discovery_demo.py @@ -1,7 +1,7 @@ import json from pathlib import Path -from dataflux.discovery import scan_module +from sampleflux.discovery import scan_module def main() -> None: diff --git a/examples/flow_graph.py b/examples/flow_graph.py new file mode 100644 index 0000000..3fdeb0b --- /dev/null +++ b/examples/flow_graph.py @@ -0,0 +1,84 @@ +"""Flow-document graphs: author a named-step graph, run it on BOTH engines, convert both ways. + +Demonstrates the graph execution model: +- a ``flow`` mapping (step-name -> op) with fan-out (two readers of one step), fan-in + (``target_from``), and a per-sample parameter bind; +- native execution on :class:`sampleflux.FlowGraph`; +- lowering to a flat context-ops list (:func:`sampleflux.to_ops`) executed by the plain + serial :class:`sampleflux.Flux` engine — with identical results; +- lifting a flat list back into a flow (:func:`sampleflux.from_ops`). + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +from confluid import configurable + +from sampleflux import FlowGraph, Flux, Sample, from_ops, to_ops +from sampleflux.ops.swap import SwapInputTargetOp + + +@configurable +class AddOp: + """Add a constant to the input. + + Args: + amount: Value added to ``sample.input``. + """ + + def __init__(self, amount: float = 1.0) -> None: + self.amount = amount + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + self.amount) + + +@configurable +class ScaleOp: + """Multiply the input by a factor. + + Args: + factor: Multiplier applied to ``sample.input``. + """ + + def __init__(self, factor: float = 2.0) -> None: + self.factor = factor + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input * self.factor) + + +def build_flow() -> dict: + """The graph: fork `a`; branch A swaps v+1 into target; branch B computes (v+1)*2 + bind.""" + return { + "a": AddOp(amount=1.0), # input: the source sample + "branch_a": {"op": SwapInputTargetOp(), "from": "a"}, # A: park v+1 in target + "branch_b": {"op": ScaleOp(factor=2.0), "from": "a"}, # B: (v+1)*2 (2nd read of a = fan-out) + "shifted": {"op": AddOp(), "from": "branch_b", "bind": {"amount": "a"}}, # per-sample param + "out": {"from": "shifted", "target_from": "branch_a"}, # fan-in + } + + +def main() -> None: + source = [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(4)] + + # 1. Native FlowGraph execution + native = [(s.input, s.target) for s in FlowGraph(source=source, flow=build_flow())] + print(f"FlowGraph (native): {native}") + + # 2. Lower to the flat context-ops list -> serial Flux engine + ops = to_ops(build_flow()) + print(f"Lowered ops: {[type(o).__name__ for o in ops]}") + serial = [(s.input, s.target) for s in Flux(source=source, ops=ops)] + print(f"Flux (lowered): {serial}") + assert native == serial, "engine parity is a hard contract" + + # 3. Lift the flat list back into a flow document + lifted, outputs = from_ops(to_ops(build_flow())) + relifted = [(s.input, s.target) for s in FlowGraph(source=source, flow=lifted, outputs=outputs)] + assert relifted == native + print(f"Lifted flow steps: {list(lifted)} (outputs={outputs!r})") + print("flow -> ops -> flow round-trip: parity holds ✓") + + +if __name__ == "__main__": + main() diff --git a/examples/hdf5_pipeline.py b/examples/hdf5_pipeline.py index dcd9099..7a90b3a 100644 --- a/examples/hdf5_pipeline.py +++ b/examples/hdf5_pipeline.py @@ -3,8 +3,8 @@ import confluid # type: ignore[import-not-found] import numpy as np -from dataflux.core import Flux -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.core import Flux +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source # 1. Define a simple transform @@ -29,7 +29,7 @@ def main() -> None: # 4. Serialize the Pipeline pipeline = Flux().map(rescale, scale=100.0) - print("\n--- Serialized DataFlux Pipeline ---") + print("\n--- Serialized SampleFlux Pipeline ---") yaml_state = confluid.dump(pipeline) print(yaml_state) diff --git a/examples/paired_annotations.py b/examples/paired_annotations.py index f7500d5..7bcddd2 100644 --- a/examples/paired_annotations.py +++ b/examples/paired_annotations.py @@ -9,8 +9,8 @@ import confluid # type: ignore[import-not-found] -from dataflux.paired import AnnotationJoinSource -from dataflux.sample import Sample +from sampleflux.paired import AnnotationJoinSource +from sampleflux.sample import Sample @confluid.configurable @@ -60,7 +60,7 @@ def keys(self) -> Any: # Module-level callables so they survive Confluid YAML round-trip via -# dataflux.discovery.resolve_callable("examples.paired_annotations:"). +# sampleflux.discovery.resolve_callable("examples.paired_annotations:"). def window_key(sample: Sample) -> str: return f"{sample.meta['pack_id']}:win{sample.meta['window_start_sample']:08d}" diff --git a/examples/parallel_hdf5_stream.py b/examples/parallel_hdf5_stream.py index 1b1fe37..999bce0 100644 --- a/examples/parallel_hdf5_stream.py +++ b/examples/parallel_hdf5_stream.py @@ -3,8 +3,8 @@ import numpy as np -from dataflux.core import Flux -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.core import Flux +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source def heavy_rescale(data: np.ndarray, factor: float = 1.0) -> np.ndarray: diff --git a/examples/parallel_pipeline.py b/examples/parallel_pipeline.py index 1c68165..546f4ae 100644 --- a/examples/parallel_pipeline.py +++ b/examples/parallel_pipeline.py @@ -2,7 +2,7 @@ import numpy as np -from dataflux.core import Flux +from sampleflux.core import Flux def heavy_computation(data: np.ndarray, intensity: int = 10) -> np.ndarray: diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py index 3be2b92..fc9e1aa 100644 --- a/examples/storage_roundtrip.py +++ b/examples/storage_roundtrip.py @@ -16,14 +16,14 @@ import numpy as np -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource +from sampleflux.core import Flux +from sampleflux.sample import Sample +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource def main() -> None: - with tempfile.TemporaryDirectory(prefix="dataflux-storage-demo-") as tmp: + with tempfile.TemporaryDirectory(prefix="sampleflux-storage-demo-") as tmp: root = Path(tmp) # 1. HDF5 with an array in metadata (the segmentation-mask case). diff --git a/pyproject.toml b/pyproject.toml index aa491b1..957e762 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "data-flux" +name = "sampleflux" version = "0.1.0" description = "Clean, functional data pipelines for ML research and production." authors = [{ name = "Taidal", email = "info@gearlux.ai" }] @@ -45,54 +45,61 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" # CONFLUID-CONFIGURABLE DISCOVERY -# Sources / ops live in separate submodules that `dataflux/__init__.py` +# Sources / ops live in separate submodules that `sampleflux/__init__.py` # does not eagerly import. Listing them here makes navigaitor discover # every ``@configurable`` on bootstrap without hardcoding the list. [project.entry-points."confluid.configurables"] -dataflux = "dataflux" -dataflux-core = "dataflux.core" -dataflux-sources = "dataflux.sources" -dataflux-ops-parallel = "dataflux.ops.parallel" -dataflux-ops-tee = "dataflux.ops.tee" -dataflux-ops-enable = "dataflux.ops.enable" -dataflux-ops-random-apply = "dataflux.ops.random_apply" +sampleflux = "sampleflux" +sampleflux-core = "sampleflux.core" +sampleflux-sources = "sampleflux.sources" +sampleflux-ops-parallel = "sampleflux.ops.parallel" +sampleflux-ops-tee = "sampleflux.ops.tee" +sampleflux-ops-enable = "sampleflux.ops.enable" +sampleflux-ops-random-apply = "sampleflux.ops.random_apply" # ConfigureOp (per-sample parameter injection — the helios Configure pattern); entry-point # changes need an editable reinstall before FluxStudio/navigaitor discovery sees the module. -dataflux-ops-configure = "dataflux.ops.configure" -dataflux-ops-formula = "dataflux.ops.formula" +sampleflux-ops-configure = "sampleflux.ops.configure" +sampleflux-ops-formula = "sampleflux.ops.formula" # CaptureOutputOp (records an op's @output value into metadata — the capture half of # FluxStudio's op-@output -> param wiring, paired with ConfigureOp(UnstashInputOp)). -dataflux-ops-capture = "dataflux.ops.capture" -dataflux-ops-transform-chain = "dataflux.ops.transform_chain" -dataflux-ops-sink = "dataflux.ops.sink" -dataflux-ops-stash = "dataflux.ops.stash" -dataflux-ops-numpy = "dataflux.ops.numpy" -dataflux-ops-torch = "dataflux.ops.torch" -dataflux-ops-copy = "dataflux.ops.copy" -dataflux-ops-swap = "dataflux.ops.swap" -dataflux-ops-target = "dataflux.ops.target" -dataflux-ops-image = "dataflux.ops.image" +sampleflux-ops-capture = "sampleflux.ops.capture" +sampleflux-ops-transform-chain = "sampleflux.ops.transform_chain" +# Context ops (Save/Use/Drop/Apply/Capture/Mix) — the graph-plane building blocks lowered from flow: docs +sampleflux-ops-context = "sampleflux.ops.context" +# The FlowGraph engine (flow: named-step documents + the flow<->ops converters) +sampleflux-flow = "sampleflux.flow" +# SigMF recordings (SigMFSink <-> SigMFSource) + the queryable-metadata view source +sampleflux-storage-sigmf = "sampleflux.storage.sigmf" +sampleflux-storage-query = "sampleflux.storage.query" +sampleflux-ops-sink = "sampleflux.ops.sink" +sampleflux-ops-stash = "sampleflux.ops.stash" +sampleflux-ops-numpy = "sampleflux.ops.numpy" +sampleflux-ops-torch = "sampleflux.ops.torch" +sampleflux-ops-copy = "sampleflux.ops.copy" +sampleflux-ops-swap = "sampleflux.ops.swap" +sampleflux-ops-target = "sampleflux.ops.target" +sampleflux-ops-image = "sampleflux.ops.image" # DropMetadataOp (strip metadata keys — e.g. the __taidal_stash_* snapshots) + PrintSampleOp # (log/print a per-sample summary). Entry-point changes need an editable reinstall before # FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). -dataflux-ops-metadata = "dataflux.ops.metadata" -dataflux-ops-debug = "dataflux.ops.debug" +sampleflux-ops-metadata = "sampleflux.ops.metadata" +sampleflux-ops-debug = "sampleflux.ops.debug" # Storage SINKS (HDF5Sink / ZarrGroupSink / ZarrBatchSink / DirectorySink) carry # category="sink" so FluxStudio surfaces them as DatasetProcessor sink nodes. They live under -# dataflux.storage.* (NOT re-exported from the package root), and scan_module does not recurse +# sampleflux.storage.* (NOT re-exported from the package root), and scan_module does not recurse # submodules, so each storage module needs its own entry point. The matching SOURCES in these # modules stay uncategorised, so the positive {op,source,engine,sink} allowlist surfaces only the # tagged sinks. (Entry-point changes need an editable reinstall — `aisland setup`, never --reinstall.) -dataflux-storage-hdf5 = "dataflux.storage.hdf5" -dataflux-storage-zarr = "dataflux.storage.zarr" -dataflux-storage-directory = "dataflux.storage.directory" +sampleflux-storage-hdf5 = "sampleflux.storage.hdf5" +sampleflux-storage-zarr = "sampleflux.storage.zarr" +sampleflux-storage-directory = "sampleflux.storage.directory" [tool.setuptools.packages.find] where = ["."] -include = ["dataflux*"] +include = ["sampleflux*"] [tool.setuptools.package-data] -dataflux = ["py.typed"] +sampleflux = ["py.typed"] [tool.black] line-length = 120 diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py new file mode 100644 index 0000000..6df4566 --- /dev/null +++ b/sampleflux/__init__.py @@ -0,0 +1,82 @@ +""" +SampleFlux: Modular, functional data pipelines. +""" + +from sampleflux.collate import collate, get_collate, register_collate +from sampleflux.context import Context +from sampleflux.core import Flux, JointFlux, WrappedOp +from sampleflux.flow import FlowGraph, from_ops, to_ops +from sampleflux.kinds import OpContract, SampleKind, classify_carrier, op_contract +from sampleflux.labels import LabelMap +from sampleflux.ops import RescaleOp, StandardizeOp, ToTensorOp +from sampleflux.paired import AnnotationJoinSource, AnnotationStore +from sampleflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project +from sampleflux.sample import Sample +from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName +from sampleflux.typespec import ( + AnyType, + ArrayType, + Dim, + Dtype, + DtypeFamily, + DtypeSpec, + Framework, + ListType, + MappingType, + PythonType, + SampleType, + UnionType, + infer_sample_type, + infer_type, + typed, +) + +__all__ = [ + "AnnotationJoinSource", + "AnnotationStore", + "AnyType", + "ArrayType", + "ConcatSource", + "Context", + "DatasetSplit", + "Dim", + "FlowGraph", + "OpContract", + "SampleKind", + "classify_carrier", + "collate", + "from_ops", + "get_collate", + "op_contract", + "register_collate", + "to_ops", + "Dtype", + "DtypeFamily", + "DtypeSpec", + "Flux", + "Framework", + "HuggingFaceSource", + "JointFlux", + "LabelMap", + "ListType", + "MappingType", + "ProjectionField", + "PythonType", + "RangeSource", + "RescaleOp", + "Sample", + "SampleType", + "SplitName", + "StandardizeOp", + "SupportsProjection", + "ToTensorOp", + "UnionType", + "WrappedOp", + "infer_sample_type", + "infer_type", + "iter_inputs", + "iter_targets", + "num_classes", + "project", + "typed", +] diff --git a/sampleflux/collate.py b/sampleflux/collate.py new file mode 100644 index 0000000..3f393cf --- /dev/null +++ b/sampleflux/collate.py @@ -0,0 +1,127 @@ +"""The pluggable collate registry — batch builders keyed by representation. + +Batching in sampleflux is two-stage: the engine groups carriers (``Flux.batch`` / +``FlowGraph.batch`` yield ``list``\\ s of N items) and a COLLATE function stacks a group +into one batched carrier. Historically each consumer package shipped its own collate +(classification / segmentation / detection, with two divergent metadata conventions); +this registry gives them ONE addressable home without changing any of them — consumers +``register_collate`` their task collates additively, and callers dispatch by key or by +the DETECTED carrier kind (:func:`sampleflux.kinds.classify_carrier`). + +Defaults registered here: + +- ``"sample"`` — stacks ``input``/``target`` (torch-first, numpy fallback, else kept as a + list) and gathers per-item metadata into the LIST form (``Sample.is_batched`` True — + the marainer/sonair convention). +- ``"pair"`` — a metadata-free 2-tuple batch: ``(stacked inputs, stacked targets)``. +- ``"value"`` — bare values stacked directly. + +Consumer conventions are deliberately NOT unified here (deltaid/raidar's +``{"per_sample": [...]}`` nesting stays theirs — see TASKS.md); the registry is additive. +""" + +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from loggair import get_logger + +from sampleflux.kinds import classify_carrier +from sampleflux.sample import Sample + +logger = get_logger(__name__) + +CollateFn = Callable[[Sequence[Any]], Any] + +_REGISTRY: Dict[str, CollateFn] = {} + +__all__ = ["CollateFn", "collate", "get_collate", "register_collate", "registered_collates"] + + +def register_collate(key: str) -> Callable[[CollateFn], CollateFn]: + """Register a collate function under ``key`` (a kind name or a task alias). + + Usable as a decorator:: + + @register_collate("yolo") + def yolo_collate(items): ... + + Re-registering a key overwrites it (logged at debug — consumers may deliberately + replace a default). + """ + + def _register(fn: CollateFn) -> CollateFn: + if key in _REGISTRY: + logger.debug(f"collate registry: overwriting existing collate for key {key!r}") + _REGISTRY[key] = fn + return fn + + return _register + + +def get_collate(key: str) -> CollateFn: + """The collate registered under ``key``; a miss names the known keys.""" + try: + return _REGISTRY[key] + except KeyError: + known = ", ".join(sorted(_REGISTRY)) or "" + raise KeyError(f"no collate registered for {key!r} (known: {known})") from None + + +def registered_collates() -> Tuple[str, ...]: + """The registered keys (sorted).""" + return tuple(sorted(_REGISTRY)) + + +def collate(items: Sequence[Any], key: Optional[str] = None) -> Any: + """Collate ``items`` into one batched carrier. + + ``key`` picks a registered collate explicitly; omitted, the DETECTED kind of the + first item dispatches (``sample`` / ``pair`` / ``value``). An empty batch raises. + """ + if not items: + raise ValueError("collate: cannot collate an empty batch") + return get_collate(key or classify_carrier(items[0]))(items) + + +def _stack(values: List[Any]) -> Any: + """Best-effort stacking: torch tensors -> stacked tensor, numpy -> stacked array, else a list.""" + first = values[0] + try: + import torch + + if isinstance(first, torch.Tensor): + return torch.stack(list(values)) + except ImportError: # pragma: no cover - torch is a hard dep today, defensive only + pass + try: + import numpy as np + + if isinstance(first, np.ndarray): + return np.stack(list(values)) + if isinstance(first, (int, float)) and all(isinstance(v, (int, float)) for v in values): + return np.asarray(values) + except ImportError: # pragma: no cover - numpy is a hard dep, defensive only + pass + return list(values) + + +@register_collate("sample") +def sample_collate(items: Sequence[Any]) -> Sample: + """Default Sample collate: stacked input/target + LIST-form metadata (``is_batched`` True).""" + samples = [item if isinstance(item, Sample) else Sample.from_any(item) for item in items] + return Sample( + input=_stack([s.input for s in samples]), + target=_stack([s.target for s in samples]), + metadata=[dict(s.meta) for s in samples], + ) + + +@register_collate("pair") +def pair_collate(items: Sequence[Any]) -> Tuple[Any, Any]: + """Default pair collate: ``(stacked inputs, stacked targets)`` — no metadata anywhere.""" + return _stack([item[0] for item in items]), _stack([item[1] for item in items]) + + +@register_collate("value") +def value_collate(items: Sequence[Any]) -> Any: + """Default value collate: the bare values stacked.""" + return _stack(list(items)) diff --git a/sampleflux/context.py b/sampleflux/context.py new file mode 100644 index 0000000..ee77def --- /dev/null +++ b/sampleflux/context.py @@ -0,0 +1,120 @@ +"""Per-sample named-cell store — the graph data plane for graph-shaped pipelines. + +A :class:`Context` holds named **cells** for exactly one sample's trip through the op +list: branch snapshots (a cell holding a :class:`~sampleflux.sample.Sample`), captured +``@output`` values, and per-sample parameters. The context ops in +:mod:`sampleflux.ops.context` (``Save`` / ``Use`` / ``Drop`` / ``Apply`` / ``Capture`` / +``Mix``) move data between the linear sample stream and these cells, which is what lets +a plain sequential op list execute a fan-out/fan-in graph. + +Deliberately NOT ``sample.metadata``: metadata rides *inside* each sample and is the +shared accumulating bus ops hand values to each other on. The Context is the *wiring* +plane — engine-created, per sample, empty again by the end of a well-formed graph (every +cell freed after its last read). Nothing here is ``@configurable``; a Context never +appears in YAML. + +The engine (``Flux`` — and ``FlowGraph``, which manages its env directly) creates one +Context per source item and activates it around the op loop via a +:class:`contextvars.ContextVar`, so ops reach it inside ``__call__`` with no signature +change (:func:`current` / :func:`require`). A hand-rolled loop outside an engine opts in +explicitly:: + + with activate(Context()): + for op in ops: + sample = op(sample) +""" + +import contextvars +from contextlib import contextmanager +from typing import Any, Dict, Iterator, Optional, Tuple + +__all__ = ["Context", "activate", "current", "require"] + + +class Context: + """Named-cell store for one sample's trip through a graph-shaped pipeline. + + Cells are stored and returned **by reference** — copy semantics are the reading + op's decision (``Use`` deep-copies unless it drops the cell), mirroring the stash + family's copy-on-restore convention. + """ + + __slots__ = ("_cells",) + + def __init__(self) -> None: + self._cells: Dict[str, Any] = {} + + def put(self, name: str, value: Any) -> None: + """Store ``value`` under ``name`` (overwrites an existing cell).""" + self._cells[name] = value + + def get(self, name: str) -> Any: + """Return the cell's value by reference; a missing cell is an actionable error.""" + if name not in self._cells: + live = ", ".join(sorted(self._cells)) or "" + raise KeyError( + f"Context has no cell {name!r} (live cells: {live}). " + f"A cell must be written (Save / Capture) before it is read, and is gone after " + f"a drop — check the op order and drop flags." + ) + return self._cells[name] + + def delete(self, name: str) -> None: + """Free the cell; deleting a missing cell is an error (it flags a liveness bug).""" + if name not in self._cells: + live = ", ".join(sorted(self._cells)) or "" + raise KeyError(f"Context cannot drop missing cell {name!r} (live cells: {live}).") + del self._cells[name] + + def live(self) -> Tuple[str, ...]: + """Names of all currently-held cells (sorted, for stable error messages/tests).""" + return tuple(sorted(self._cells)) + + def copy(self) -> "Context": + """Shallow copy — same cell values, independent cell *set* (for 1→N expansion children).""" + clone = Context() + clone._cells = dict(self._cells) + return clone + + def clear(self) -> None: + """Drop every cell.""" + self._cells.clear() + + def __contains__(self, name: object) -> bool: + return name in self._cells + + def __len__(self) -> int: + return len(self._cells) + + def __repr__(self) -> str: # pragma: no cover - debug aid + return f"Context(cells={sorted(self._cells)})" + + +_CURRENT: contextvars.ContextVar[Optional[Context]] = contextvars.ContextVar("sampleflux_context", default=None) + + +def current() -> Optional[Context]: + """The active per-sample :class:`Context`, or ``None`` outside an engine/`activate` block.""" + return _CURRENT.get() + + +def require(op_name: str = "context op") -> Context: + """The active Context, or an actionable error naming the op that needed it.""" + ctx = _CURRENT.get() + if ctx is None: + raise RuntimeError( + f"{op_name}: no active Context. Context ops need the per-sample Context the engine " + f"creates — run the pipeline through Flux/FlowGraph, or wrap a manual loop in " + f"`with sampleflux.context.activate(Context()):`." + ) + return ctx + + +@contextmanager +def activate(ctx: Context) -> Iterator[Context]: + """Activate ``ctx`` as the current per-sample Context for the enclosed block.""" + token = _CURRENT.set(ctx) + try: + yield ctx + finally: + _CURRENT.reset(token) diff --git a/dataflux/core.py b/sampleflux/core.py similarity index 61% rename from dataflux/core.py rename to sampleflux/core.py index 567c1b7..9be7279 100644 --- a/dataflux/core.py +++ b/sampleflux/core.py @@ -12,6 +12,7 @@ Iterable, Iterator, List, + NamedTuple, Optional, Tuple, Union, @@ -25,11 +26,12 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from dataflux.projection import ProjectionField -from dataflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, Sample +from sampleflux.context import Context, activate +from sampleflux.projection import ProjectionField +from sampleflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, Sample if TYPE_CHECKING: # pragma: no cover - typing only - from dataflux.typespec import SampleType + from sampleflux.typespec import SampleType logger = get_logger(__name__) @@ -138,7 +140,7 @@ class WrappedOp: """ def __init__(self, f: Union[str, Callable] = "", s: str = "input", kw: Optional[Dict[str, Any]] = None): - from dataflux.discovery import get_callable_path + from sampleflux.discovery import get_callable_path # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` # property). EXPLICIT: always store the string path for serialization. @@ -151,7 +153,7 @@ def __init__(self, f: Union[str, Callable] = "", s: str = "input", kw: Optional[ @property def func(self) -> Callable: if self._func_cache is None: - from dataflux.discovery import resolve_callable + from sampleflux.discovery import resolve_callable self._func_cache = resolve_callable(self.f) return self._func_cache @@ -171,14 +173,128 @@ def __call__(self, sample: Sample) -> Optional[Sample]: raise e -def _worker_task(sample: Sample, ops: List[Any]) -> Optional[Sample]: - """Top-level helper for multiprocess workers. Must be at top level for pickling.""" - current_sample: Optional[Sample] = sample - for op in ops: - if current_sample is None: - return None - current_sample = _apply_op(current_sample, op) - return current_sample +class _Carried(NamedTuple): + """A carrier travelling the streamed route together with its per-sample Context. + + ``sample`` is a :class:`Sample` on the default route; under ``Flux(native=True)`` it + may be any native carrier (a metadata-free pair, a bare value). + """ + + sample: Any + ctx: Context + + +def _apply_op_native(carrier: Any, op: Any) -> Any: + """Apply one op to a NATIVE carrier (Sample / metadata-free pair / bare value). + + The op's introspected contract (:func:`sampleflux.kinds.op_contract`) picks the + adaptation: + + - a **pair-op** on a Sample carrier receives ``(input, target)`` and its returned + pair merges back via ``_replace`` (metadata preserved); + - a **sample-op** on a pair/value carrier receives a PROMOTED Sample view + (``Sample.from_any`` — promotion is one-way and sticky, so op-written metadata is + never dropped); + - an **any-op** receives the carrier verbatim (untyped ops behave exactly as today). + """ + from sampleflux.kinds import op_contract + + contract = op_contract(op) + if isinstance(carrier, Sample): + if contract.accepts == "pair": + result = op(carrier.to_pair()) + if result is None: + return None + if isinstance(result, tuple) and len(result) == 2: + return carrier._replace(input=result[0], target=result[1]) + return result + return _apply_op(carrier, op) + if contract.accepts == "sample": + return _apply_op(Sample.from_any(carrier), op) # promotion is sticky + result = op(carrier) + return result + + +def _expand(op: Any, carrier: Any) -> List[Any]: + """Run a 1→N EXPANDING op and return its flattened, type-refreshed children.""" + raw = op(carrier) + if raw is None: + return [] + children: List[Any] = [] + for child in raw: + if child is None: + continue + children.append(_refresh_type(child, op) if isinstance(child, Sample) else child) + return children + + +def _worker_task(sample: Any, ops: List[Any], native: bool = False) -> Optional[Any]: + """Single-result worker for STRICTLY 1→1 op lists (the ``Parallel`` op's contract). + + Kept for callers that need exactly one carrier back; expanding ops raise here — + route expanding pipelines through :func:`_worker_task_multi`. + """ + results = _worker_task_multi(sample, ops, native=native, allow_expansion=False) + return results[0] if results else None + + +def _worker_task_multi(sample: Any, ops: List[Any], native: bool = False, allow_expansion: bool = True) -> List[Any]: + """Top-level helper for multiprocess workers. Must be at top level for pickling. + + Runs one source carrier through the op list and returns EVERY resulting carrier — + usually one, zero when filtered, several when a 1→N EXPANDING op fired (detected via + :func:`sampleflux.kinds.op_contract`; each expansion child continues through the + REMAINING ops with a shallow copy of the per-sample Context, depth-first so sibling + order matches the nested-loop intuition). + + Activates ONE fresh per-carrier :class:`~sampleflux.context.Context` around the op + loop so context ops (``Save``/``Use``/``Apply``/``Capture``/``Mix``) can move data + between the linear stream and named cells — the executor itself stays a plain + ``for op in ops`` loop. Contexts are created inside the worker (spawn-safe: ops + pickle, a Context never crosses a process boundary). + + ``native=True`` keeps the carrier's own kind (Sample / pair / value) and adapts it + per op via :func:`_apply_op_native` instead of coercing everything to Sample. + """ + from collections import deque + + from sampleflux.kinds import op_contract + + pending: "deque[Tuple[Any, Context, int]]" = deque([(sample, Context(), 0)]) + out: List[Any] = [] + while pending: + current, ctx, start = pending.popleft() + alive = True + with activate(ctx): + i = start + while i < len(ops): + op = ops[i] + i += 1 + if op_contract(op).expands: + if not allow_expansion: + raise TypeError( + f"op {type(op).__name__!r} is a 1→N expanding op, which this strictly " + "1→1 route cannot carry — run it through the Flux iteration paths." + ) + children = _expand(op, current) + if not children: + alive = False + break + # Depth-first: the first child continues inline; its siblings go to the + # FRONT of the queue (reversed, so sibling order is preserved) — output + # order matches the nested-loop intuition even for chained expansions. + for child in reversed(children[1:]): + pending.appendleft((child, ctx.copy(), i)) + current = children[0] + continue + result = _apply_op_native(current, op) if native else _apply_op(current, op) + if result is None: + alive = False + break + current = result + if alive and current is not None: + out.append(current) + return out @configurable(category="engine") @@ -209,11 +325,11 @@ def __len__(self) -> int: @configurable(category="engine") class Flux(torch.utils.data.Dataset[Sample]): """ - The primary stream engine for DataFlux. + The primary stream engine for SampleFlux. Wraps any iterable or indexed dataset and provides a functional API. Annotation design (kept intentionally ``Any``): - Per the DataFlux mandate "Functional Purity: Transforms are plain + Per the SampleFlux mandate "Functional Purity: Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations.", ``source`` is duck-typed (any iterable; the Indexable protocol if ``__getitem__``/``__len__`` are present) and @@ -230,6 +346,9 @@ class Flux(torch.utils.data.Dataset[Sample]): source: Any iterable or indexable dataset (duck-typed) to wrap; ``None`` yields an empty stream. ops: Ordered callables ``Sample -> Optional[Sample]`` applied lazily on access (``None`` = no ops). chunk_size: Parallel-processing chunk size; ``0`` (the default) processes sequentially. + native: Opt-in multi-type mode — carriers keep their own kind (Sample / metadata-free pair / + bare value) and each op is adapted per its introspected contract (``sampleflux.kinds``). + ``False`` (the default) coerces every item to ``Sample`` exactly as before. """ def __init__( @@ -237,9 +356,11 @@ def __init__( source: Optional[Iterable[Any]] = None, ops: Optional[List[Any]] = None, chunk_size: Optional[int] = 0, + native: bool = False, ) -> None: self.source = source self.ops: List[Any] = ops or [] + self.native = bool(native) self._workers = 1 self._chunk_size = chunk_size or 0 # Populated on first random access when the source is iterable-only @@ -290,16 +411,51 @@ def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Fl ops = list(_confluid_materialize(raw_ops)) return cls(source=source, ops=ops) + @classmethod + def from_flow_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": + """Attach a ``{flow: {...}}`` graph document to ``source``, LOWERED to the serial form. + + The named-step flow document (see :mod:`sampleflux.flow`) is compiled into a flat + context-ops list via :func:`sampleflux.flow.to_ops`, so the graph executes on this + plain serial engine. ``FlowGraph.from_yaml`` is the native-engine twin. + """ + from sampleflux.flow import flow_yaml_to_flux + + return cast("Flux", flow_yaml_to_flux(path, source=source)) + + @property + def _expands(self) -> bool: + """True when any (materialized) op is a 1→N expanding op — the pipeline is then iterable-only.""" + from sampleflux.kinds import op_contract + + return any(not isinstance(op, _ConfluidFluid) and op_contract(op).expands for op in self.ops) + + def _guard_not_expanding(self, operation: str) -> None: + if self._expands: + from sampleflux.kinds import op_contract + + culprit = next( + type(op).__name__ for op in self.ops if not isinstance(op, _ConfluidFluid) and op_contract(op).expands + ) + raise TypeError( + f"Flux.{operation}: the pipeline contains the 1→N expanding op {culprit!r}, so the " + "expanded length/index mapping is unknowable up front — the pipeline is ITERABLE-ONLY. " + "Iterate it (or wrap in a torch IterableDataset); for random access, window/expand at " + "the source instead, or materialize with list(flux) first." + ) + def __len__(self) -> int: """Return the length of the underlying source if available. Surfaces a clear error when the source is still a deferred Confluid marker so downstream callers (e.g. torch's DataLoader) don't end up - reporting the opaque ``num_samples=0``. + reporting the opaque ``num_samples=0``, and when the pipeline contains + a 1→N expanding op (iterable-only — the true length is unknowable). """ from collections.abc import Sized source = self._guard_live_source() + self._guard_not_expanding("__len__") if isinstance(source, Sized): return len(source) return 0 @@ -324,6 +480,7 @@ def __getitem__(self, index: int) -> Sample: source = self._guard_live_source() if source is None: raise TypeError("Flux source is None — cannot index. Pass a DataSource / iterable to Flux(source=...).") + self._guard_not_expanding("__getitem__") if hasattr(source, "__getitem__"): raw = source[index] @@ -342,17 +499,18 @@ def __getitem__(self, index: int) -> Sample: "it in ``list(...)`` before handing it to Flux." ) _check_ops_materialized(self.ops) - sample = Sample.from_any(raw) - for op in self.ops: - result = _apply_op(sample, op) - if result is None: - raise IndexError(f"Sample {index} filtered out by {op}") - sample = result - return sample + sample: Any = raw if self.native else Sample.from_any(raw) + with activate(Context()): + for op in self.ops: + result = _apply_op_native(sample, op) if self.native else _apply_op(sample, op) + if result is None: + raise IndexError(f"Sample {index} filtered out by {op}") + sample = result + return cast(Sample, sample) def to_sink(self, sink: Any) -> None: """Write the entire flux to a DataSink.""" - from dataflux.storage.base import Storage + from sampleflux.storage.base import Storage # Open sink if it's a context-aware storage; otherwise no-op context. target_sink: Any = sink if isinstance(sink, Storage) else nullcontext() @@ -400,7 +558,7 @@ def __iter__(self) -> Iterator[Any]: Routing: * Any op exposes a callable ``stream`` attribute (e.g. - :class:`dataflux.ops.parallel.Parallel`) → :meth:`_iter_streamed`, + :class:`sampleflux.ops.parallel.Parallel`) → :meth:`_iter_streamed`, which composes the upstream iterator through stream-level ops. * Else ``self._workers > 1`` → legacy :meth:`_iter_parallel`. * Else :meth:`_iter_sequential`. @@ -432,35 +590,73 @@ def _iter_streamed(self) -> Iterator[Sample]: Per-sample ops are applied via ``op(sample)``. Ops that implement ``.stream(sample_iter)`` (e.g. - :class:`dataflux.ops.parallel.Parallel`) are handed the upstream + :class:`sampleflux.ops.parallel.Parallel`) are handed the upstream generator and yield transformed samples themselves. ``None`` results are filtered, matching :meth:`_iter_sequential`. + + Each sample travels with its own per-sample :class:`Context` (a private + ``(sample, ctx)`` carrier between per-sample stages), activated around + every ``_apply_op`` call. A stream-level op is a Context boundary: the + carrier is stripped to a bare sample before ``op.stream(...)`` (raising + if cells are still live — cross-``Parallel`` graphs are a documented v1 + limit; ``Parallel``'s INNER chain gets its own contexts via + :func:`_worker_task`), and samples emerging downstream get fresh + contexts. """ source = self._guard_live_source() if source is None: return _check_ops_materialized(self.ops) - def to_samples() -> Iterator[Sample]: + def to_carried() -> Iterator[Optional[_Carried]]: for item in source: - yield Sample.from_any(item) + yield _Carried(item if self.native else Sample.from_any(item), Context()) - def per_sample(stream: Iterator[Optional[Sample]], op: Any) -> Iterator[Optional[Sample]]: - for s in stream: - if s is None: + def per_sample(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[_Carried]]: + from sampleflux.kinds import op_contract + + expands = op_contract(op).expands + for c in stream: + if c is None: continue - yield _apply_op(s, op) + with activate(c.ctx): + if expands: + children = _expand(op, c.sample) + else: + s = _apply_op_native(c.sample, op) if self.native else _apply_op(c.sample, op) + if expands: + for j, child in enumerate(children): + yield _Carried(child, c.ctx if j == 0 else c.ctx.copy()) + else: + yield None if s is None else _Carried(s, c.ctx) + + def strip(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[Sample]]: + for c in stream: + if c is None: + yield None + continue + if c.ctx.live(): + raise RuntimeError( + f"Flux: context cells {c.ctx.live()!r} are still live at the stream-level op " + f"{type(op).__name__!r}. Context cells cannot cross a stream-op boundary " + f"(e.g. Parallel) — drop them before it, or move the whole graph inside it." + ) + yield c.sample + + def wrap(stream: Iterator[Optional[Sample]]) -> Iterator[Optional[_Carried]]: + for s in stream: + yield None if s is None else _Carried(s, Context()) - stream: Iterator[Optional[Sample]] = to_samples() + carried: Iterator[Optional[_Carried]] = to_carried() for op in self.ops: if hasattr(op, "stream") and callable(op.stream): - stream = op.stream(stream) + carried = wrap(op.stream(strip(carried, op))) else: - stream = per_sample(stream, op) + carried = per_sample(carried, op) - for s in stream: - if s is not None: - yield s + for c in carried: + if c is not None: + yield c.sample def _iter_sequential(self) -> Iterator[Sample]: """Standard single-threaded execution.""" @@ -469,10 +665,8 @@ def _iter_sequential(self) -> Iterator[Sample]: return _check_ops_materialized(self.ops) for item in source: - sample = Sample.from_any(item) - result = _worker_task(sample, self.ops) - if result is not None: - yield result + sample = item if self.native else Sample.from_any(item) + yield from _worker_task_multi(sample, self.ops, native=self.native) def _iter_parallel(self) -> Iterator[Sample]: """Multiprocess execution engine.""" @@ -487,13 +681,11 @@ def _iter_parallel(self) -> Iterator[Sample]: with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: futures = [] for item in source: - sample = Sample.from_any(item) - futures.append(executor.submit(_worker_task, sample, self.ops)) + sample = item if self.native else Sample.from_any(item) + futures.append(executor.submit(_worker_task_multi, sample, self.ops, self.native)) for future in futures: - result = future.result() - if result is not None: - yield result + yield from future.result() def collect(self) -> List[Sample]: """Materialize the full flux into a list.""" @@ -502,7 +694,7 @@ def collect(self) -> List[Sample]: def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: """Yield pipeline-output Samples carrying only ``fields`` (the projection primitive). - Implements :class:`dataflux.projection.SupportsProjection`. Flux must run + Implements :class:`sampleflux.projection.SupportsProjection`. Flux must run its op chain to produce each Sample (an op may consume the input), so this is the generic "iterate, then drop unrequested fields" form — it cannot skip input construction the way a leaf source (e.g. an image dataset that diff --git a/dataflux/discovery.py b/sampleflux/discovery.py similarity index 98% rename from dataflux/discovery.py rename to sampleflux/discovery.py index 94fa9d0..16af5e3 100644 --- a/dataflux/discovery.py +++ b/sampleflux/discovery.py @@ -1,4 +1,4 @@ -"""Passive introspection for DataFlux callables. +"""Passive introspection for SampleFlux callables. A round-trip bridge between live Python callables (sources, ops, plain functions) and JSON-serializable schemas, so downstream tools can discover and @@ -141,7 +141,7 @@ def introspect_callable(func: Callable) -> Dict[str, Any]: def _spec_dict(spec: Any) -> Optional[Dict[str, Any]]: - """JSON-serialize a declared ``ACCEPTS`` / ``PRODUCES`` (a :class:`~dataflux.typespec.SampleType`), + """JSON-serialize a declared ``ACCEPTS`` / ``PRODUCES`` (a :class:`~sampleflux.typespec.SampleType`), or ``None`` when undeclared. Duck-typed so ``discovery`` needn't import ``typespec``.""" to_dict = getattr(spec, "to_dict", None) return cast(Dict[str, Any], to_dict()) if callable(to_dict) else None diff --git a/sampleflux/flow.py b/sampleflux/flow.py new file mode 100644 index 0000000..226ba9b --- /dev/null +++ b/sampleflux/flow.py @@ -0,0 +1,781 @@ +"""The ``flow:`` document, the :class:`FlowGraph` engine, and the flow⇄ops converters. + +A **flow document** is the readable, named-step form of a graph-shaped pipeline: a +mapping of ``step-name → op``, where a step's name is also the name later steps use to +reference its result. It is the authoring format (humans and the FluxStudio exporter +write it); the flat context-ops form (:mod:`sampleflux.ops.context`) is the serial +execution format the plain :class:`~sampleflux.core.Flux` engine runs. The two convert +**bidirectionally**: :func:`to_ops` lowers a flow into a flat op list, :func:`from_ops` +lifts a flat op list back into a flow — with execution parity in both directions. + +.. code-block:: yaml + + flow: + spec: !class:waivefront.SpectrogramOp() # input: the source sample + rescaled: !class:sampleflux.ops.numpy.RescaleOp() # input: previous step + masked: !class:waivefront.SegmentOp() {from: spec} # 2nd reader of spec = fan-out + thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: masked} + denoised: !class:waivefront.torchsig.processing.NoiseFloorOp() + from: rescaled + bind: {low_level: thresh} # per-sample param := thresh's result + out: {from: denoised, target_from: masked} # pure fan-in (no op) + outputs: out + +Step grammar (the four RESERVED step keys, stripped before the op is built): + +- ``from:`` — the step supplying this step's input sample. Omitted = the previous step + (the first step reads the source sample). Must name an EARLIER step: document order is + the schedule, so forward references are errors and cycles are inexpressible. +- ``target_from:`` / ``metadata_from:`` — fan-in: compose the incoming sample's target / + metadata from another step's result before the op runs (the ``Mix`` slot semantics — + a Sample result contributes its corresponding field, metadata merges last-write-wins). +- ``bind:`` — ``{param: ref}`` per-sample parameters: ``ref`` is a step name (its result + sample's ``input``, or the raw value) or ``step.attr`` (the step op's live ``@output`` + after it ran — lowered through ``Capture``; stochastic-correct). + +A step may be a plain mapping with no op (``out: {from: a, target_from: b}``) — a pure +fan-in/identity step; ``{}`` is the identity (used to give the source a referable name). +``outputs:`` names the step whose result the pipeline yields (default: the last step). + +Cell-lifetime management is AUTOMATIC in both forms: :class:`FlowGraph` frees each step +result after its last reader, and :func:`to_ops` computes the same liveness into +``drop`` flags on the emitted context ops. +""" + +import inspect +from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Sequence, Tuple, Union, cast + +import torch.utils.data +from confluid import configurable, flow +from confluid import resolve as _confluid_resolve +from confluid.fluid import Fluid as _ConfluidFluid +from loggair import get_logger + +from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, Mix, Save, Use, _read_output +from sampleflux.sample import Sample + +logger = get_logger(__name__) + +RESERVED_STEP_KEYS = ("from", "target_from", "metadata_from", "bind") +"""Step-grammar keys stripped from a step mapping before the op is constructed.""" + +__all__ = ["FlowGraph", "FlowStep", "from_ops", "parse_flow", "to_ops", "RESERVED_STEP_KEYS"] + + +class FlowStep(NamedTuple): + """One parsed step of a flow document.""" + + name: str + op: Optional[Any] # live op callable; None = pure fan-in / identity step + from_: Optional[str] # None = previous step (first step: the source sample) + target_from: Optional[str] + metadata_from: Optional[str] + bind: Dict[str, str] # param -> "step" | "step.attr" + + +class _BindRef(NamedTuple): + """A parsed ``bind:`` reference.""" + + step: str + attr: Optional[str] # None = the step's result; else the step op's @output attribute + + +def _parse_bind_ref(ref: str, known: Sequence[str]) -> _BindRef: + head, dot, attr = str(ref).partition(".") + if head not in known: + raise ValueError( + f"flow: bind reference {ref!r} does not name an earlier step " + f"(known steps at this point: {list(known)!r})" + ) + return _BindRef(head, attr if dot else None) + + +def _check_reserved_collision(op: Any, step_name: str) -> None: + """Raise if the op's constructor has a param named like a reserved step key. + + Reserved keys are stripped from the step mapping before the op is built, so such a + param could never be configured inline — fail loudly instead of silently stealing it. + (``from`` is a Python keyword and can never be a param, but the others could.) + """ + try: + params = inspect.signature(type(op).__init__).parameters + except (TypeError, ValueError): # pragma: no cover - C-extension ctor + return + clash = [k for k in RESERVED_STEP_KEYS if k in params] + if clash: + raise ValueError( + f"flow step {step_name!r}: op {type(op).__name__!r} has constructor parameter(s) " + f"{clash!r} that collide with reserved flow step keys {RESERVED_STEP_KEYS!r} — " + "such an op cannot be configured in a flow document; rename the parameter or " + "wire the op in the flat ops form instead." + ) + + +def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[List[FlowStep], str]: + """Parse a flow mapping into ordered :class:`FlowStep`\\ s + the resolved output step name. + + ``flow_doc`` is the ``flow:`` mapping — step values may be confluid markers (from + ``resolve()``/``load()``), plain dicts (pure fan-in steps, or programmatic + ``{"op": , "from": ...}`` form), or live op callables. Reserved keys are popped; + markers are flowed per step (confluid does not auto-flow two-levels-nested markers). + Validates: step names carry no dots, every reference points to an EARLIER step. + + ``build=False`` keeps a marker step UNBUILT (the op stays a Fluid marker) — for + structural consumers (converters/importers) that must not materialize ops (hoisted + dotted ``!ref:`` values would resolve outside their document); such steps skip the + reserved-ctor-param check and their live construction happens at first call. + """ + if not isinstance(flow_doc, dict) or not flow_doc: + raise ValueError("flow: expected a non-empty mapping of step-name -> op") + + steps: List[FlowStep] = [] + seen: List[str] = [] + for name, value in flow_doc.items(): + name = str(name) + if "." in name: + raise ValueError(f"flow: step name {name!r} may not contain '.' (reserved for @output refs)") + if name in seen: + raise ValueError(f"flow: duplicate step name {name!r}") + + reserved: Dict[str, Any] = {} + op: Optional[Any] + if isinstance(value, _ConfluidFluid): + for key in RESERVED_STEP_KEYS: + if key in value.kwargs: + reserved[key] = value.kwargs.pop(key) + op = flow(value) if build else value + elif isinstance(value, dict): + extra = value.get("op") + reserved = {k: v for k, v in value.items() if k in RESERVED_STEP_KEYS} + unknown = [k for k in value if k not in RESERVED_STEP_KEYS and k != "op"] + if unknown: + raise ValueError( + f"flow step {name!r}: unknown step key(s) {unknown!r} — a plain-mapping step " + f"accepts only {RESERVED_STEP_KEYS!r} and 'op'" + ) + op = flow(extra) if (build and isinstance(extra, _ConfluidFluid)) else extra + elif callable(value): + op = value + elif value is None: + op = None + else: + raise TypeError(f"flow step {name!r}: expected an op, a marker, or a mapping — got {type(value).__name__}") + + if op is not None and not isinstance(op, _ConfluidFluid) and not callable(op): + raise TypeError(f"flow step {name!r}: op is not callable ({type(op).__name__})") + if op is not None and not isinstance(op, _ConfluidFluid): + _check_reserved_collision(op, name) + + from_ = reserved.get("from") + target_from = reserved.get("target_from") + metadata_from = reserved.get("metadata_from") + for key, ref in (("from", from_), ("target_from", target_from), ("metadata_from", metadata_from)): + if ref is not None and str(ref) not in seen: + raise ValueError( + f"flow step {name!r}: {key}: {ref!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) + bind_raw = reserved.get("bind") or {} + if not isinstance(bind_raw, dict): + raise TypeError(f"flow step {name!r}: bind must be a mapping of param -> step[.output]") + bind: Dict[str, str] = {} + for param, ref in bind_raw.items(): + _parse_bind_ref(str(ref), seen) # validates + bind[str(param)] = str(ref) + if bind and op is None: + raise ValueError(f"flow step {name!r}: bind requires an op to configure") + + steps.append( + FlowStep( + name=name, + op=op, + from_=None if from_ is None else str(from_), + target_from=None if target_from is None else str(target_from), + metadata_from=None if metadata_from is None else str(metadata_from), + bind=bind, + ) + ) + seen.append(name) + + out = str(outputs) if outputs else steps[-1].name + if out not in seen: + raise ValueError(f"flow: outputs {out!r} does not name a step (steps: {seen!r})") + return steps, out + + +def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[Tuple[int, str]]]: + """Step-result cell -> ordered ``(consumer_index, slot)`` reads. + + Slot granularity matters: one consumer step may read the SAME producer through several + slots (its input AND a ``bind`` param), and only the ``"in"`` slot of the immediately + following step can ride the linear stream. Slots: ``"in"`` (input), ``"target"``, + ``"meta"``, ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A + ``bind`` step-result reference counts; an ``@output`` (``step.attr``) reference does + NOT (it reads the op instance, not the result cell). + """ + readers: Dict[str, List[Tuple[int, str]]] = {s.name: [] for s in steps} + for i, step in enumerate(steps): + implicit = steps[i - 1].name if i > 0 else None + source = step.from_ or implicit + if source is not None: + readers[source].append((i, "in")) + if step.target_from is not None: + readers[step.target_from].append((i, "target")) + if step.metadata_from is not None: + readers[step.metadata_from].append((i, "meta")) + for ref in step.bind.values(): + parsed = _BindRef(*ref.partition(".")[::2]) if "." in ref else _BindRef(ref, None) + if parsed.attr is None: + readers[parsed.step].append((i, "bind")) + readers[outputs].append((len(steps), "out")) + return readers + + +# --------------------------------------------------------------------------- +# The FlowGraph engine +# --------------------------------------------------------------------------- + + +@configurable(category="engine") +class FlowGraph(torch.utils.data.Dataset[Sample]): + """Named-step graph engine — executes a ``flow:`` document natively. + + The readable twin of :class:`~sampleflux.core.Flux`: steps run in document order over + a per-sample environment of named results, with fan-out isolation (copy-on-read, move + on last read) and automatic cell lifetimes. Any FlowGraph converts to a flat op list + for the serial engine (:func:`to_ops`) and back (:func:`from_ops`) — execution parity + between the two is a pinned contract. + + Args: + source: Any iterable or indexable dataset (duck-typed) to wrap; ``None`` yields an empty stream. + flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. + outputs: Name of the step whose result is yielded. Blank (default) = the last step. + chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single samples. + """ + + def __init__( + self, + source: Optional[Any] = None, + flow: Optional[Union[Dict[str, Any], List[FlowStep]]] = None, + outputs: str = "", + chunk_size: int = 0, + ) -> None: + # Lazy / zero-arg: store config only; parsing/validation happen in the cached property. + self.source = source + self.flow = flow + self.outputs = str(outputs) + self._chunk_size = int(chunk_size) + self._workers = 1 + self._parsed: Optional[Tuple[List[FlowStep], str]] = None + + # -- parsing ----------------------------------------------------------- + + @property + def steps(self) -> List[FlowStep]: + """The parsed, validated steps (cached; recomputed only if ``flow`` is reassigned).""" + return self._ensure_parsed()[0] + + @property + def output_step(self) -> str: + """The resolved output step name.""" + return self._ensure_parsed()[1] + + def _ensure_parsed(self) -> Tuple[List[FlowStep], str]: + if self._parsed is None: + if self.flow is None: + raise ValueError("FlowGraph.flow is not set — provide a flow mapping or FlowStep list.") + if isinstance(self.flow, list) and all(isinstance(s, FlowStep) for s in self.flow): + names = [s.name for s in self.flow] + out = self.outputs or (names[-1] if names else "") + if out not in names: + raise ValueError(f"FlowGraph: outputs {out!r} does not name a step ({names!r})") + self._parsed = (list(self.flow), out) + else: + self._parsed = parse_flow(cast(Dict[str, Any], self.flow), self.outputs) + return self._parsed + + @classmethod + def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": + """Build a FlowGraph from a ``{flow: {...}, outputs: ...}`` YAML document (or inline string). + + Uses ``confluid.resolve`` so step markers stay UNbuilt until :func:`parse_flow` + pops the reserved step keys and flows each op itself. + """ + doc = _confluid_resolve(path) + if not isinstance(doc, dict) or "flow" not in doc: + raise ValueError(f"FlowGraph.from_yaml: {path!r} has no 'flow:' mapping") + return cls(source=source, flow=doc["flow"], outputs=str(doc.get("outputs", "") or "")) + + @classmethod + def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": + """Lift a flat ``{ops: [...]}`` YAML document into a FlowGraph (via :func:`from_ops`).""" + from sampleflux.core import Flux + + flux = Flux.from_ops_yaml(path, source=source) + flow_doc, outputs = from_ops(flux.ops) + return cls(source=source, flow=flow_doc, outputs=outputs) + + # -- execution --------------------------------------------------------- + + def _run(self, seed: Sample) -> Optional[Sample]: + """Run one sample through the steps; ``None`` = filtered (an op returned None).""" + steps, outputs = self._ensure_parsed() + readers = _result_readers(steps, outputs) + env: Dict[str, Any] = {} + remaining = {name: len(idx) for name, idx in readers.items()} + + def read_result(name: str, *, copy: bool) -> Any: + value = env[name] + remaining[name] -= 1 + if remaining[name] <= 0: + del env[name] + elif copy: + from copy import deepcopy + + value = deepcopy(value) + return value + + prev: Optional[str] = None + for step in steps: + # 1. the input sample (implicit stream reads move; explicit fan-out reads copy) + if step.from_ is not None: + base = read_result(step.from_, copy=True) + elif prev is not None: + base = read_result(prev, copy=False) + else: + base = seed + sample = Sample.from_any(base) + + # 2. fan-in slots (Mix semantics) + if step.target_from is not None or step.metadata_from is not None: + metadata = dict(sample.meta) + target = sample.target + if step.target_from is not None: + value = read_result(step.target_from, copy=True) + target = value.target if isinstance(value, Sample) else value + if isinstance(value, Sample): + metadata.update(value.meta) + if step.metadata_from is not None: + value = read_result(step.metadata_from, copy=True) + extra = value.meta if isinstance(value, Sample) else value + if not isinstance(extra, dict): + raise TypeError( + f"flow step {step.name!r}: metadata_from holds {type(extra).__name__}, " + "expected a Sample or a dict" + ) + metadata.update(extra) + sample = sample._replace(target=target, metadata=metadata) + + # 3. per-sample parameter binds + if step.op is not None: + op = step.op + from sampleflux.kinds import op_contract as _op_contract + + if _op_contract(op).expands: + raise NotImplementedError( + f"flow step {step.name!r}: {type(op).__name__!r} is a 1→N expanding op — " + "FlowGraph steps are strictly 1→1 (a named-step env has one result per step). " + "Run expanding pipelines through the Flux engine (iterable-only)." + ) + for param, ref in step.bind.items(): + if "." in ref: + head, _, attr = ref.partition(".") + producer = next(s for s in steps if s.name == head) + value = _read_output(producer.op, attr) + if value is _MISSING: + raise AttributeError( + f"flow step {step.name!r}: bind {param}={ref!r} — " + f"step {head!r} op has no @output attribute {attr!r}" + ) + else: + value = read_result(ref, copy=False) + if isinstance(value, Sample): + value = value.input + setattr(op, param, value) + result = op(sample) + if result is None: + return None + sample = cast(Sample, result) + + env[step.name] = sample + prev = step.name + + return cast(Optional[Sample], env.get(outputs)) if outputs in env else None + + def __iter__(self) -> Iterator[Any]: + if self.source is None: + return + it = self._iter_samples() + if self._chunk_size > 0: + batch: List[Sample] = [] + for sample in it: + batch.append(sample) + if len(batch) == self._chunk_size: + yield batch + batch = [] + if batch: + yield batch + else: + yield from it + + def _iter_samples(self) -> Iterator[Sample]: + if self._workers > 1: + yield from self._iter_parallel() + return + assert self.source is not None + for item in self.source: + result = self._run(Sample.from_any(item)) + if result is not None: + yield result + + def _iter_parallel(self) -> Iterator[Sample]: + """Multiprocess execution — delegates to the serial engine over the LOWERED op list. + + Lowering + Flux's spawn pool is the sanctioned parallel path (one worker + implementation, guaranteed parity by the to_ops contract); a native process pool + here would duplicate it for no gain. + """ + from sampleflux.core import Flux + + assert self.source is not None + flux = Flux(source=self.source, ops=to_ops(self.steps, self.output_step)).parallel(self._workers) + yield from flux + + def __len__(self) -> int: + from collections.abc import Sized + + if isinstance(self.source, Sized): + return len(self.source) + return 0 + + def __getitem__(self, index: int) -> Sample: + if self.source is None: + raise TypeError("FlowGraph source is None — cannot index.") + if hasattr(self.source, "__getitem__"): + raw = self.source[index] + else: + raise TypeError( + f"FlowGraph source {type(self.source).__name__} does not support indexing; " + "wrap it in a list or use iteration." + ) + result = self._run(Sample.from_any(raw)) + if result is None: + raise IndexError(f"Sample {index} filtered out by the flow") + return result + + def parallel(self, workers: int = 4) -> "FlowGraph": + """Enable multiprocess execution (spawn, via the lowered serial form).""" + self._workers = workers + return self + + def batch(self, chunk_size: int) -> "FlowGraph": + """Group yielded samples into lists of ``chunk_size``.""" + self._chunk_size = chunk_size + return self + + def collect(self) -> List[Any]: + """Materialize the full stream into a list.""" + return list(self) + + def to_flux(self) -> Any: + """The serial-engine twin: a Flux running the LOWERED flat op list (same results).""" + from sampleflux.core import Flux + + return Flux(source=self.source, ops=to_ops(self.steps, self.output_step)) + + +# --------------------------------------------------------------------------- +# Lowering: flow -> flat context-ops list +# --------------------------------------------------------------------------- + + +def to_ops(steps: Union[Sequence[FlowStep], Dict[str, Any]], outputs: str = "") -> List[Any]: + """Lower a flow (parsed steps or a raw flow mapping) into a flat context-ops list. + + The result runs on the plain serial :class:`~sampleflux.core.Flux` engine and is the + serialization form FluxStudio's ``--serial`` export emits. Cell names are the step + names (deterministic, diffable); liveness is compiled into ``drop`` flags so a + well-formed graph leaves the Context empty. A purely linear flow lowers to the bare + op list — zero context ops. + """ + if isinstance(steps, dict): + parsed, outputs = parse_flow(steps, outputs) + else: + parsed = list(steps) + outputs = outputs or (parsed[-1].name if parsed else "") + + readers = _result_readers(parsed, outputs) + # Which step results must live in a cell? Every read EXCEPT the one that can ride the + # linear stream: the immediately-next step's INPUT slot, or the final output read when + # this is the last step. Slot granularity matters — a consumer may read the same + # producer through its input slot AND a bind slot (only the input slot can stream). + needs_cell: Dict[str, bool] = {} + cell_reads_left: Dict[str, int] = {} + for i, step in enumerate(parsed): + consumers = list(readers[step.name]) + stream_read: Optional[Tuple[int, str]] = None + if i + 1 < len(parsed) and (parsed[i + 1].from_ or step.name) == step.name: + stream_read = (i + 1, "in") + elif i == len(parsed) - 1: + stream_read = (len(parsed), "out") + cell_reads = [c for c in consumers if c != stream_read] + needs_cell[step.name] = bool(cell_reads) + cell_reads_left[step.name] = len(cell_reads) + + ops: List[Any] = [] + attr_cells: Dict[str, str] = {} # "step.attr" -> cell name + + # Pre-scan @output refs: the producer op must be wrapped in Capture at ITS step. + attr_refs: Dict[str, List[str]] = {} + for step in parsed: + for ref in step.bind.values(): + if "." in ref: + head, _, attr = ref.partition(".") + attr_refs.setdefault(head, []) + if attr not in attr_refs[head]: + attr_refs[head].append(attr) + + def take_cell(name: str) -> Tuple[str, bool]: + """(cell, is_last_read) — decrement the read counter.""" + cell_reads_left[name] -= 1 + return name, cell_reads_left[name] <= 0 + + prev_name: Optional[str] = None + for i, step in enumerate(parsed): + # 1. input slot (explicit from == previous step consumes the stream — no Use) + if step.from_ is not None and step.from_ != prev_name: + cell, last = take_cell(step.from_) + ops.append(Use(name=cell, drop=last)) + + # 2. fan-in slots + if step.target_from is not None or step.metadata_from is not None: + drops: List[str] = [] + kwargs: Dict[str, Any] = {} + if step.target_from is not None: + cell, last = take_cell(step.target_from) + kwargs["target_from"] = cell + if last: + drops.append(cell) + if step.metadata_from is not None: + cell, last = take_cell(step.metadata_from) + kwargs["metadata_from"] = cell + if last: + drops.append(cell) + ops.append(Mix(drop=drops, **kwargs)) + + # 3. the op, wrapped for binds (Apply) and @output captures (Capture) + emitted: Optional[Any] = step.op + if emitted is not None: + for param, ref in step.bind.items(): + if "." in ref: + cell = attr_cells[ref] + cell_reads_left.setdefault(cell, 1) + cell_reads_left[cell] -= 1 + emitted = Apply(op=emitted, param=param, source=cell, drop=cell_reads_left[cell] <= 0) + else: + cell, last = take_cell(ref) + emitted = Apply(op=emitted, param=param, source=cell, drop=last) + captures = attr_refs.get(step.name, []) + if captures: + for attr in captures: + cell = f"{step.name}.{attr}" + attr_cells[cell] = cell + cell_reads_left[cell] = sum( + 1 for s in parsed for r in s.bind.values() if r == f"{step.name}.{attr}" + ) + if len(captures) == 1: + emitted = Capture(op=emitted, output=captures[0], name=f"{step.name}.{captures[0]}") + else: + emitted = Capture(op=emitted, captures={a: f"{step.name}.{a}" for a in captures}) + ops.append(emitted) + elif step.target_from is None and step.metadata_from is None and step.from_ is None and i == 0: + # identity first step ({}: names the source) — nothing to run + pass + + # 4. persist the result for non-stream readers + if needs_cell[step.name]: + ops.append(Save(name=step.name)) + + prev_name = step.name + + # 5. the output: if it is not the final stream, fetch it. + if parsed and outputs != parsed[-1].name: + cell, last = take_cell(outputs) + ops.append(Use(name=cell, drop=last)) + + # 6. safety net: any cells the liveness pass left alive get an explicit Drop. + leftovers = [name for name, left in cell_reads_left.items() if left > 0 and needs_cell.get(name, True)] + if leftovers: + ops.append(Drop(names=sorted(leftovers))) + + return ops + + +# --------------------------------------------------------------------------- +# Lifting: flat context-ops list -> flow +# --------------------------------------------------------------------------- + + +_CONTEXT_OP_CLASSES = (Save, Use, Drop, Apply, Capture, Mix) + + +def _ctx_view(raw: Any) -> Optional[type]: + """The context-op class ``raw`` represents, live instance OR confluid marker; else None.""" + if isinstance(raw, _ConfluidFluid): + target = getattr(raw, "target", None) + return target if isinstance(target, type) and target in _CONTEXT_OP_CLASSES else None + return type(raw) if isinstance(raw, _CONTEXT_OP_CLASSES) else None + + +def _ctx_field(raw: Any, name: str, default: Any = None) -> Any: + """Read a context-op field off a live instance OR a marker's kwargs.""" + if isinstance(raw, _ConfluidFluid): + return raw.kwargs.get(name, default) + return getattr(raw, name, default) + + +def _capture_items(raw: Any) -> Dict[str, str]: + """A Capture's ``{output_attr: cell}`` map, live instance or marker.""" + if not isinstance(raw, _ConfluidFluid): + return cast(Capture, raw)._items() + items = dict(raw.kwargs.get("captures") or {}) + output = str(raw.kwargs.get("output", "") or "") + if output: + items.setdefault(output, str(raw.kwargs.get("name", "") or "") or output) + return items + + +def _auto_name(op: Any, index: int, taken: Dict[str, int]) -> str: + if op is None: + base = "step" + elif isinstance(op, _ConfluidFluid): + target = getattr(op, "target", None) + base = getattr(target, "__name__", str(target)).lower() + else: + base = type(op).__name__.lower() + taken[base] = taken.get(base, 0) + 1 + return base if taken[base] == 1 else f"{base}_{taken[base]}" + + +def from_ops(ops: Sequence[Any], outputs: str = "") -> Tuple[Dict[str, Any], str]: + """Lift a flat op list into a ``(flow_mapping, outputs)`` pair. + + Context ops are absorbed into step grammar: ``Save`` names the preceding step (or an + identity first step for a source fork), ``Use`` starts a branch (``from:``), ``Mix`` + becomes ``target_from``/``metadata_from`` on the following step (or a pure fan-in + step), ``Apply``/``Capture`` unwrap into ``bind:`` references, and ``Drop`` vanishes + (liveness is recomputed on lowering). A plain linear list lifts to a linear flow with + auto-generated step names. The result round-trips: ``to_ops(from_ops(ops))`` is + execution-equivalent to ``ops``. + + Accepts LIVE ops or confluid ``Instance``/``Class`` MARKERS interchangeably (the + FluxStudio exporter lifts compiled marker lists without materializing them, keeping + hoisted-constant ``!ref:``\\ s intact); a real op arrives in the flow mapping verbatim + (marker in, marker out). + """ + flow_map: Dict[str, Dict[str, Any]] = {} + taken: Dict[str, int] = {} + prev_name: Optional[str] = None + capture_cells: Dict[str, str] = {} # cell -> "step.attr" bind ref + pending: Dict[str, Any] = {} # accumulating step grammar (from/target_from/...) + + def cell_ref(cell: str) -> str: + """Map a cell name to its bind reference (an @output capture or a step result).""" + return capture_cells.get(cell, cell) + + def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: + nonlocal prev_name, pending + name = explicit_name or _auto_name(op, len(flow_map), taken) + entry: Any + if op is not None and not pending: + entry = op # a grammar-less step is just its op (the compact document form) + else: + entry = dict(pending) + if op is not None: + entry["op"] = op + flow_map[name] = entry + pending = {} + prev_name = name + return name + + for raw in ops: + view = _ctx_view(raw) + if view is Save: + save_name = str(_ctx_field(raw, "name", "")) + if prev_name is not None: + # rename the just-flushed step to the cell name + entry = flow_map.pop(prev_name) + # keep bind refs pointing at the old auto name consistent + for e in flow_map.values(): + b = e.get("bind") if isinstance(e, dict) else None + if b: + for p, r in list(b.items()): + head, dot, attr = r.partition(".") + if head == prev_name: + b[p] = save_name + (dot + attr if dot else "") + flow_map[save_name] = entry + for cell, ref in list(capture_cells.items()): + head, dot, attr = ref.partition(".") + if head == prev_name: + capture_cells[cell] = save_name + (dot + attr if dot else "") + prev_name = save_name + else: + # Save before any op: an identity step naming the source + flow_map[save_name] = {} + prev_name = save_name + continue + if view is Use: + pending["from"] = cell_ref(str(_ctx_field(raw, "name", ""))) + continue + if view is Mix: + mix_grammar: Dict[str, Any] = {} + if _ctx_field(raw, "input_from", ""): + mix_grammar["from"] = cell_ref(str(_ctx_field(raw, "input_from"))) + if _ctx_field(raw, "target_from", ""): + mix_grammar["target_from"] = cell_ref(str(_ctx_field(raw, "target_from"))) + if _ctx_field(raw, "metadata_from", ""): + mix_grammar["metadata_from"] = cell_ref(str(_ctx_field(raw, "metadata_from"))) + pending.update(mix_grammar) + pending["__mix_pending__"] = True + continue + if view is Drop: + continue # liveness is recomputed on lowering + + # A real op (possibly Apply/Capture-wrapped): unwrap into bind grammar. + bind: Dict[str, str] = {} + captures: Dict[str, str] = {} + op: Any = raw + while _ctx_view(op) in (Apply, Capture): + if _ctx_view(op) is Capture: + for attr, cell in _capture_items(op).items(): + captures[cell] = attr + else: + bind[str(_ctx_field(op, "param", ""))] = cell_ref(str(_ctx_field(op, "source", ""))) + op = _ctx_field(op, "op") + pending.pop("__mix_pending__", None) + if bind: + pending["bind"] = bind + name = flush_step(op) + for cell, attr in captures.items(): + capture_cells[cell] = f"{name}.{attr}" + + # A trailing Mix (or Use) with no following op = a pure fan-in step. + if pending: + pending.pop("__mix_pending__", None) + flush_step(None) + + if not flow_map: + raise ValueError("from_ops: no steps could be lifted (empty op list?)") + out = outputs or prev_name or next(reversed(flow_map)) + return flow_map, out + + +def flow_yaml_to_flux(path: str, source: Optional[Any] = None) -> Any: + """Convenience: load a ``flow:`` YAML and return the SERIAL engine (lowered Flux).""" + from sampleflux.core import Flux + + doc = _confluid_resolve(path) + if not isinstance(doc, dict) or "flow" not in doc: + raise ValueError(f"flow_yaml_to_flux: {path!r} has no 'flow:' mapping") + parsed, outputs = parse_flow(doc["flow"], str(doc.get("outputs", "") or "")) + return Flux(source=source, ops=to_ops(parsed, outputs)) diff --git a/sampleflux/kinds.py b/sampleflux/kinds.py new file mode 100644 index 0000000..f6afb5f --- /dev/null +++ b/sampleflux/kinds.py @@ -0,0 +1,151 @@ +"""Op-kind introspection — what carrier an op accepts/produces, detected from its annotations. + +The native multi-type engine (``Flux(native=True)``) lets carriers other than +:class:`~sampleflux.sample.Sample` flow through a pipeline — metadata-free **pairs** like +``(image, label)`` / ``(tensor, mask)`` / ``(tensor, coco_dict)``, or bare **values**. +Ops can process everything: the engine detects each op's contract by INTROSPECTING the +``__call__`` type annotations (``__call__(self, sample: Sample)`` vs +``__call__(self, pair: tuple[np.ndarray, int])`` vs untyped = works-on-anything) and +adapts the carrier per op. Explicit class attributes (``SAMPLE_KIND_IN`` / +``SAMPLE_KIND_OUT`` / ``EXPANDS``) override detection for cases introspection can't see +(C-extension callables, wrappers around raw functions). + +The same introspection powers 1→N detection: a ``-> Iterator[Sample]`` / +``-> Iterable[Sample]`` return annotation (or ``EXPANDS = True``) marks an EXPANDING op — +one carrier in, several out — which makes the pipeline iterable-only (see +``Flux.__len__``/``__getitem__``). +""" + +import collections.abc +import inspect +from dataclasses import dataclass +from typing import Any, Dict, Literal, Tuple, Union, get_args, get_origin, get_type_hints + +from sampleflux.sample import Sample + +SampleKind = Literal["sample", "pair", "value", "any"] +"""The carrier taxonomy: a full Sample triplet, a metadata-free 2-tuple, a bare value, or anything.""" + +SAMPLE_KINDS: Tuple[str, ...] = get_args(SampleKind) + +__all__ = ["OpContract", "SAMPLE_KINDS", "SampleKind", "classify_carrier", "op_contract"] + +_EXPANDING_ORIGINS = ( + list, + set, + frozenset, + collections.abc.Iterable, + collections.abc.Iterator, + collections.abc.Generator, + collections.abc.Sequence, +) + + +@dataclass(frozen=True) +class OpContract: + """What an op consumes and produces. + + ``accepts``/``produces`` are :data:`SampleKind` members; ``expands`` marks a 1→N op + (returns an iterable of carriers instead of one). + """ + + accepts: SampleKind = "any" + produces: SampleKind = "any" + expands: bool = False + + +_ANY_CONTRACT = OpContract() +_contract_cache: Dict[type, OpContract] = {} + + +def classify_carrier(obj: Any) -> SampleKind: + """The carrier kind of a runtime object: Sample -> ``sample``, 2-tuple -> ``pair``, else ``value``.""" + if isinstance(obj, Sample): + return "sample" + if isinstance(obj, tuple) and len(obj) == 2: + return "pair" + return "value" + + +def _unwrap_optional(anno: Any) -> Any: + """``Optional[X]`` / ``Union[X, None]`` -> ``X`` (multi-arm unions are left as-is).""" + if get_origin(anno) is Union: + args = [a for a in get_args(anno) if a is not type(None)] + if len(args) == 1: + return args[0] + return anno + + +def _kind_of(anno: Any) -> SampleKind: + """The carrier kind an annotation names; unknown/absent/Any -> ``any``.""" + anno = _unwrap_optional(anno) + if anno is inspect.Parameter.empty or anno is Any or anno is None: + return "any" + if anno is Sample: + return "sample" + if anno is tuple or get_origin(anno) is tuple: + return "pair" + if isinstance(anno, type) and issubclass(anno, Sample): + return "sample" + return "any" + + +def _return_contract(anno: Any) -> Tuple[SampleKind, bool]: + """(produced kind, expands) from a return annotation.""" + anno = _unwrap_optional(anno) + origin = get_origin(anno) + if origin in _EXPANDING_ORIGINS or (isinstance(anno, type) and anno in _EXPANDING_ORIGINS): + args = get_args(anno) + element = args[0] if args else Any + return _kind_of(element), True + return _kind_of(anno), False + + +def op_contract(op: Any) -> OpContract: + """The introspected (cached per type) carrier contract of an op. + + Explicit class attributes win: ``SAMPLE_KIND_IN`` / ``SAMPLE_KIND_OUT`` (a + :data:`SampleKind` string) and ``EXPANDS`` (bool) override whatever the annotations + say — the escape hatch for callables introspection can't read. Annotation resolution + failures (lazy imports, unresolvable forward refs) degrade to ``any`` so an untyped or + exotic op behaves exactly as today. + """ + cls = type(op) + cached = _contract_cache.get(cls) + if cached is not None: + return _explicit_overrides(op, cached) + + accepts: SampleKind = "any" + produces: SampleKind = "any" + expands = False + call = getattr(cls, "__call__", None) + if call is not None: + try: + signature = inspect.signature(call) + hints = get_type_hints(call) + except Exception: # noqa: BLE001 - degrade to "any" on ANY introspection failure + signature, hints = None, {} + if signature is not None: + params = [p for name, p in signature.parameters.items() if name != "self"] + if params: + first = params[0] + accepts = _kind_of(hints.get(first.name, first.annotation)) + produces, expands = _return_contract(hints.get("return", inspect.Parameter.empty)) + + contract = OpContract(accepts=accepts, produces=produces, expands=expands) + _contract_cache[cls] = contract + return _explicit_overrides(op, contract) + + +def _explicit_overrides(op: Any, base: OpContract) -> OpContract: + """Apply the ``SAMPLE_KIND_IN``/``SAMPLE_KIND_OUT``/``EXPANDS`` class-attr escape hatches.""" + kind_in = getattr(op, "SAMPLE_KIND_IN", None) + kind_out = getattr(op, "SAMPLE_KIND_OUT", None) + expands = getattr(op, "EXPANDS", None) + if kind_in is None and kind_out is None and expands is None: + return base + return OpContract( + accepts=kind_in if kind_in in SAMPLE_KINDS else base.accepts, + produces=kind_out if kind_out in SAMPLE_KINDS else base.produces, + expands=bool(expands) if expands is not None else base.expands, + ) diff --git a/dataflux/labels.py b/sampleflux/labels.py similarity index 90% rename from dataflux/labels.py rename to sampleflux/labels.py index 853851d..57c4289 100644 --- a/dataflux/labels.py +++ b/sampleflux/labels.py @@ -1,7 +1,7 @@ """``LabelMap`` — a bidirectional class-name ↔ integer-id map. -The *fittable* companion to the config-pinned :class:`~dataflux.ops.target.EncodeTargetOp` / -:class:`~dataflux.ops.target.DecodeTargetOp`. Those ops carry an explicit ``mapping`` that is +The *fittable* companion to the config-pinned :class:`~sampleflux.ops.target.EncodeTargetOp` / +:class:`~sampleflux.ops.target.DecodeTargetOp`. Those ops carry an explicit ``mapping`` that is **pinned in config, NOT fitted** at run time, so train / eval / predict share one identical label→id ordering. :class:`LabelMap` is the piece that *produces* such a pinned mapping: @@ -9,7 +9,7 @@ (backed by scikit-learn's ``LabelEncoder``) — the one-time fit that happens at **train** time. * :meth:`LabelMap.save` / :meth:`LabelMap.load` persist it (in marainer's ``class_names.json`` format) so **eval / predict** reload the *same* mapping rather than refitting on a subset. -* :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the dataflux ops that apply it. +* :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the sampleflux ops that apply it. So fitting happens once, then the mapping is pinned/persisted — it does NOT contradict the "mapping pinned in config, not fitted" discipline of the ops; it is how the pin gets created. @@ -17,7 +17,7 @@ Zero-arg constructible (``LabelMap()`` succeeds with an empty mapping) and side-effect-free in ``__init__`` per the workspace "Lazy Initialization & Zero-Arg Construction" convention; the non-empty requirement is validated lazily in the properties, not in the constructor. scikit-learn -is imported lazily inside :meth:`fit` so importing dataflux never pulls it in. +is imported lazily inside :meth:`fit` so importing sampleflux never pulls it in. """ import json @@ -26,7 +26,7 @@ from confluid import configurable -from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp +from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp @configurable @@ -35,7 +35,7 @@ class LabelMap: Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`label_names`, builds the - :class:`~dataflux.ops.target.EncodeTargetOp` / :class:`~dataflux.ops.target.DecodeTargetOp` + :class:`~sampleflux.ops.target.EncodeTargetOp` / :class:`~sampleflux.ops.target.DecodeTargetOp` that apply it, and round-trips to disk in marainer's ``class_names.json`` format. Args: @@ -74,11 +74,11 @@ def inverse(self) -> Dict[int, str]: return {v: k for k, v in self._require().items()} def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTargetOp: - """Return an :class:`~dataflux.ops.target.EncodeTargetOp` that maps name → id via this map.""" + """Return an :class:`~sampleflux.ops.target.EncodeTargetOp` that maps name → id via this map.""" return EncodeTargetOp(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTargetOp: - """Return a :class:`~dataflux.ops.target.DecodeTargetOp` that maps id → name via this map.""" + """Return a :class:`~sampleflux.ops.target.DecodeTargetOp` that maps id → name via this map.""" return DecodeTargetOp(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) @classmethod diff --git a/sampleflux/ops/__init__.py b/sampleflux/ops/__init__.py new file mode 100644 index 0000000..ca2f11a --- /dev/null +++ b/sampleflux/ops/__init__.py @@ -0,0 +1,109 @@ +""" +SampleFlux operations. + +Submodules: + - sampleflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, + ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp, SqueezeOp, + UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, + WindowOp, SpectrumScalingOp (ndarray) + - sampleflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp, SqueezeOp, + UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, + WindowOp, SpectrumScalingOp (tensor) + - sampleflux.windows: get_window / scale_spectrum + the WindowName / + SpectrumScaling Literals — the window + unit-scaling math the FFT ops share + - sampleflux.ops.tee: Tee (fan-out branching) + - sampleflux.ops.parallel: Parallel (worker-pool sub-pipeline) + - sampleflux.ops.enable: Enable (toggle an op-list via one named CLI flag) + - sampleflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) + - sampleflux.ops.configure: ConfigureOp (per-sample parameter injection — the helios Configure pattern) + - sampleflux.ops.capture: CaptureOutputOp (record an op's @output value into metadata) + - sampleflux.ops.formula: FormulaOp (math formula over sample.input — the Math node's op form) + - sampleflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) + - sampleflux.ops.transform_chain: TransformChain (sequential op-chain grouping) + - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, Mix (per-sample Context + graph plane — the flat-list building blocks a branchy flow: document lowers to) + - sampleflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp + - sampleflux.ops.swap: SwapInputTargetOp + - sampleflux.ops.stash: StashInputOp, UnstashInputOp, StashTargetOp, UnstashTargetOp + - sampleflux.ops.target: MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp (target field) + +Flat imports default to torch variants for the data ops; flow / copy / +swap / stash / target utilities are field-agnostic. +""" + +from sampleflux.ops.capture import CaptureOutputOp +from sampleflux.ops.configure import ConfigureOp +from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use +from sampleflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp +from sampleflux.ops.enable import Enable +from sampleflux.ops.formula import FormulaOp +from sampleflux.ops.parallel import Parallel +from sampleflux.ops.random_apply import RandomApply +from sampleflux.ops.sink import SampleSinkOp +from sampleflux.ops.stash import StashInputOp, StashTargetOp, UnstashInputOp, UnstashTargetOp +from sampleflux.ops.swap import SwapInputTargetOp +from sampleflux.ops.target import ( + CocoToTorchVisionDetectionOp, + DecodeTargetOp, + EncodeTargetOp, + MasksToDetectionBoxesOp, + MetadataToTargetOp, +) +from sampleflux.ops.tee import Tee +from sampleflux.ops.torch import ( + FftShiftOp, + FourierOp, + IfftShiftOp, + InverseFourierOp, + RescaleOp, + SpectrumScalingOp, + SqueezeOp, + StandardizeOp, + ToTensorOp, + UnsqueezeOp, + WindowOp, +) +from sampleflux.ops.transform_chain import TransformChain + +__all__ = [ + "CaptureOutputOp", + "ConfigureOp", + "CopyInputOp", + "CopyMetadataOp", + "CopySampleOp", + "Apply", + "Capture", + "CopyTargetOp", + "DecodeTargetOp", + "Drop", + "Enable", + "Mix", + "Save", + "Use", + "FftShiftOp", + "FormulaOp", + "FourierOp", + "IfftShiftOp", + "InverseFourierOp", + "EncodeTargetOp", + "MetadataToTargetOp", + "CocoToTorchVisionDetectionOp", + "MasksToDetectionBoxesOp", + "Parallel", + "RandomApply", + "RescaleOp", + "SampleSinkOp", + "SpectrumScalingOp", + "SqueezeOp", + "StandardizeOp", + "StashInputOp", + "StashTargetOp", + "SwapInputTargetOp", + "Tee", + "TransformChain", + "ToTensorOp", + "UnstashInputOp", + "UnstashTargetOp", + "UnsqueezeOp", + "WindowOp", +] diff --git a/dataflux/ops/capture.py b/sampleflux/ops/capture.py similarity index 97% rename from dataflux/ops/capture.py rename to sampleflux/ops/capture.py index 614adc9..29bb7e3 100644 --- a/dataflux/ops/capture.py +++ b/sampleflux/ops/capture.py @@ -11,7 +11,7 @@ The value MUST be captured from the real run — many ``@output``\\ s are stochastic (``applied_snr_db`` is a random SNR draw) and so cannot be re-derived by re-running the op. -Modality-neutral — it threads any ``Sample`` through any op — so it lives in core dataflux +Modality-neutral — it threads any ``Sample`` through any op — so it lives in core sampleflux (``compose`` group, alongside ``ConfigureOp`` / ``FormulaOp`` / the stash family). """ @@ -20,7 +20,7 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from dataflux.sample import Sample +from sampleflux.sample import Sample _MISSING = object() @@ -42,7 +42,7 @@ class CaptureOutputOp: .. code-block:: yaml - - !class:dataflux.ops.capture.CaptureOutputOp + - !class:sampleflux.ops.capture.CaptureOutputOp op: !class:waivefront.torchsig.processing.NoiseFloorOp {} output: applied_snr_db key: __captured_snr diff --git a/dataflux/ops/configure.py b/sampleflux/ops/configure.py similarity index 93% rename from dataflux/ops/configure.py rename to sampleflux/ops/configure.py index bb6bcd6..be44321 100644 --- a/dataflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -9,7 +9,7 @@ to the ORIGINAL sample. Modality-neutral — it threads any ``Sample`` through any ops — so it lives in core -dataflux (compose group, alongside ``Tee`` / ``Enable`` / ``RandomApply``). +sampleflux (compose group, alongside ``Tee`` / ``Enable`` / ``RandomApply``). """ from typing import Any, List, Optional, cast @@ -17,7 +17,7 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from dataflux.sample import Sample +from sampleflux.sample import Sample @configurable(category="op", group="compose") @@ -38,10 +38,10 @@ class ConfigureOp: .. code-block:: yaml - - !class:dataflux.ops.configure.ConfigureOp + - !class:sampleflux.ops.configure.ConfigureOp ops: - - !class:dataflux.ops.numpy.MaxOp {} - target: !class:dataflux.ops.numpy.ThresholdOp + - !class:sampleflux.ops.numpy.MaxOp {} + target: !class:sampleflux.ops.numpy.ThresholdOp low_op: ">=" param: low_level diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py new file mode 100644 index 0000000..6affe1f --- /dev/null +++ b/sampleflux/ops/context.py @@ -0,0 +1,316 @@ +"""Context ops — move data between the per-sample :class:`~sampleflux.context.Context` and the stream. + +The six flat-list building blocks of graph-shaped pipelines: ``Save`` (fork snapshot), +``Use`` (branch start), ``Drop`` (cell hygiene), ``Apply`` (per-sample parameter from a +cell), ``Capture`` (an op's ``@output`` into a cell), and ``Mix`` (fan-in). A branchy +canvas graph or ``flow:`` document lowers to a plain sequential op list containing these +(``sampleflux.flow.to_ops``), executable by the ordinary ``Flux`` engine — and lifts back +(``from_ops``). + +Unlike the stash family these NEVER touch ``sample.metadata``: graph wiring lives on the +engine-created Context data plane, so the metadata bus stays byte-identical to a linear +run. Cells are stored by reference (ops are copy-on-write by convention); ``Use`` copies +on read unless it drops the cell — mirroring ``UnstashInputOp(copy=True, remove=True)``. +""" + +from copy import deepcopy +from typing import Any, Dict, List, Optional, cast + +from confluid import configurable, flow +from confluid.fluid import Fluid + +from sampleflux.context import require +from sampleflux.sample import Sample + +_MISSING = object() + + +def _flow_if_fluid(value: Any) -> Any: + """Materialize a still-deferred confluid marker (nested op values need per-item flow).""" + return flow(value) if isinstance(value, Fluid) else value + + +def _read_output(op: Any, name: str) -> Any: + """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. + + Mirrors ``CaptureOutputOp._read_output`` but also descends our own ``Apply.op`` slot so + ``Capture(op=Apply(op=X, …))`` reaches X's ``@output``. Returns ``_MISSING`` when absent. + """ + cur, seen = op, set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + value = getattr(cur, name, _MISSING) + if value is not _MISSING: + return value + cur = getattr(cur, "target", None) or getattr(cur, "op", None) + return _MISSING + + +def _cell_field(value: Any, field: str) -> Any: + """A cell's contribution to a Sample field: the Sample's own field, or the raw value verbatim.""" + if isinstance(value, Sample): + return getattr(value, field) + return value + + +@configurable(category="op", group="structure") +class Save: + """Snapshot the stream sample into a Context cell (pass-through). + + The sample continues down the linear stream unchanged AND becomes readable by later + ``Use`` / ``Apply`` / ``Mix`` steps — the fork point of a fan-out. Stored by + reference (readers copy); ops are copy-on-write by convention, so the snapshot stays + intact as the stream continues (insert ``CopySampleOp`` before an in-place op). + + Args: + name: Context cell to store the sample under; required at call time, validated lazily. + """ + + def __init__(self, name: str = "") -> None: + # Lazy / zero-arg: store config only; the cell name is validated at first call. + self.name = str(name) + + def __call__(self, sample: Sample) -> Sample: + if not self.name: + raise ValueError("Save: 'name' (the context cell to write) is required") + require("Save").put(self.name, sample) + return sample + + +@configurable(category="op", group="structure") +class Use: + """Replace the stream sample with a Context cell's value (a branch start). + + The incoming sample is discarded; the cell's value becomes the stream sample + (``Sample.from_any`` coerces a raw cell value). Reads a DEEP COPY so two branches + reading one fork stay independent — unless ``drop`` frees the cell, which skips the + copy (move semantics, the right choice for a cell's LAST reader). + + Args: + name: Context cell to read; required at call time, validated lazily. + drop: When True, free the cell after reading and skip the defensive copy (move semantics). + """ + + def __init__(self, name: str = "", drop: bool = False) -> None: + # Lazy / zero-arg: store config only; the cell name is validated at first call. + self.name = str(name) + self.drop = bool(drop) + + def __call__(self, sample: Sample) -> Sample: + if not self.name: + raise ValueError("Use: 'name' (the context cell to read) is required") + ctx = require("Use") + value = ctx.get(self.name) + if self.drop: + ctx.delete(self.name) + else: + value = deepcopy(value) + return Sample.from_any(value) + + +@configurable(category="op", group="structure") +class Drop: + """Free Context cells (pass-through) — the explicit liveness hygiene step. + + Deleting a missing cell raises: in a compiled graph that means the liveness pass and + the op order disagree, which should fail loudly rather than leak. + + Args: + names: Context cells to delete after this point; an empty list (default) is a no-op. + """ + + def __init__(self, names: Optional[List[str]] = None) -> None: + # Lazy / zero-arg: store config only. + self.names = list(names) if names else [] + + def __call__(self, sample: Sample) -> Sample: + if self.names: + ctx = require("Drop") + for name in self.names: + ctx.delete(name) + return sample + + +@configurable(category="op", group="structure") +class Apply: + """Set a wrapped op's parameter from a Context cell, then apply the op. + + The declarative per-sample-parameter step (``ConfigureOp`` with the value coming from + a cell instead of an inline compute chain): the cell holds a prior branch's result — + a Sample cell contributes its ``input``, a raw cell value (e.g. a ``Capture``\\ d + ``@output``) is used as-is. The value is ``setattr``'d as ``param`` on ``op`` + post-construction (the confluid paradigm), then ``op`` runs on the incoming sample. + + Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call + (like ``ConfigureOp``), so an ``Apply()`` built from YAML costs nothing. + + Args: + op: The op to configure and apply; required at call time, validated lazily. + param: Attribute name on ``op`` to set with the cell value; required at call time. + source: Context cell holding the value; required at call time, validated lazily. + drop: When True, free the source cell after reading it. + """ + + def __init__( + self, + op: Optional[object] = None, + param: str = "", + source: str = "", + drop: bool = False, + ) -> None: + # Lazy / zero-arg: store config only; op/param/source are validated at first call. + self.op = op + self.param = str(param) + self.source = str(source) + self.drop = bool(drop) + + def __call__(self, sample: Sample) -> Optional[Sample]: + if self.op is None: + raise ValueError("Apply: an 'op' to configure and apply is required") + if not self.param: + raise ValueError("Apply: 'param' (the op attribute to set) is required") + if not self.source: + raise ValueError("Apply: 'source' (the context cell holding the value) is required") + self.op = _flow_if_fluid(self.op) + ctx = require("Apply") + value = ctx.get(self.source) + if self.drop: + ctx.delete(self.source) + value = _cell_field(value, "input") + op = cast(Any, self.op) + setattr(op, self.param, value) + return cast(Optional[Sample], op(sample)) + + def close(self) -> None: + """Propagate close() to the wrapped op if it owns resources.""" + close_fn = getattr(self.op, "close", None) + if callable(close_fn): + close_fn() + + +@configurable(category="op", group="structure") +class Capture: + """Apply an op, then record its ``@output`` attribute(s) into Context cells. + + The Context twin of ``CaptureOutputOp``: the wrapped op runs once (stochastic-correct + — the value is read from the actual run, never recomputed) and each requested + ``@output`` is stored as a raw cell value for a later ``Apply``/``Mix`` to read. The + returned sample is ``op(sample)`` — transformations are kept. + + Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call, + so a ``Capture()`` built from YAML costs nothing. + + Args: + op: The op to apply; its ``@output`` attributes are read after it runs. Required at call time. + output: A single ``@output`` attribute name to capture. Blank = capture only the ``captures`` entries. + name: Context cell for the ``output`` value. Blank (default) = the ``output`` name itself. + captures: Mapping of ``@output`` attribute name -> context cell, for capturing several outputs in one apply. + """ + + def __init__( + self, + op: Optional[object] = None, + output: str = "", + name: str = "", + captures: Optional[Dict[str, str]] = None, + ) -> None: + # Lazy / zero-arg: store config only; op/outputs are validated at first call. + self.op = op + self.output = str(output) + self.name = str(name) + self.captures = dict(captures) if captures else {} + + def _items(self) -> Dict[str, str]: + """The full ``{output_name: cell_name}`` map — ``captures`` plus the single-output form.""" + items = dict(self.captures) + if self.output: + items.setdefault(self.output, self.name or self.output) + return items + + def __call__(self, sample: Sample) -> Optional[Sample]: + if self.op is None: + raise ValueError("Capture: an 'op' to apply is required") + items = self._items() + if not items: + raise ValueError("Capture: nothing to capture — set 'output' (and 'name') or 'captures'") + self.op = _flow_if_fluid(self.op) + ctx = require("Capture") + op = cast(Any, self.op) + result = op(sample) + if result is None: + return None # the wrapped op filtered the sample (FilterOp semantics) + for attr, cell in items.items(): + value = _read_output(op, attr) + if value is _MISSING: + raise AttributeError(f"Capture: {type(op).__name__!r} has no @output attribute {attr!r} to capture") + ctx.put(cell, value) + return cast(Optional[Sample], result) + + def close(self) -> None: + """Propagate close() to the wrapped op if it owns resources.""" + close_fn = getattr(self.op, "close", None) + if callable(close_fn): + close_fn() + + +@configurable(category="op", group="structure") +class Mix: + """Fan-in: compose one sample from Context cells and the incoming stream sample. + + Each named slot reads its cell — a Sample cell contributes its corresponding field, a + raw cell value is used verbatim — while an unnamed slot keeps the incoming sample's + field. Metadata merges incoming-first, then each named cell's metadata in slot order + (``input_from``, ``target_from``, ``metadata_from`` — last write wins), so branch + traceability survives the merge and ``metadata_from`` has the final say. + + Args: + input_from: Context cell providing the mixed ``input``. Blank (default) = keep the incoming input. + target_from: Context cell providing the mixed ``target``. Blank (default) = keep the incoming target. + metadata_from: Context cell whose metadata merges LAST (wins conflicts). Blank (default) = none. + drop: Context cells to free after mixing (defaults to none). + """ + + def __init__( + self, + input_from: str = "", + target_from: str = "", + metadata_from: str = "", + drop: Optional[List[str]] = None, + ) -> None: + # Lazy / zero-arg: store config only; cell names are resolved at first call. + self.input_from = str(input_from) + self.target_from = str(target_from) + self.metadata_from = str(metadata_from) + self.drop = list(drop) if drop else [] + + def __call__(self, sample: Sample) -> Sample: + ctx = require("Mix") + + mixed_input = sample.input + mixed_target = sample.target + metadata: Dict[str, Any] = dict(sample.meta) + + for cell_name, field in ((self.input_from, "input"), (self.target_from, "target")): + if not cell_name: + continue + value = ctx.get(cell_name) + if field == "input": + mixed_input = _cell_field(value, "input") + else: + mixed_target = _cell_field(value, "target") + if isinstance(value, Sample): + metadata.update(value.meta) + if self.metadata_from: + value = ctx.get(self.metadata_from) + extra = value.meta if isinstance(value, Sample) else value + if not isinstance(extra, dict): + raise TypeError( + f"Mix: metadata_from cell {self.metadata_from!r} holds {type(extra).__name__}, " + "expected a Sample or a dict" + ) + metadata.update(extra) + + for cell_name in self.drop: + ctx.delete(cell_name) + + return Sample(input=mixed_input, target=mixed_target, metadata=metadata) diff --git a/dataflux/ops/copy.py b/sampleflux/ops/copy.py similarity index 97% rename from dataflux/ops/copy.py rename to sampleflux/ops/copy.py index d5944ca..197c6e5 100644 --- a/dataflux/ops/copy.py +++ b/sampleflux/ops/copy.py @@ -11,7 +11,7 @@ from confluid import configurable -from dataflux.sample import Sample +from sampleflux.sample import Sample @configurable(category="op", group="structure") diff --git a/dataflux/ops/debug.py b/sampleflux/ops/debug.py similarity index 99% rename from dataflux/ops/debug.py rename to sampleflux/ops/debug.py index 0bb6a73..5f341ce 100644 --- a/dataflux/ops/debug.py +++ b/sampleflux/ops/debug.py @@ -5,7 +5,7 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample +from sampleflux.sample import Sample logger = get_logger(__name__) diff --git a/dataflux/ops/enable.py b/sampleflux/ops/enable.py similarity index 94% rename from dataflux/ops/enable.py rename to sampleflux/ops/enable.py index dbdbef2..f532686 100644 --- a/dataflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -3,7 +3,7 @@ A compose-group op (alongside ``Tee`` / ``Parallel``): wrap an inner op-list so the whole chain can be switched on or off from one boolean attribute whose name becomes the CLI flag. Modality-neutral — it threads any ``Sample`` -through any ops — so it lives in core dataflux, not a domain package. +through any ops — so it lives in core sampleflux, not a domain package. """ from typing import List, Optional, Tuple @@ -11,7 +11,7 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample +from sampleflux.sample import Sample logger = get_logger(__name__) @@ -36,10 +36,10 @@ class Enable: .. code-block:: yaml - - !class:dataflux.ops.enable.Enable + - !class:sampleflux.ops.enable.Enable visualize: false # ← any boolean attribute name works; this name IS the CLI flag ops: - - !class:dataflux.ops.image.ConvertToImageOp {} + - !class:sampleflux.ops.image.ConvertToImageOp {} - !class:waivefront.visualizers.SaveImageOp output_dir: ./segments_png @@ -64,11 +64,11 @@ class Enable: .. code-block:: yaml - - !class:dataflux.ops.enable.Enable + - !class:sampleflux.ops.enable.Enable name: overlay # dotted-override key visualize: false # same attr name is fine — name scopes it ops: [render-with-overlays, save-to ./debug_png] - - !class:dataflux.ops.enable.Enable + - !class:sampleflux.ops.enable.Enable name: labelstudio visualize: false ops: [render-clean, save-to ./ls_png] @@ -88,7 +88,7 @@ class Enable: Constraints: * ``ops`` is required and must be a non-empty list — validated **lazily** - on first call (zero-arg construction stays valid per the dataflux + on first call (zero-arg construction stays valid per the sampleflux "Lazy Initialization & Zero-Arg Construction" convention). * Exactly one boolean attribute (other than ``ops`` / ``name`` and dunders) may be set on the wrapper — that's the toggle. diff --git a/dataflux/ops/formula.py b/sampleflux/ops/formula.py similarity index 98% rename from dataflux/ops/formula.py rename to sampleflux/ops/formula.py index 7fa535d..8463e86 100644 --- a/dataflux/ops/formula.py +++ b/sampleflux/ops/formula.py @@ -14,7 +14,7 @@ from confluid import configurable -from dataflux.sample import Sample +from sampleflux.sample import Sample # Every public ``math`` symbol + the scalar built-in helpers, mirroring the canvas Math # node's namespace. The bound variable shadows same-named constants (e.g. ``e``). diff --git a/dataflux/ops/image.py b/sampleflux/ops/image.py similarity index 98% rename from dataflux/ops/image.py rename to sampleflux/ops/image.py index cc8b5ba..393e943 100644 --- a/dataflux/ops/image.py +++ b/sampleflux/ops/image.py @@ -1,9 +1,9 @@ -"""Generic, modality-agnostic image conversion for DataFlux pipelines. +"""Generic, modality-agnostic image conversion for SampleFlux pipelines. This is the single home for "turn an arbitrary value into an image": the :class:`ConvertToImageOp` op plus the library functions (:func:`value_to_image` / :func:`sample_to_image`) that back it and FluxStudio's -sample preview. It lives in dataflux (not waivefront) because the conversion is +sample preview. It lives in sampleflux (not waivefront) because the conversion is fully generic — a 2-D map, a CHW tensor, a PIL image, a boolean mask all render the same way regardless of domain — so every project (waivefront's spectrogram render, any image dataset preview, FluxStudio nodes) reuses ONE implementation. @@ -13,7 +13,7 @@ op produces, and ``RenderSignalPlotOp`` builds IQ time/freq/constellation panels. Those need signal semantics; this op does not. -PIL is a hard dependency here (already used by ``dataflux.typespec``). Matplotlib +PIL is a hard dependency here (already used by ``sampleflux.typespec``). Matplotlib is imported lazily inside :func:`_apply_colormap` — only non-``"gray"`` colormaps need it, so the pure-greyscale path stays matplotlib-free. """ @@ -26,11 +26,11 @@ from loggair import get_logger from PIL import Image, ImageDraw -from dataflux.sample import Sample -from dataflux.typespec import ArrayType as _ArrayType -from dataflux.typespec import PythonType, SampleType, UnionType +from sampleflux.sample import Sample +from sampleflux.typespec import ArrayType as _ArrayType +from sampleflux.typespec import PythonType, SampleType, UnionType -logger = get_logger("dataflux.ops.image") +logger = get_logger("sampleflux.ops.image") # Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob @@ -147,7 +147,7 @@ def _bound_longest_side(rgb: np.ndarray, max_size: int) -> np.ndarray: def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: """Render an arbitrary value (a Sample's ``input`` OR ``target``) to an ``(H, W, 3)`` uint8 RGB image. - A generic, modality-agnostic preview usable from any DataFlux pipeline (and + A generic, modality-agnostic preview usable from any SampleFlux pipeline (and by FluxStudio's sample extractor, which renders the selected field). Handles: * ``PIL.Image`` — converted to RGB; @@ -175,7 +175,7 @@ def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: in Thin wrapper over :func:`value_to_image` (which does the modality-agnostic rendering) applied to ``sample.input``. Kept as the canonical "preview a - sample" entry point for DataFlux pipelines; use :func:`value_to_image` + sample" entry point for SampleFlux pipelines; use :func:`value_to_image` directly to render an arbitrary value such as ``sample.target``. Args: @@ -195,7 +195,7 @@ def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: in # values. Pure functions (NOT @configurable ops): they measure/derive, they don't # transform a Sample, so they're library helpers like value_to_image — not canvas # nodes. They live here (not in the FluxStudio node) so the computation is reusable -# and unit-tested, per the workspace "rendering/analysis lives in dataflux" mandate. +# and unit-tested, per the workspace "rendering/analysis lives in sampleflux" mandate. # --------------------------------------------------------------------------- # diff --git a/dataflux/ops/metadata.py b/sampleflux/ops/metadata.py similarity index 98% rename from dataflux/ops/metadata.py rename to sampleflux/ops/metadata.py index db96916..48f1253 100644 --- a/dataflux/ops/metadata.py +++ b/sampleflux/ops/metadata.py @@ -5,7 +5,7 @@ from confluid import configurable -from dataflux.sample import Sample +from sampleflux.sample import Sample def _matches_any(key: str, patterns: Tuple[str, ...]) -> bool: diff --git a/dataflux/ops/numpy.py b/sampleflux/ops/numpy.py similarity index 98% rename from dataflux/ops/numpy.py rename to sampleflux/ops/numpy.py index b97a54c..a4293f5 100644 --- a/dataflux/ops/numpy.py +++ b/sampleflux/ops/numpy.py @@ -7,9 +7,9 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample -from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType -from dataflux.windows import ( +from sampleflux.sample import Sample +from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType +from sampleflux.windows import ( WINDOW_SUM_KEY, WINDOW_SUMSQ_KEY, SpectrumScaling, @@ -452,9 +452,9 @@ def connected_component_bboxes( Components smaller than ``min_area_bins`` are dropped. ``connectivity`` is ``4`` (orthogonal neighbors) or ``8`` (orthogonal + diagonal). This is the shared scipy core behind :class:`ConnectedComponentsOp` (signal-domain bin bboxes on ``input``) - AND :class:`dataflux.ops.target.MasksToDetectionBoxesOp` (its ``connected=True`` + AND :class:`sampleflux.ops.target.MasksToDetectionBoxesOp` (its ``connected=True`` mode, which lifts the tuples to xyxy-pixel detection boxes). Requires ``scipy`` - (``pip install data-flux[vision]``). + (``pip install sampleflux[vision]``). """ if min_area_bins < 1: raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") @@ -465,7 +465,7 @@ def connected_component_bboxes( except ImportError as exc: raise ImportError( "connected-components labeling requires scipy. " - "Install with `pip install data-flux[vision]` or add scipy to your environment." + "Install with `pip install sampleflux[vision]` or add scipy to your environment." ) from exc structure = generate_binary_structure(2, 1 if connectivity == 4 else 2) @@ -643,7 +643,7 @@ class ConnectedComponentsOp: * ``8`` — orthogonal + diagonal neighbors; diagonally touching components merge. - Requires ``scipy`` (install via ``pip install data-flux[vision]``). + Requires ``scipy`` (install via ``pip install sampleflux[vision]``). Args: min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). @@ -664,7 +664,7 @@ def __call__(self, sample: Sample) -> Sample: raise TypeError(f"ConnectedComponentsOp expects an np.ndarray on sample.input, got {type(mask).__name__}") if mask.ndim != 2: raise ValueError(f"ConnectedComponentsOp expects a 2-D mask; got shape {mask.shape}") - # Shared scipy core (also used by dataflux.ops.target.MasksToDetectionBoxesOp); + # Shared scipy core (also used by sampleflux.ops.target.MasksToDetectionBoxesOp); # validates min_area_bins / connectivity and raises the scipy ImportError. bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) return sample._replace(input=bboxes) @@ -711,7 +711,7 @@ class FourierOp: of signals transforms per row. :class:`InverseFourierOp` is the inverse; set ``shift=True`` to center the zero-frequency bin (the standalone :class:`FftShiftOp` does the same independently). - **Windowing & units.** ``window`` applies a :func:`dataflux.windows.get_window` taper before the + **Windowing & units.** ``window`` applies a :func:`sampleflux.windows.get_window` taper before the transform (default ``"boxcar"`` = no taper = unchanged behaviour) and stashes the window correction into the metadata; ``scaling`` then returns the spectrum in real units — ``"amplitude"`` (V), ``"power"`` (V²) or ``"density"`` (V²/Hz, using ``sample_rate``) — dividing @@ -891,7 +891,7 @@ def __call__(self, sample: Sample) -> Sample: class WindowOp: """Apply a window taper to ``sample.input`` and record the unit-scaling correction. - Multiplies the signal by a :func:`dataflux.windows.get_window` taper (broadcast along ``axis``) + Multiplies the signal by a :func:`sampleflux.windows.get_window` taper (broadcast along ``axis``) — the standard first step of spectral analysis, controlling FFT spectral leakage — and stashes the window's correction factors into ``sample.metadata`` (``window`` / ``window_sum`` ``S1`` / ``window_sum_sq`` ``S2`` / ``window_enbw_bins`` / ``window_coherent_gain``) so a later diff --git a/dataflux/ops/parallel.py b/sampleflux/ops/parallel.py similarity index 95% rename from dataflux/ops/parallel.py rename to sampleflux/ops/parallel.py index e69caf3..a42a2bf 100644 --- a/dataflux/ops/parallel.py +++ b/sampleflux/ops/parallel.py @@ -1,6 +1,6 @@ """``Parallel`` — explicit parallel sub-pipeline op. -Place inside a :class:`~dataflux.core.Flux`'s ops list to dispatch each +Place inside a :class:`~sampleflux.core.Flux`'s ops list to dispatch each upstream sample through an inner sub-pipeline (``self.ops``) in a spawn-context worker pool. Bounded prefetch caps outstanding work so the executor queue can't grow unboundedly with source length. @@ -25,8 +25,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from dataflux.core import _worker_task -from dataflux.sample import Sample +from sampleflux.core import _worker_task +from sampleflux.sample import Sample @configurable(category="op", group="compose") diff --git a/dataflux/ops/random_apply.py b/sampleflux/ops/random_apply.py similarity index 92% rename from dataflux/ops/random_apply.py rename to sampleflux/ops/random_apply.py index df2dce4..3b23901 100644 --- a/dataflux/ops/random_apply.py +++ b/sampleflux/ops/random_apply.py @@ -5,7 +5,7 @@ the time. Samples that are skipped pass through unchanged. Modality-neutral — it threads any ``Sample`` through any op — so it lives -in core dataflux, not a domain package. +in core sampleflux, not a domain package. """ import random @@ -14,7 +14,7 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample +from sampleflux.sample import Sample logger = get_logger(__name__) @@ -34,9 +34,9 @@ class RandomApply: .. code-block:: yaml - - !class:dataflux.ops.random_apply.RandomApply + - !class:sampleflux.ops.random_apply.RandomApply probability: 0.5 - op: !class:dataflux.ops.numpy.RescaleOp + op: !class:sampleflux.ops.numpy.RescaleOp in_min: -1.0 in_max: 1.0 diff --git a/dataflux/ops/sink.py b/sampleflux/ops/sink.py similarity index 86% rename from dataflux/ops/sink.py rename to sampleflux/ops/sink.py index 711a07a..b928ce8 100644 --- a/dataflux/ops/sink.py +++ b/sampleflux/ops/sink.py @@ -1,10 +1,10 @@ -"""``SampleSinkOp`` — adapt a :class:`~dataflux.storage.base.DataSink` as a pass-through op. +"""``SampleSinkOp`` — adapt a :class:`~sampleflux.storage.base.DataSink` as a pass-through op. Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, the waivefront JSON sinks …) slot into a ``Sample``-based op chain: on first call it opens the sink, every call writes the sample and returns it unchanged, and ``close()`` flushes + closes. Modality-neutral (duck-typed ``open``/``write``/``close``), -so it lives in core dataflux. +so it lives in core sampleflux. """ from typing import Any @@ -12,14 +12,14 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample +from sampleflux.sample import Sample logger = get_logger(__name__) @configurable(category="op", group="sink") class SampleSinkOp: - """Adapter: wrap a :class:`dataflux.storage.base.DataSink` as a pass-through op. + """Adapter: wrap a :class:`sampleflux.storage.base.DataSink` as a pass-through op. Sinks (``JsonPerWindowSink``, ``JsonSink``, ``HDF5Sink`` …) implement the ``open()`` / ``write(sample)`` / ``close()`` protocol and are normally @@ -34,12 +34,12 @@ class SampleSinkOp: On the first call the adapter calls ``sink.open()`` (when present); each subsequent call forwards the Sample to ``sink.write(sample)`` and returns the Sample unchanged. ``close()`` flushes (when present) and closes the - underlying sink — propagated by :class:`dataflux.ops.enable.Enable` and + underlying sink — propagated by :class:`sampleflux.ops.enable.Enable` and :class:`waivefront.sinks.DetectionPredictionsSink` at end-of-run. YAML:: - - !class:dataflux.ops.sink.SampleSinkOp + - !class:sampleflux.ops.sink.SampleSinkOp sink: !class:waivefront.sinks.JsonPerWindowSink output_dir: ./predictions_per_window diff --git a/dataflux/ops/stash.py b/sampleflux/ops/stash.py similarity index 99% rename from dataflux/ops/stash.py rename to sampleflux/ops/stash.py index a38f74a..f30d88d 100644 --- a/dataflux/ops/stash.py +++ b/sampleflux/ops/stash.py @@ -20,7 +20,7 @@ from confluid import configurable -from dataflux.sample import Sample +from sampleflux.sample import Sample @configurable(category="op", group="structure") diff --git a/dataflux/ops/swap.py b/sampleflux/ops/swap.py similarity index 93% rename from dataflux/ops/swap.py rename to sampleflux/ops/swap.py index 9542270..69c237c 100644 --- a/dataflux/ops/swap.py +++ b/sampleflux/ops/swap.py @@ -6,7 +6,7 @@ from confluid import configurable -from dataflux.sample import Sample +from sampleflux.sample import Sample @configurable(category="op", group="structure") diff --git a/dataflux/ops/target.py b/sampleflux/ops/target.py similarity index 94% rename from dataflux/ops/target.py rename to sampleflux/ops/target.py index d4d018d..201ef9e 100644 --- a/dataflux/ops/target.py +++ b/sampleflux/ops/target.py @@ -1,8 +1,8 @@ """Move and encode the supervised ``target`` field. -Companions to the input↔metadata movers (:class:`~dataflux.ops.stash.StashInputOp` / -:class:`~dataflux.ops.stash.UnstashInputOp`) and -:class:`~dataflux.ops.swap.SwapInputTargetOp`: +Companions to the input↔metadata movers (:class:`~sampleflux.ops.stash.StashInputOp` / +:class:`~sampleflux.ops.stash.UnstashInputOp`) and +:class:`~sampleflux.ops.swap.SwapInputTargetOp`: * :class:`MetadataToTargetOp` moves a value from ``metadata`` onto ``sample.target``. * :class:`EncodeTargetOp` / :class:`DecodeTargetOp` map ``sample.target`` through an @@ -26,7 +26,7 @@ from confluid import configurable -from dataflux.sample import Sample +from sampleflux.sample import Sample #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo #: fails at the call site and UIs / form-specs enumerate the choices. @@ -38,7 +38,7 @@ def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: Shared by :class:`EncodeTargetOp` / :class:`DecodeTargetOp`. A plain module-level function (NOT a base class) so the ops stay independent - callables — DataFlux Functional Purity. + callables — SampleFlux Functional Purity. """ if value in mapping: return mapping[value] @@ -56,8 +56,8 @@ def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: class MetadataToTargetOp: """Set ``sample.target := metadata[key]``; optionally copy it to ``metadata[target_key]``. - The metadata→target counterpart of :class:`~dataflux.ops.stash.StashInputOp` / - :class:`~dataflux.ops.stash.UnstashInputOp` (which move input↔metadata). Typical + The metadata→target counterpart of :class:`~sampleflux.ops.stash.StashInputOp` / + :class:`~sampleflux.ops.stash.UnstashInputOp` (which move input↔metadata). Typical use: a raw label rides in ``metadata`` and must become the supervised ``target`` before :class:`EncodeTargetOp` overwrites it with a class id. @@ -165,8 +165,8 @@ class CocoToTorchVisionDetectionOp: The modality-neutral, image-detection counterpart of waivefront's signal-domain :class:`~waivefront.targets.RegionsToDetectionBoxesOp` (which projects time/frequency - regions) — it lives in core dataflux because the COCO→xyxy conversion is fully generic. - The input image is left untouched (tensorize it with :class:`~dataflux.ops.torch.ToTensorOp`). + regions) — it lives in core sampleflux because the COCO→xyxy conversion is fully generic. + The input image is left untouched (tensorize it with :class:`~sampleflux.ops.torch.ToTensorOp`). An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract torchvision detectors accept). @@ -238,12 +238,12 @@ class MasksToDetectionBoxesOp: object (box = the tight extent of ``mask == value``). Penn-Fudan's ``instance_id`` mask (pixels ``1..N``, one per pedestrian) is exactly this — exact even when objects touch. * ``connected=True`` — a **binary / semantic mask**: binarize (non-zero), then split into connected - components via :func:`dataflux.ops.numpy.connected_component_bboxes` (one box per blob). Use for + components via :func:`sampleflux.ops.numpy.connected_component_bboxes` (one box per blob). Use for a semantic mask (all objects share one value) or a model's predicted foreground mask. Every box gets class id ``label`` (one foreground class; class 0 stays background — so a 1-class dataset like Penn-Fudan derives ``num_classes = 2``). The input image is left untouched (tensorize - with :class:`~dataflux.ops.torch.ToTensorOp` ``mode="RGB"``). An empty mask yields empty ``[0,4]`` / + with :class:`~sampleflux.ops.torch.ToTensorOp` ``mode="RGB"``). An empty mask yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract torchvision detectors accept). Args: @@ -277,7 +277,7 @@ def __call__(self, sample: Sample) -> Sample: boxes: list = [] if self.connected: - from dataflux.ops.numpy import connected_component_bboxes + from sampleflux.ops.numpy import connected_component_bboxes # row/col-inclusive (r0,r1,c0,c1) → xyxy-pixel (x0,y0,x1,y1) with exclusive far edge. for r0, r1, c0, c1 in connected_component_bboxes(mask != 0, self.min_area, self.connectivity): diff --git a/dataflux/ops/tee.py b/sampleflux/ops/tee.py similarity index 98% rename from dataflux/ops/tee.py rename to sampleflux/ops/tee.py index 8ed9386..076f998 100644 --- a/dataflux/ops/tee.py +++ b/sampleflux/ops/tee.py @@ -14,7 +14,7 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from dataflux.sample import Sample +from sampleflux.sample import Sample @configurable(category="op", group="compose") diff --git a/dataflux/ops/torch.py b/sampleflux/ops/torch.py similarity index 97% rename from dataflux/ops/torch.py rename to sampleflux/ops/torch.py index f763f56..8495a16 100644 --- a/dataflux/ops/torch.py +++ b/sampleflux/ops/torch.py @@ -4,9 +4,9 @@ import torch from confluid import configurable -from dataflux.sample import Sample -from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType -from dataflux.windows import ( +from sampleflux.sample import Sample +from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType +from sampleflux.windows import ( WINDOW_SUM_KEY, WINDOW_SUMSQ_KEY, SpectrumScaling, @@ -259,7 +259,7 @@ def _scale_spectrum( one_sided: bool, dim: int, ) -> torch.Tensor: - """Torch mirror of :func:`dataflux.windows.scale_spectrum` (assumes ``norm="backward"``).""" + """Torch mirror of :func:`sampleflux.windows.scale_spectrum` (assumes ``norm="backward"``).""" if scaling == "none": out = spectrum elif scaling == "amplitude": @@ -305,7 +305,7 @@ class FourierOp: does the same independently). **Windowing & units.** Mirrors the numpy ``FourierOp``: ``window`` applies a - :func:`dataflux.windows.get_window` taper before the transform (default ``"boxcar"`` = none) and + :func:`sampleflux.windows.get_window` taper before the transform (default ``"boxcar"`` = none) and stashes the window correction; ``scaling`` returns the spectrum in real units — ``"amplitude"`` (V), ``"power"`` (V²) or ``"density"`` (V²/Hz, via ``sample_rate``). ``scaling="none"`` (default) leaves the raw complex spectrum. Calibrated ``scaling`` requires ``norm="backward"`` (any other @@ -497,8 +497,8 @@ def __call__(self, sample: Sample) -> Sample: class WindowOp: """Apply a window taper to ``sample.input`` and record the unit-scaling correction (torch mirror). - The tensor counterpart of :class:`dataflux.ops.numpy.WindowOp`: multiplies the signal by a - :func:`dataflux.windows.get_window` taper (broadcast along ``dim``) and stashes the window + The tensor counterpart of :class:`sampleflux.ops.numpy.WindowOp`: multiplies the signal by a + :func:`sampleflux.windows.get_window` taper (broadcast along ``dim``) and stashes the window correction (``window`` / ``window_sum`` ``S1`` / ``window_sum_sq`` ``S2`` / ``window_enbw_bins`` / ``window_coherent_gain``) into ``sample.metadata`` for a later :class:`SpectrumScalingOp`. dtype/device-preserving — real stays real, complex stays complex. @@ -543,7 +543,7 @@ def __call__(self, sample: Sample) -> Sample: class SpectrumScalingOp: """Scale a (complex) FFT spectrum to physical units using the window correction (torch mirror). - The tensor counterpart of :class:`dataflux.ops.numpy.SpectrumScalingOp`: amplitude (V) / power + The tensor counterpart of :class:`sampleflux.ops.numpy.SpectrumScalingOp`: amplitude (V) / power (V²) / density (V²/Hz), dividing out the window ``S1``/``S2`` read from the ``window_*`` metadata (rectangular ``S1=S2=N`` if absent). Assumes the spectrum came from the unscaled forward transform (``norm="backward"``). Output is complex for ``"none"``/``"amplitude"``, real for diff --git a/dataflux/ops/transform_chain.py b/sampleflux/ops/transform_chain.py similarity index 83% rename from dataflux/ops/transform_chain.py rename to sampleflux/ops/transform_chain.py index c177a2b..8aa0bb9 100644 --- a/dataflux/ops/transform_chain.py +++ b/sampleflux/ops/transform_chain.py @@ -2,8 +2,8 @@ A compose-group op (alongside ``Enable`` / ``Tee`` / ``Parallel``): wrap an ordered list of ``Sample → Sample`` callables so they appear as -one node in FluxStudio (dynamic ``op_0``, ``op_1``, … ``DATAFLUX_OP`` -inputs instead of N wired ``DATAFLUX_SAMPLE`` connections) and one named +one node in FluxStudio (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` +inputs instead of N wired ``SAMPLEFLUX_SAMPLE`` connections) and one named block in a Confluid YAML. Unlike ``Enable`` there is no boolean gate — the chain always fires. @@ -17,7 +17,7 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample +from sampleflux.sample import Sample logger = get_logger(__name__) @@ -27,7 +27,7 @@ class TransformChain: """Apply a fixed sequence of ops to every sample, always. Wrap a list of ops into one named unit so they appear as a single node - in FluxStudio (dynamic ``op_0``, ``op_1``, … ``DATAFLUX_OP`` inputs) + in FluxStudio (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` inputs) and one block in Confluid YAML instead of N separate connections. If any op in the chain returns ``None`` the remaining ops are skipped @@ -36,17 +36,17 @@ class TransformChain: Inner ops keep full autonomy over their own randomness; ``TransformChain`` itself is deterministic. Nest a - :class:`~dataflux.ops.random_apply.RandomApply` inside the chain to + :class:`~sampleflux.ops.random_apply.RandomApply` inside the chain to gate individual ops stochastically. YAML example:: - - !class:dataflux.ops.transform_chain.TransformChain + - !class:sampleflux.ops.transform_chain.TransformChain ops: - - !class:dataflux.ops.random_apply.RandomApply + - !class:sampleflux.ops.random_apply.RandomApply op: !class:waivefront.torchsig.processing.AWGNOp {} probability: 0.8 - - !class:dataflux.ops.torch.ToTensorOp {} + - !class:sampleflux.ops.torch.ToTensorOp {} Args: ops: Ordered list of callables ``Sample -> Optional[Sample]`` applied diff --git a/dataflux/paired.py b/sampleflux/paired.py similarity index 98% rename from dataflux/paired.py rename to sampleflux/paired.py index 8bf06a4..bbc52a7 100644 --- a/dataflux/paired.py +++ b/sampleflux/paired.py @@ -29,8 +29,8 @@ from confluid import configurable from loggair import get_logger -from dataflux.discovery import get_callable_path, resolve_callable -from dataflux.sample import Sample +from sampleflux.discovery import get_callable_path, resolve_callable +from sampleflux.sample import Sample logger = get_logger(__name__) @@ -47,7 +47,7 @@ class AnnotationStore(Protocol): """The read contract :class:`AnnotationJoinSource` needs from its annotation store: membership + lookup + key enumeration (``key -> record``). - Structural (a ``Protocol``), so it does NOT couple dataflux to annotaide: + Structural (a ``Protocol``), so it does NOT couple sampleflux to annotaide: annotaide's ``JSONFileAnnotationStore`` satisfies it — and so does a plain ``dict`` — without any import or inheritance. The write side (``save`` / ``delete``) lives in annotaide, not here. diff --git a/dataflux/projection.py b/sampleflux/projection.py similarity index 93% rename from dataflux/projection.py rename to sampleflux/projection.py index b881729..06e5631 100644 --- a/dataflux/projection.py +++ b/sampleflux/projection.py @@ -1,4 +1,4 @@ -"""Field projection for DataFlux sources — read only the input or only the target. +"""Field projection for SampleFlux sources — read only the input or only the target. Walking a source for a single field (the canonical case: counting classes from *targets*) should not pay for constructing the fields you don't need — e.g. @@ -13,21 +13,21 @@ Design notes ------------ * :class:`SupportsProjection` is a ``Protocol`` (never a base class), so it - composes with the DataFlux **Functional Purity** mandate — a source opts in by + composes with the SampleFlux **Functional Purity** mandate — a source opts in by *defining* ``project``, not by inheriting. * Every public function is a lazy generator (**Lazy Evaluation** mandate) — nothing materializes the whole source. * :func:`num_classes` (integer class-id semantics) is a free function, *not* a - method on the generic :class:`~dataflux.core.Flux` engine — counting classes is + method on the generic :class:`~sampleflux.core.Flux` engine — counting classes is a classification concern, and bolting it onto the task-agnostic engine would make every ``Flux`` look classification-capable to duck-typed consumers. """ from typing import Any, Collection, Iterator, Literal, Protocol, Tuple, get_args, runtime_checkable -from dataflux.sample import Sample +from sampleflux.sample import Sample -#: The projectable :class:`~dataflux.sample.Sample` fields, as a *closed* +#: The projectable :class:`~sampleflux.sample.Sample` fields, as a *closed* #: ``Literal`` rather than a bare ``str``. Typing the field set this way lets #: UIs, form-spec builders, and MCP tool schemas enumerate the allowed values #: straight from the annotation (``typing.get_args(ProjectionField)``) and lets @@ -44,7 +44,7 @@ @runtime_checkable class SupportsProjection(Protocol): - """A source that can yield partial :class:`~dataflux.sample.Sample` records. + """A source that can yield partial :class:`~sampleflux.sample.Sample` records. Implementers SHOULD avoid building unrequested fields — e.g. skip decoding the input image when only ``target`` is asked for; that efficiency is the whole diff --git a/dataflux/py.typed b/sampleflux/py.typed similarity index 100% rename from dataflux/py.typed rename to sampleflux/py.typed diff --git a/dataflux/sample.py b/sampleflux/sample.py similarity index 92% rename from dataflux/sample.py rename to sampleflux/sample.py index 9307f65..c6f3638 100644 --- a/dataflux/sample.py +++ b/sampleflux/sample.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Tuple, Union, cast if TYPE_CHECKING: # pragma: no cover - typing only - from dataflux.typespec import SampleType + from sampleflux.typespec import SampleType # Reserved metadata keys carrying a sample's stored type description (JSON strings so they survive # every storage backend's metadata round-trip — HDF5 attrs / Zarr attrs / Directory YAML / HF / Confluid). @@ -21,7 +21,7 @@ # Standardized Sample: (input, target, metadata) -# This allows DataFlux to handle complex pipelines while remaining +# This allows SampleFlux to handle complex pipelines while remaining # compatible with simple PyTorch/HF (input, target) pairs. class Sample(NamedTuple): input: Any @@ -31,6 +31,10 @@ class Sample(NamedTuple): def to_tuple(self) -> Tuple[Any, Any, Metadata]: return (self.input, self.target, self.metadata) + def to_pair(self) -> Tuple[Any, Any]: + """The metadata-free ``(input, target)`` view (the native-engine pair carrier).""" + return (self.input, self.target) + @property def is_batched(self) -> bool: """True if this Sample holds a BATCH — ``metadata`` is a ``list`` of per-item dicts (one per @@ -64,13 +68,13 @@ def batch_meta(self) -> List[Dict[str, Any]]: return self.metadata def describe(self) -> "SampleType": - """Return this sample's :class:`~dataflux.typespec.SampleType`. + """Return this sample's :class:`~sampleflux.typespec.SampleType`. Prefers the stored type (the reserved metadata keys, set explicitly via :meth:`with_type` or carried by a serialized dataset); otherwise infers it from the live ``input`` / ``target``. A batched sample carries no per-reserved-key type, so it always infers from the live data. """ - from dataflux.typespec import SampleType, infer_sample_type + from sampleflux.typespec import SampleType, infer_sample_type meta = self.metadata if isinstance(meta, dict): diff --git a/dataflux/sources.py b/sampleflux/sources.py similarity index 97% rename from dataflux/sources.py rename to sampleflux/sources.py index ca2193a..1558770 100644 --- a/dataflux/sources.py +++ b/sampleflux/sources.py @@ -5,7 +5,7 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample +from sampleflux.sample import Sample logger = get_logger(__name__) @@ -52,8 +52,8 @@ def _resolve_metadata_features( @configurable(category="source") class HuggingFaceSource: """ - DataFlux Source for Hugging Face Datasets. - Configurable mapping of dataset features to DataFlux Sample triplets. + SampleFlux Source for Hugging Face Datasets. + Configurable mapping of dataset features to SampleFlux Sample triplets. Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and @@ -199,15 +199,15 @@ class DatasetSplit: ``.test`` — and because two ``!ref:`` to the same key flow the *same* instance, the partition and the source load are shared across all three references:: - my_split: !class:dataflux.sources.DatasetSplit() + my_split: !class:sampleflux.sources.DatasetSplit() source: !ref:hf_train val_fraction: 0.1 test_fraction: 0.1 seed: 42 - train_set: !class:dataflux.core.Flux() + train_set: !class:sampleflux.core.Flux() source: !ref:my_split.train - val_set: !class:dataflux.core.Flux() + val_set: !class:sampleflux.core.Flux() source: !ref:my_split.val **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one @@ -408,10 +408,10 @@ def __len__(self) -> int: class ConcatSource: """Concatenates multiple indexable sources into one longer indexable source. - The indexable counterpart to :class:`dataflux.core.JointFlux` (which is iteration-only): + The indexable counterpart to :class:`sampleflux.core.JointFlux` (which is iteration-only): ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning sub-source, so a ``ConcatSource`` can itself be wrapped by :class:`DatasetSplit` / - :class:`RangeSource`. (Distinct from :class:`dataflux.paired.AnnotationJoinSource`, which + :class:`RangeSource`. (Distinct from :class:`sampleflux.paired.AnnotationJoinSource`, which *column-joins* annotations onto samples — this one *concatenates* sequences end to end.) Each sub-source must implement ``__len__`` and ``__getitem__``. diff --git a/dataflux/storage/base.py b/sampleflux/storage/base.py similarity index 89% rename from dataflux/storage/base.py rename to sampleflux/storage/base.py index c60b5e6..37115fd 100644 --- a/dataflux/storage/base.py +++ b/sampleflux/storage/base.py @@ -2,7 +2,7 @@ import torch -from dataflux.sample import Sample +from sampleflux.sample import Sample def to_numpy(data: Any) -> Any: @@ -18,7 +18,7 @@ def to_numpy(data: Any) -> Any: @runtime_checkable class DataSource(Protocol): - """Minimum contract for a DataFlux data source.""" + """Minimum contract for a SampleFlux data source.""" def __iter__(self) -> Iterator[Sample]: """Iterate over samples in the source.""" @@ -31,7 +31,7 @@ def __len__(self) -> int: @runtime_checkable class DataSink(Protocol): - """Minimum contract for a DataFlux data sink.""" + """Minimum contract for a SampleFlux data sink.""" def write(self, sample: Sample) -> None: """Write a single sample to the sink.""" diff --git a/dataflux/storage/cache.py b/sampleflux/storage/cache.py similarity index 100% rename from dataflux/storage/cache.py rename to sampleflux/storage/cache.py diff --git a/dataflux/storage/directory.py b/sampleflux/storage/directory.py similarity index 90% rename from dataflux/storage/directory.py rename to sampleflux/storage/directory.py index 87b98ad..9d7e00f 100644 --- a/dataflux/storage/directory.py +++ b/sampleflux/storage/directory.py @@ -4,11 +4,11 @@ import confluid import numpy as np -from dataflux.sample import Sample -from dataflux.storage.base import DataSink, Storage +from sampleflux.sample import Sample +from sampleflux.storage.base import DataSink, Storage -# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). +# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). @confluid.configurable(category="sink") class DirectorySink(Storage, DataSink): """ diff --git a/dataflux/storage/hdf5.py b/sampleflux/storage/hdf5.py similarity index 88% rename from dataflux/storage/hdf5.py rename to sampleflux/storage/hdf5.py index 0ac3ed6..02984a5 100644 --- a/dataflux/storage/hdf5.py +++ b/sampleflux/storage/hdf5.py @@ -7,10 +7,10 @@ from confluid import configurable from loggair import get_logger -from dataflux.sample import Sample -from dataflux.storage.base import DataSink, DataSource, Storage, to_numpy +from sampleflux.sample import Sample +from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy -logger = get_logger("dataflux.storage.hdf5") +logger = get_logger("sampleflux.storage.hdf5") @configurable @@ -66,8 +66,18 @@ def __len__(self) -> int: return 0 return len([k for k in self._file.keys() if k.endswith("_data")]) + def iter_metadata(self) -> "Iterator[tuple[str, dict]]": + """(prefix, metadata) per sample WITHOUT loading data arrays (SupportsMetadataScan). -# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). + Array-valued metadata appears as shape/dtype stub strings — see + :func:`sampleflux.storage.query.scan_hdf5_metadata`. + """ + from sampleflux.storage.query import scan_hdf5_metadata + + yield from scan_hdf5_metadata(self.path) + + +# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). @configurable(category="sink") class HDF5Sink(Storage, DataSink): """High-performance HDF5 data sink focused on Sample triplets.""" diff --git a/sampleflux/storage/query.py b/sampleflux/storage/query.py new file mode 100644 index 0000000..f21c5af --- /dev/null +++ b/sampleflux/storage/query.py @@ -0,0 +1,172 @@ +"""Queryable metadata — filter stored samples by metadata predicates WITHOUT loading arrays. + +Two pieces (mirroring the ``sampleflux.projection`` protocol-plus-fallback design): + +- :class:`SupportsMetadataScan` — a source opts in by implementing + ``iter_metadata() -> Iterator[(key, metadata_dict)]`` that reads ONLY the metadata + (HDF5 attrs, Zarr ``.zattrs``, a SigMF ``.sigmf-meta`` JSON) — never the data arrays. + Free-function scanners for the shipped sources live here (``scan_hdf5_metadata`` / + ``scan_zarr_metadata``); ``SigMFSource.iter_metadata`` implements the protocol + directly. +- :class:`MetadataFilterSource` — a view source (``category="source"``) yielding only + the samples whose metadata passes a predicate: the YAML-friendly ``where`` expression + (the same restricted-eval namespace as ``FormulaOp`` — metadata keys become variables) + and/or a programmatic ``predicate`` callable. The matching index set is computed + lazily from the metadata scan (cached), so arrays load only for matches; a source + without the protocol falls back to a full iteration filter. + +Existing HDF5/Zarr files are queryable with NO rewrite — their metadata already lives in +attrs/``.zattrs``. (A ``.metaindex`` sidecar accelerator is a TASKS.md follow-up if +scans ever become hot.) +""" + +from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, cast, runtime_checkable + +import h5py +from confluid import configurable +from loggair import get_logger + +from sampleflux.ops.formula import _FORMULA_NAMESPACE +from sampleflux.sample import Sample + +logger = get_logger("sampleflux.storage.query") + +__all__ = ["MetadataFilterSource", "SupportsMetadataScan", "scan_hdf5_metadata", "scan_zarr_metadata"] + + +@runtime_checkable +class SupportsMetadataScan(Protocol): + """A source that can enumerate per-sample metadata WITHOUT loading data arrays.""" + + def iter_metadata(self) -> Iterator[Tuple[str, Dict[str, Any]]]: + """Yield ``(sample key, metadata dict)`` pairs, array payloads untouched.""" + ... # pragma: no cover - protocol + + +def scan_hdf5_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: + """Scan an ``HDF5Sink`` file's metadata: dataset attrs + array-metadata SHAPE/DTYPE stubs. + + Array-valued metadata (datasets under ``{prefix}_meta/``) is represented by a stub + string ``""`` — queries can test presence/shape without a + single array read. + """ + with h5py.File(str(path), "r") as handle: + prefixes = sorted(k.split("_data")[0] for k in handle.keys() if k.endswith("_data")) + for prefix in prefixes: + metadata: Dict[str, Any] = dict(handle[f"{prefix}_data"].attrs) + meta_grp = handle.get(f"{prefix}_meta") + if isinstance(meta_grp, h5py.Group): + for key, dset in meta_grp.items(): + metadata[key] = f"" + yield prefix, metadata + + +def scan_zarr_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: + """Scan a ``ZarrGroupSink`` store's metadata: each sample group's ``.zattrs``.""" + import zarr + + root = zarr.open_group(str(path), mode="r") + for name in sorted(root.group_keys()): + yield name, dict(root[name].attrs) + + +def _where_predicate(where: str) -> Callable[[Dict[str, Any]], bool]: + """Compile a ``where`` expression into a metadata predicate. + + The expression evaluates in the FormulaOp restricted namespace (``math.*`` + + ``abs``/``min``/``max``/``round``/``pow``, no builtins) with the metadata KEYS bound + as variables — e.g. ``"snr_db > 10 and drone == 'DJI'"``. A missing key (NameError) + means the sample does not match (logged at trace-equivalent debug); any other + evaluation error raises (a malformed expression must fail loudly). + """ + + def _predicate(metadata: Dict[str, Any]) -> bool: + namespace = {**_FORMULA_NAMESPACE, **metadata} + try: + return bool(eval(where, {"__builtins__": {}}, namespace)) # noqa: S307 - restricted namespace + except NameError as exc: + logger.debug(f"MetadataFilterSource: where={where!r} — {exc}; sample treated as non-matching") + return False + except Exception as exc: + raise ValueError(f"MetadataFilterSource: where expression {where!r} failed: {exc}") from exc + + return _predicate + + +@configurable(category="source") +class MetadataFilterSource: + """A view source yielding only the samples whose metadata matches. + + Filtering uses the wrapped source's :class:`SupportsMetadataScan` protocol when + available (metadata-only scan — data arrays load ONLY for matching samples, via the + source's ``__getitem__``), else falls back to full-iteration filtering (the + projection-module pattern). Match criteria compose with AND: the ``where`` expression + and the programmatic ``predicate`` must both pass when both are set. + + Args: + source: The wrapped source; required at use time, validated lazily. + where: Restricted boolean expression over metadata keys (e.g. ``"snr_db > 10"``). Blank = no expression. + predicate: Programmatic ``metadata -> bool`` callable (not serialized; the ``FilterOp.p`` convention). + """ + + def __init__( + self, + source: Optional[Any] = None, + where: str = "", + predicate: Optional[Callable[[Dict[str, Any]], bool]] = None, + ) -> None: + # Lazy / zero-arg: store config only; matching indices compute lazily on first access. + self.source = source + self.where = str(where) + self.predicate = predicate + self._matches: Optional[List[int]] = None + + def _match(self, metadata: Dict[str, Any]) -> bool: + if self.where and not _where_predicate(self.where)(metadata): + return False + if self.predicate is not None and not self.predicate(metadata): + return False + return True + + @property + def matches(self) -> List[int]: + """Indices of matching samples (computed once per instance; ``_matches = None`` resets).""" + if self._matches is None: + if self.source is None: + raise ValueError("MetadataFilterSource: a 'source' is required") + if not self.where and self.predicate is None: + raise ValueError("MetadataFilterSource: set 'where' and/or 'predicate' — an empty filter is a bug") + if isinstance(self.source, SupportsMetadataScan): + self._matches = [i for i, (_key, meta) in enumerate(self.source.iter_metadata()) if self._match(meta)] + else: + logger.debug( + f"MetadataFilterSource: {type(self.source).__name__} has no iter_metadata — " + "falling back to full-iteration filtering (arrays load for every sample)." + ) + self._matches = [i for i, sample in enumerate(self.source) if self._match(dict(sample.meta))] + return self._matches + + def __iter__(self) -> Iterator[Sample]: + matches = self.matches # validates the source before iteration + source: Any = self.source + if hasattr(source, "__getitem__"): + for index in matches: + yield cast(Sample, source[index]) + else: + match_set = set(matches) + for i, sample in enumerate(source): + if i in match_set: + yield cast(Sample, sample) + + def __len__(self) -> int: + return len(self.matches) + + def __getitem__(self, index: int) -> Sample: + source_index = self.matches[index] + source: Any = self.source + if hasattr(source, "__getitem__"): + return cast(Sample, source[source_index]) + for i, sample in enumerate(source): + if i == source_index: + return cast(Sample, sample) + raise IndexError(index) diff --git a/sampleflux/storage/sigmf.py b/sampleflux/storage/sigmf.py new file mode 100644 index 0000000..a9d799a --- /dev/null +++ b/sampleflux/storage/sigmf.py @@ -0,0 +1,270 @@ +"""SigMF storage — one recording (``.sigmf-data`` + ``.sigmf-meta``) per sample. + +`SigMF `_ is the open Signal Metadata Format for raw recordings: a +binary sample file plus a JSON metadata file with ``global`` / ``captures`` / +``annotations`` sections. ``SigMFSink``/``SigMFSource`` are the sampleflux carrier pair +(siblings of the HDF5/Zarr pairs, additive — no migration of existing datasets): + +- the sink writes ``Sample.input`` as the raw ``.sigmf-data`` payload (``core:datatype`` + derived from the numpy dtype) and the sample metadata into the ``.sigmf-meta`` JSON; +- the source reads a directory of recordings back into Sample triplets. + +sampleflux stays domain-neutral: metadata keys are carried VERBATIM — recognised +``core:``-prefixed keys land in their SigMF section, everything else rides the +namespaced ``sampleflux:`` extension in ``global`` (SigMF explicitly supports +namespaced extensions). The waveform VOCABULARY (mapping ``samplerate`` → +``core:sample_rate``, regions → annotations, the torchsig collisions) lives in +``waivefront.vocab`` and plugs in via the ``meta_encoder``/``meta_decoder`` hooks +(dotted callable paths, lazily resolved like ``WrappedOp.f``). + +JSON is hand-rolled deliberately (the format is a stable, simple spec; no dependency to +churn). ``core:sha512`` is optional (``checksum=True``). +""" + +import hashlib +import json +from pathlib import Path +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from sampleflux.sample import Sample +from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy + +logger = get_logger("sampleflux.storage.sigmf") + +SIGMF_VERSION = "1.0.0" +_EXTENSION_PREFIX = "sampleflux:" + +# numpy dtype <-> SigMF core:datatype (little-endian; the practical interchange subset). +_DTYPE_TO_SIGMF: Dict[str, str] = { + "complex64": "cf32_le", + "complex128": "cf64_le", + "float32": "rf32_le", + "float64": "rf64_le", + "int16": "ri16_le", + "int32": "ri32_le", + "uint8": "ru8", + "int8": "ri8", +} +_SIGMF_TO_DTYPE: Dict[str, str] = {v: k for k, v in _DTYPE_TO_SIGMF.items()} + +MetaEncoder = Callable[[Dict[str, Any]], Tuple[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]]] +MetaDecoder = Callable[[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]], Dict[str, Any]] + + +def _json_safe(value: Any) -> Optional[Any]: + """``value`` if JSON-serializable (numpy scalars unwrapped), else None.""" + if isinstance(value, np.generic): + value = value.item() + try: + json.dumps(value) + except (TypeError, ValueError): + return None + return value + + +def _passthrough_encode(meta: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]]: + """The domain-neutral default encoder: core:* keys verbatim, the rest namespaced into global.""" + global_section: Dict[str, Any] = {} + for key, value in meta.items(): + safe = _json_safe(value) + if safe is None: + logger.debug(f"SigMFSink: metadata key {key!r} is not JSON-serializable — skipped") + continue + if str(key).startswith("core:"): + global_section[str(key)] = safe + else: + global_section[f"{_EXTENSION_PREFIX}{key}"] = safe + return global_section, [], [] + + +def _passthrough_decode( + global_section: Dict[str, Any], captures: List[Dict[str, Any]], annotations: List[Dict[str, Any]] +) -> Dict[str, Any]: + """Inverse of :func:`_passthrough_encode` — unwrap the namespaced keys, keep core:* verbatim.""" + meta: Dict[str, Any] = {} + for key, value in global_section.items(): + if key.startswith(_EXTENSION_PREFIX): + meta[key[len(_EXTENSION_PREFIX) :]] = value + elif key.startswith("core:") and key not in ("core:datatype", "core:version"): + meta[key] = value + if captures: + meta["core:captures"] = captures + if annotations: + meta["core:annotations"] = annotations + return meta + + +def _resolve_hook(hook: Union[str, Callable[..., Any], None], default: Callable[..., Any]) -> Callable[..., Any]: + """A dotted-path / callable / empty hook resolved lazily (the ``WrappedOp.f`` pattern). + + Accepts BOTH ``module:function`` (the discovery-native form) and the friendlier + dotted ``module.function`` (last dot promoted to the separator). + """ + if hook is None or hook == "": + return default + if callable(hook): + return hook + from sampleflux.discovery import resolve_callable + + path = str(hook) + if ":" not in path and "." in path: + module, _, attr = path.rpartition(".") + path = f"{module}:{attr}" + return resolve_callable(path) + + +# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). +@configurable(category="sink") +class SigMFSink(Storage, DataSink): + """Write each Sample as a SigMF recording pair in a directory. + + ``Sample.input`` becomes the raw ``.sigmf-data`` payload; metadata is encoded into + the ``.sigmf-meta`` JSON via ``meta_encoder`` (default: the domain-neutral + passthrough — ``core:*`` keys verbatim, others under ``sampleflux:``; wire + ``waivefront.vocab.to_sigmf`` for the waveform vocabulary). A JSON-serializable + ``Sample.target`` rides ``sampleflux:target`` (SigMF is an input-centric recording + format; array targets are skipped with a debug note). + + Args: + path: Directory the recordings are written into; required at write time, validated lazily. + prefix: Recording filename prefix; files are ``.sigmf-{data,meta}``. + meta_encoder: Dotted path or callable, metadata -> (global, captures, annotations). Blank = passthrough. + checksum: When True, write the ``core:sha512`` of the data payload into the metadata. + """ + + def __init__( + self, + path: Union[str, Path] = "", + prefix: str = "rec_", + meta_encoder: Union[str, MetaEncoder] = "", + checksum: bool = False, + ) -> None: + # Lazy / zero-arg: store config only; the directory is created lazily in open(). + self.path = Path(path) + self.prefix = str(prefix) + self.meta_encoder = meta_encoder + self.checksum = bool(checksum) + self._counter = 0 + self._opened = False + + def open(self) -> "SigMFSink": + if not self._opened: + if str(self.path) in ("", "."): + raise ValueError("SigMFSink: 'path' (the output directory) is required") + self.path.mkdir(parents=True, exist_ok=True) + self._opened = True + return self + + def close(self) -> None: + self._opened = False + + def write(self, sample: Sample) -> None: + self.open() + data = np.ascontiguousarray(to_numpy(sample.input)) + datatype = _DTYPE_TO_SIGMF.get(str(data.dtype)) + if datatype is None: + raise TypeError( + f"SigMFSink: dtype {data.dtype!s} has no SigMF core:datatype mapping " + f"(supported: {sorted(_DTYPE_TO_SIGMF)})" + ) + stem = self.path / f"{self.prefix}{self._counter:05d}" + data.tofile(stem.with_suffix(".sigmf-data")) + + encoder = _resolve_hook(self.meta_encoder, _passthrough_encode) + global_section, captures, annotations = encoder(dict(sample.meta)) + global_section = { + "core:datatype": datatype, + "core:version": SIGMF_VERSION, + **global_section, + } + if self.checksum: + global_section["core:sha512"] = hashlib.sha512(data.tobytes()).hexdigest() + target = _json_safe(sample.target) + if sample.target is not None: + if target is None: + logger.debug("SigMFSink: non-JSON-serializable target skipped (SigMF is input-centric)") + else: + global_section[f"{_EXTENSION_PREFIX}target"] = target + if not captures: + captures = [{"core:sample_start": 0}] + + meta_doc = {"global": global_section, "captures": captures, "annotations": annotations} + stem.with_suffix(".sigmf-meta").write_text(json.dumps(meta_doc, indent=2, sort_keys=True)) + self._counter += 1 + + def flush(self) -> None: + return None + + +@configurable +class SigMFSource(Storage, DataSource): + """Read a directory of SigMF recordings back into Sample triplets. + + The inverse of :class:`SigMFSink`: each ``.sigmf-meta``/``.sigmf-data`` pair yields + one Sample — the payload as ``input`` (dtype from ``core:datatype``), metadata + decoded via ``meta_decoder`` (default: the passthrough inverse; wire + ``waivefront.vocab.from_sigmf`` for the waveform vocabulary), and a stored + ``sampleflux:target`` restored to ``Sample.target``. + + Args: + path: Directory holding the recordings; required at read time, validated lazily. + meta_decoder: Dotted path or callable, (global, captures, annotations) -> metadata. Blank = passthrough. + """ + + def __init__(self, path: Union[str, Path] = "", meta_decoder: Union[str, MetaDecoder] = "") -> None: + # Lazy / zero-arg: store config only; the directory is validated on first access. + self.path = Path(path) + self.meta_decoder = meta_decoder + + def open(self) -> "SigMFSource": + return self + + def close(self) -> None: + return None + + def _meta_files(self) -> List[Path]: + if str(self.path) in ("", ".") or not self.path.is_dir(): + raise ValueError(f"SigMFSource: 'path' {str(self.path)!r} is not a directory of SigMF recordings") + return sorted(self.path.glob("*.sigmf-meta")) + + def _read(self, meta_path: Path) -> Sample: + doc = json.loads(meta_path.read_text()) + global_section: Dict[str, Any] = doc.get("global", {}) + captures: List[Dict[str, Any]] = doc.get("captures", []) + annotations: List[Dict[str, Any]] = doc.get("annotations", []) + + datatype = str(global_section.get("core:datatype", "")) + dtype = _SIGMF_TO_DTYPE.get(datatype) + if dtype is None: + raise ValueError(f"SigMFSource: {meta_path.name}: unsupported core:datatype {datatype!r}") + data = np.fromfile(meta_path.with_suffix(".sigmf-data"), dtype=np.dtype(dtype)) + + decoder = _resolve_hook(self.meta_decoder, _passthrough_decode) + target_key = f"{_EXTENSION_PREFIX}target" + target = global_section.get(target_key) + decodable = {k: v for k, v in global_section.items() if k != target_key} + metadata = decoder(decodable, captures, annotations) + return Sample(input=data, target=target, metadata=metadata) + + def __iter__(self) -> Iterator[Sample]: + for meta_path in self._meta_files(): + yield self._read(meta_path) + + def __len__(self) -> int: + return len(self._meta_files()) + + def __getitem__(self, index: int) -> Sample: + return self._read(self._meta_files()[index]) + + def iter_metadata(self) -> Iterator[Tuple[str, Dict[str, Any]]]: + """(recording stem, decoded metadata) WITHOUT loading any data payload (SupportsMetadataScan).""" + decoder = _resolve_hook(self.meta_decoder, _passthrough_decode) + target_key = f"{_EXTENSION_PREFIX}target" + for meta_path in self._meta_files(): + doc = json.loads(meta_path.read_text()) + global_section = {k: v for k, v in doc.get("global", {}).items() if k != target_key} + yield meta_path.stem, decoder(global_section, doc.get("captures", []), doc.get("annotations", [])) diff --git a/dataflux/storage/zarr.py b/sampleflux/storage/zarr.py similarity index 91% rename from dataflux/storage/zarr.py rename to sampleflux/storage/zarr.py index b4343f5..c286a52 100644 --- a/dataflux/storage/zarr.py +++ b/sampleflux/storage/zarr.py @@ -6,11 +6,11 @@ import torch import zarr -from dataflux.sample import Sample -from dataflux.storage.base import DataSink, DataSource, Storage, to_numpy +from sampleflux.sample import Sample +from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy -# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). +# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). @confluid.configurable(category="sink") class ZarrGroupSink(Storage, DataSink): """ @@ -110,8 +110,14 @@ def __len__(self) -> int: return 0 return len(list(self._root.group_keys())) + def iter_metadata(self) -> "Iterator[tuple[str, dict]]": + """(group name, ``.zattrs`` metadata) per sample WITHOUT loading arrays (SupportsMetadataScan).""" + from sampleflux.storage.query import scan_zarr_metadata -# category="sink": surfaced as a FluxStudio sink node (DATAFLUX_OBJECT:sink → DatasetProcessor.sink). + yield from scan_zarr_metadata(self.path) + + +# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). @confluid.configurable(category="sink") class ZarrBatchSink(Storage, DataSink): """ @@ -172,7 +178,7 @@ class ZarrBatchSource(Storage, DataSource): The batch sink appends every sample's input along axis 0 of a single ``data`` array and stores no per-sample target or metadata, so this source - yields input-only :class:`~dataflux.sample.Sample` objects — one per row of + yields input-only :class:`~sampleflux.sample.Sample` objects — one per row of the leading axis. Args: diff --git a/dataflux/typespec.py b/sampleflux/typespec.py similarity index 99% rename from dataflux/typespec.py rename to sampleflux/typespec.py index 5b0cb30..79b9a25 100644 --- a/dataflux/typespec.py +++ b/sampleflux/typespec.py @@ -1,11 +1,11 @@ -"""Type-spec system: describe and match the types flowing through a :class:`~dataflux.sample.Sample`. +"""Type-spec system: describe and match the types flowing through a :class:`~sampleflux.sample.Sample`. Two gaps this fills: 1. A ``Sample`` carries no description of *what kind of data* sits in its ``input`` / ``target``. 2. Ops and sources don't declare which input/target types they accept or produce. -The model is a small set of frozen value objects (no base class — see DataFlux "Functional Purity" +The model is a small set of frozen value objects (no base class — see SampleFlux "Functional Purity" mandate; these are values, not data ops) describing one slot: * :class:`AnyType` — matches everything (the default when nothing is declared). @@ -60,7 +60,7 @@ import numpy as np if TYPE_CHECKING: # pragma: no cover - typing only - from dataflux.sample import Sample + from sampleflux.sample import Sample # A single-slot type spec. Defined as a forward-ref union so leaf classes can annotate it before the # alias is bound at runtime (annotations are strings under ``from __future__ import annotations``). diff --git a/dataflux/windows.py b/sampleflux/windows.py similarity index 97% rename from dataflux/windows.py rename to sampleflux/windows.py index 8a22b3d..68eaf33 100644 --- a/dataflux/windows.py +++ b/sampleflux/windows.py @@ -3,7 +3,7 @@ A raw FFT is *uncalibrated*: to read a spectrum in real units you must (1) taper the signal with a window to control spectral leakage and (2) divide out the window's gain. This module is the single, framework-neutral (pure-numpy — scipy is only an optional -dataflux dependency) source of both: +sampleflux dependency) source of both: * :func:`get_window` builds the taper (``WindowName`` — Hann, Hamming, Blackman-Harris, flat-top, Kaiser, …). @@ -13,10 +13,10 @@ * :func:`scale_spectrum` turns a windowed FFT into the chosen ``SpectrumScaling`` units (amplitude V, power V², density V²/Hz). -It is a library module (like :mod:`dataflux.labels` / :mod:`dataflux.projection`), **not** -``@configurable`` and not entry-pointed. The numpy ops in :mod:`dataflux.ops.numpy` +It is a library module (like :mod:`sampleflux.labels` / :mod:`sampleflux.projection`), **not** +``@configurable`` and not entry-pointed. The numpy ops in :mod:`sampleflux.ops.numpy` (``WindowOp`` / ``SpectrumScalingOp`` / ``FourierOp``) and their torch mirrors in -:mod:`dataflux.ops.torch` all reuse it — the torch ops take the numpy window coefficients +:mod:`sampleflux.ops.torch` all reuse it — the torch ops take the numpy window coefficients and the scalar ``S1``/``S2`` corrections, then do the array arithmetic with torch. Calibration assumes the **unscaled forward transform** (``numpy.fft.fft`` / diff --git a/tests/test_cache.py b/tests/test_cache.py index 45ae16b..4deff70 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -8,7 +8,7 @@ import pytest -from dataflux.storage.cache import CacheBudgetExceeded, DiskCache +from sampleflux.storage.cache import CacheBudgetExceeded, DiskCache def _write(payload: bytes) -> Callable[[Path], None]: diff --git a/tests/test_categories.py b/tests/test_categories.py index 971c7f6..b588284 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -1,5 +1,5 @@ # mypy: disable-error-code="attr-defined" -"""Discovery-category coverage for dataflux ``@configurable`` classes. +"""Discovery-category coverage for sampleflux ``@configurable`` classes. These ``category=`` tags drive navigaitor's ``list_configurable_classes(category=...)`` MCP tool and, downstream, the visual-editor form-spec picker (``get_node_form_spec``). @@ -9,16 +9,16 @@ from confluid.registry import get_registry -from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp -from dataflux.ops.capture import CaptureOutputOp -from dataflux.ops.configure import ConfigureOp -from dataflux.ops.copy import CopyInputOp -from dataflux.ops.debug import PrintSampleOp -from dataflux.ops.enable import Enable -from dataflux.ops.formula import FormulaOp -from dataflux.ops.image import ConvertToImageOp, NormalizeToUint8Op -from dataflux.ops.metadata import DropMetadataOp -from dataflux.ops.numpy import ( +from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp +from sampleflux.ops.capture import CaptureOutputOp +from sampleflux.ops.configure import ConfigureOp +from sampleflux.ops.copy import CopyInputOp +from sampleflux.ops.debug import PrintSampleOp +from sampleflux.ops.enable import Enable +from sampleflux.ops.formula import FormulaOp +from sampleflux.ops.image import ConvertToImageOp, NormalizeToUint8Op +from sampleflux.ops.metadata import DropMetadataOp +from sampleflux.ops.numpy import ( FftShiftOp, FourierOp, IfftShiftOp, @@ -29,29 +29,29 @@ ThresholdOp, WindowOp, ) -from dataflux.ops.parallel import Parallel -from dataflux.ops.sink import SampleSinkOp -from dataflux.ops.stash import StashTargetOp, UnstashTargetOp -from dataflux.ops.target import ( +from sampleflux.ops.parallel import Parallel +from sampleflux.ops.sink import SampleSinkOp +from sampleflux.ops.stash import StashTargetOp, UnstashTargetOp +from sampleflux.ops.target import ( CocoToTorchVisionDetectionOp, DecodeTargetOp, EncodeTargetOp, MasksToDetectionBoxesOp, MetadataToTargetOp, ) -from dataflux.ops.tee import Tee -from dataflux.ops.torch import FftShiftOp as TorchFftShiftOp -from dataflux.ops.torch import FourierOp as TorchFourierOp -from dataflux.ops.torch import IfftShiftOp as TorchIfftShiftOp -from dataflux.ops.torch import InverseFourierOp as TorchInverseFourierOp -from dataflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp -from dataflux.ops.torch import ToTensorOp -from dataflux.ops.torch import WindowOp as TorchWindowOp -from dataflux.ops.transform_chain import TransformChain -from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource -from dataflux.storage.directory import DirectorySink -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink +from sampleflux.ops.tee import Tee +from sampleflux.ops.torch import FftShiftOp as TorchFftShiftOp +from sampleflux.ops.torch import FourierOp as TorchFourierOp +from sampleflux.ops.torch import IfftShiftOp as TorchIfftShiftOp +from sampleflux.ops.torch import InverseFourierOp as TorchInverseFourierOp +from sampleflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp +from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.torch import WindowOp as TorchWindowOp +from sampleflux.ops.transform_chain import TransformChain +from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource +from sampleflux.storage.directory import DirectorySink +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.zarr import ZarrBatchSink, ZarrGroupSink def test_engine_classes_tagged() -> None: @@ -121,8 +121,8 @@ def test_op_classes_tagged() -> None: def test_storage_sink_classes_tagged() -> None: - """The DataFlux storage SINKS carry ``category="sink"`` so FluxStudio surfaces them as - ``DatasetProcessor`` sink nodes (``DATAFLUX_OBJECT:sink``). Their matching SOURCES stay + """The SampleFlux storage SINKS carry ``category="sink"`` so FluxStudio surfaces them as + ``DatasetProcessor`` sink nodes (``SAMPLEFLUX_OBJECT:sink``). Their matching SOURCES stay UNcategorised — they read a sink's layout back via YAML ``!class:``, they are not canvas nodes. (``SampleSinkOp`` is the op-FORM sink, ``category="op"`` — a different thing, asserted above.)""" assert HDF5Sink.__confluid_category__ == "sink" @@ -134,7 +134,7 @@ def test_storage_sink_classes_tagged() -> None: def test_op_group_tags() -> None: - """Ops carry a path-like ``group`` (FluxStudio palette nesting: Taidal/DataFlux/Op/). + """Ops carry a path-like ``group`` (FluxStudio palette nesting: Taidal/SampleFlux/Op/). Presentation-only — orthogonal to the category that gates discovery. A renamed/dropped group re-files the node in the palette but never hides it; pinned so the taxonomy is a regression gate.""" diff --git a/tests/test_context.py b/tests/test_context.py new file mode 100644 index 0000000..aaf4168 --- /dev/null +++ b/tests/test_context.py @@ -0,0 +1,442 @@ +"""Tests for the per-sample Context (`sampleflux.context`) and the context ops +(`sampleflux.ops.context`: Save / Use / Drop / Apply / Capture / Mix). + +Covers the Phase-2 contract of the graph execution model: +- a flat op list containing context ops executes a fan-out/fan-in graph on the plain + Flux engine (sequential, spawn-parallel, streamed, and random-access routes); +- Context never touches ``sample.metadata`` (the metadata-untouched invariant); +- copy-vs-move semantics mirror the stash family (`Use` deep-copies unless it drops); +- every context op round-trips through confluid dump/load (Pipeline Parity). +""" + +from pathlib import Path + +import numpy as np +import pytest +from confluid import configurable, dump, load, materialize, output + +from sampleflux.context import Context, activate, current, require +from sampleflux.core import Flux +from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use +from sampleflux.ops.swap import SwapInputTargetOp +from sampleflux.sample import Sample + +# --------------------------------------------------------------------------- +# Test ops (module-level so they pickle for the spawn route) +# --------------------------------------------------------------------------- + + +@configurable +class AddOp: + """Add a constant to the (numeric or array) input. + + Args: + amount: Value added to ``sample.input`` on every call. + """ + + def __init__(self, amount: float = 1.0) -> None: + self.amount = amount + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + self.amount) + + +@configurable +class ScaleOp: + """Multiply the input by a factor. + + Args: + factor: Multiplier applied to ``sample.input`` on every call. + """ + + def __init__(self, factor: float = 2.0) -> None: + self.factor = factor + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input * self.factor) + + +@configurable +class StampOp: + """Write one metadata key (tests branch-metadata survival through Mix). + + Args: + key: Metadata key to write. + value: Value written under ``key``. + """ + + def __init__(self, key: str = "stamp", value: str = "x") -> None: + self.key = key + self.value = value + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(metadata={**sample.meta, self.key: self.value}) + + +@configurable +class DrawOp: + """Pass-through op with a stochastic-style @output (captures must read the real run).""" + + def __init__(self) -> None: + self._last: float = 0.0 + self._calls: int = 0 + + @property + @output + def drawn(self) -> float: + """The value produced by the last application.""" + return self._last + + def __call__(self, sample: Sample) -> Sample: + self._calls += 1 + self._last = float(sample.input) * 10.0 + self._calls + return sample + + +@configurable +class MutateInPlaceOp: + """Deliberately mutate the input array IN PLACE (isolation tests).""" + + def __call__(self, sample: Sample) -> Sample: + sample.input[0] = -999.0 + return sample + + +def _samples(n: int = 3) -> list: + return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] + + +# --------------------------------------------------------------------------- +# Context core +# --------------------------------------------------------------------------- + + +class TestContext: + def test_put_get_delete_live(self) -> None: + ctx = Context() + ctx.put("a", 1) + ctx.put("b", 2) + assert ctx.get("a") == 1 + assert ctx.live() == ("a", "b") + assert "a" in ctx and len(ctx) == 2 + ctx.delete("a") + assert ctx.live() == ("b",) + + def test_get_missing_is_actionable(self) -> None: + with pytest.raises(KeyError, match="live cells"): + Context().get("nope") + + def test_delete_missing_is_actionable(self) -> None: + with pytest.raises(KeyError, match="missing cell"): + Context().delete("nope") + + def test_copy_is_shallow_with_independent_cell_set(self) -> None: + ctx = Context() + payload = [1, 2] + ctx.put("a", payload) + clone = ctx.copy() + clone.delete("a") + assert "a" in ctx # independent cell set + assert ctx.get("a") is payload # shared values (shallow) + + def test_activate_sets_and_resets(self) -> None: + assert current() is None + ctx = Context() + with activate(ctx): + assert current() is ctx + assert current() is None + + def test_require_outside_engine_is_actionable(self) -> None: + with pytest.raises(RuntimeError, match="no active Context"): + require("Save") + + +# --------------------------------------------------------------------------- +# Context ops — unit behavior +# --------------------------------------------------------------------------- + + +class TestContextOps: + def test_save_requires_name(self) -> None: + with activate(Context()): + with pytest.raises(ValueError, match="'name'"): + Save()(Sample(1)) + + def test_use_requires_name(self) -> None: + with activate(Context()): + with pytest.raises(ValueError, match="'name'"): + Use()(Sample(1)) + + def test_save_then_use_copies_by_default(self) -> None: + s = Sample(input=np.array([1.0, 2.0]), metadata={"m": 1}) + with activate(Context()) as ctx: + Save(name="cell")(s) + restored = Use(name="cell")(Sample(input=None)) + assert restored.input is not s.input # deep copy + np.testing.assert_array_equal(restored.input, s.input) + assert "cell" in ctx # kept + + def test_use_with_drop_moves_without_copy(self) -> None: + s = Sample(input=np.array([1.0, 2.0])) + with activate(Context()) as ctx: + Save(name="cell")(s) + restored = Use(name="cell", drop=True)(Sample(input=None)) + assert restored.input is s.input # move: no copy + assert "cell" not in ctx # freed + + def test_two_readers_are_isolated_against_inplace_mutation(self) -> None: + s = Sample(input=np.array([1.0, 2.0])) + with activate(Context()): + Save(name="fork")(s) + branch_a = Use(name="fork")(Sample(input=None)) + MutateInPlaceOp()(branch_a) # mutates branch A's copy in place + branch_b = Use(name="fork", drop=True)(Sample(input=None)) + assert branch_b.input[0] == 1.0 # untouched by branch A + + def test_use_coerces_raw_cell_value(self) -> None: + with activate(Context()) as ctx: + ctx.put("raw", 42.0) + restored = Use(name="raw", drop=True)(Sample(input=None)) + assert restored.input == 42.0 + + def test_drop_frees_cells_and_flags_liveness_bugs(self) -> None: + with activate(Context()) as ctx: + ctx.put("a", 1) + ctx.put("b", 2) + Drop(names=["a", "b"])(Sample(1)) + assert ctx.live() == () + with pytest.raises(KeyError, match="missing cell"): + Drop(names=["a"])(Sample(1)) + + def test_drop_empty_is_noop_without_context(self) -> None: + # No names -> never needs the Context (works outside an engine too). + assert Drop()(Sample(1)).input == 1 + + def test_apply_sets_param_from_sample_cell_input(self) -> None: + with activate(Context()) as ctx: + ctx.put("thresh", Sample(input=5.0)) + op = Apply(op=AddOp(amount=0.0), param="amount", source="thresh", drop=True) + result = op(Sample(input=1.0)) + assert result is not None and result.input == 6.0 + assert "thresh" not in ctx + + def test_apply_sets_param_from_raw_cell_value(self) -> None: + with activate(Context()) as ctx: + ctx.put("factor", 3.0) + result = Apply(op=ScaleOp(), param="factor", source="factor")(Sample(input=2.0)) + assert result is not None and result.input == 6.0 + + def test_apply_validations(self) -> None: + with activate(Context()): + with pytest.raises(ValueError, match="'op'"): + Apply(param="p", source="s")(Sample(1)) + with pytest.raises(ValueError, match="'param'"): + Apply(op=AddOp(), source="s")(Sample(1)) + with pytest.raises(ValueError, match="'source'"): + Apply(op=AddOp(), param="p")(Sample(1)) + + def test_capture_records_live_output_into_cell(self) -> None: + with activate(Context()) as ctx: + draw = DrawOp() + result = Capture(op=draw, output="drawn", name="snr")(Sample(input=2.0)) + assert result is not None + assert ctx.get("snr") == 21.0 # 2*10 + 1st call — the REAL run's value + # A second application overwrites with the fresh draw (stochastic-correct). + Capture(op=draw, output="drawn", name="snr")(Sample(input=2.0)) + assert ctx.get("snr") == 22.0 + + def test_capture_reads_through_apply_wrapper(self) -> None: + with activate(Context()) as ctx: + ctx.put("noop", 0.0) + inner = DrawOp() + wrapped = Apply(op=inner, param="_unused", source="noop") + Capture(op=wrapped, output="drawn", name="d")(Sample(input=1.0)) + assert ctx.get("d") == 11.0 + + def test_capture_missing_output_is_actionable(self) -> None: + with activate(Context()): + with pytest.raises(AttributeError, match="has no @output attribute"): + Capture(op=AddOp(), output="nope")(Sample(1.0)) + + def test_capture_then_apply_wires_output_to_param(self) -> None: + with activate(Context()): + Capture(op=DrawOp(), output="drawn", name="d")(Sample(input=1.0)) + result = Apply(op=AddOp(amount=0.0), param="amount", source="d", drop=True)(Sample(input=0.5)) + assert result is not None and result.input == 0.5 + 11.0 + + def test_mix_slots_and_metadata_merge_order(self) -> None: + with activate(Context()) as ctx: + ctx.put("a", Sample(input="A", target="tA", metadata={"who": "a", "a_only": 1})) + ctx.put("b", Sample(input="B", target="tB", metadata={"who": "b", "b_only": 2})) + incoming = Sample(input="in", target="t_in", metadata={"who": "incoming", "in_only": 0}) + mixed = Mix(input_from="a", target_from="b", drop=["a", "b"])(incoming) + assert mixed is not None + assert mixed.input == "A" and mixed.target == "tB" + # incoming first, then input_from, then target_from (last write wins) + assert mixed.meta["who"] == "b" + assert mixed.meta["in_only"] == 0 and mixed.meta["a_only"] == 1 and mixed.meta["b_only"] == 2 + assert ctx.live() == () + + def test_mix_metadata_from_wins_last(self) -> None: + with activate(Context()) as ctx: + ctx.put("a", Sample(input="A", metadata={"who": "a"})) + ctx.put("m", Sample(input=None, metadata={"who": "meta"})) + mixed = Mix(input_from="a", metadata_from="m", drop=["a", "m"])(Sample(input="in", metadata={"who": "i"})) + assert mixed is not None and mixed.meta["who"] == "meta" + + def test_mix_metadata_from_accepts_raw_dict_and_rejects_nondict(self) -> None: + with activate(Context()) as ctx: + ctx.put("m", {"k": "v"}) + mixed = Mix(metadata_from="m")(Sample(input="in")) + assert mixed is not None and mixed.meta["k"] == "v" + ctx.put("bad", 3.0) + with pytest.raises(TypeError, match="metadata_from"): + Mix(metadata_from="bad")(Sample(input="in")) + + def test_mix_empty_slots_keep_incoming(self) -> None: + with activate(Context()): + incoming = Sample(input="in", target="t", metadata={"m": 1}) + mixed = Mix()(incoming) + assert mixed is not None + assert mixed.input == "in" and mixed.target == "t" and mixed.meta == {"m": 1} + + def test_ops_outside_engine_raise_actionable(self) -> None: + with pytest.raises(RuntimeError, match="no active Context"): + Save(name="x")(Sample(1)) + + +# --------------------------------------------------------------------------- +# Engine integration — the four execution routes +# --------------------------------------------------------------------------- + +# A fan-out/fan-in graph as a flat op list: +# fork the incoming value; branch A computes value+1 and swaps it into its TARGET slot +# (Mix's target_from reads the cell-sample's target field); branch B computes value*2 on +# the stream; Mix yields input = B's (stream), target = A's (cell). +_GRAPH_OPS = [ + Save(name="fork"), + AddOp(amount=1.0), # branch A rides the stream + SwapInputTargetOp(), # park A's result in the target field for Mix + Save(name="branch_a"), + Use(name="fork", drop=True), # branch B restarts from the fork + ScaleOp(factor=2.0), + Mix(target_from="branch_a", drop=["branch_a"]), # input = B (stream), target = A +] + + +def _expected_graph(values: list) -> list: + return [(v * 2.0, v + 1.0) for v in values] + + +class TestEngineRoutes: + def test_sequential_graph_execution(self) -> None: + flux = Flux(source=_samples(4), ops=list(_GRAPH_OPS)) + got = [(s.input, s.target) for s in flux] + assert _expected_graph([0.0, 1.0, 2.0, 3.0]) == got + + def test_metadata_untouched_invariant(self) -> None: + # Context wiring must not leak anything into sample.metadata. + flux = Flux(source=_samples(3), ops=list(_GRAPH_OPS)) + for i, s in enumerate(flux): + assert s.meta == {"idx": i} + + def test_random_access_getitem(self) -> None: + flux = Flux(source=_samples(5), ops=list(_GRAPH_OPS)) + s = flux[3] + assert s.input == 6.0 + + def test_spawn_parallel_parity(self) -> None: + seq = [s.input for s in Flux(source=_samples(4), ops=list(_GRAPH_OPS))] + par = [s.input for s in Flux(source=_samples(4), ops=list(_GRAPH_OPS)).parallel(2)] + assert seq == par + + def test_streamed_route_with_parallel_op(self) -> None: + from sampleflux.ops.parallel import Parallel + + # Whole graph INSIDE Parallel: each worker's _worker_task provides the Context. + flux = Flux(source=_samples(4), ops=[Parallel(ops=list(_GRAPH_OPS), workers=2)]) + got = [(s.input, s.target) for s in flux] + assert got == _expected_graph([0.0, 1.0, 2.0, 3.0]) + + def test_streamed_route_cells_may_not_cross_stream_boundary(self) -> None: + from sampleflux.ops.parallel import Parallel + + flux = Flux(source=_samples(2), ops=[Save(name="fork"), Parallel(ops=[AddOp()], workers=1)]) + with pytest.raises(RuntimeError, match="stream-level op"): + list(flux) + + def test_streamed_route_context_ops_before_and_after_boundary(self) -> None: + from sampleflux.ops.parallel import Parallel + + # Cells used and FREED before the boundary, new cells after — both legal. + ops = [ + Save(name="pre"), + Use(name="pre", drop=True), + Parallel(ops=[AddOp(amount=1.0)], workers=1), + Save(name="post"), + Mix(target_from="post", drop=["post"]), + ] + results = list(Flux(source=_samples(3), ops=ops)) + assert [s.input for s in results] == [1.0, 2.0, 3.0] + + def test_context_is_fresh_per_sample(self) -> None: + # A cell saved for sample N must never be visible to sample N+1: use a + # drop-less Save; if contexts leaked across samples, Use would see the + # PREVIOUS sample's fork (values would shift) or cells would pile up. + ops = [Save(name="fork"), AddOp(amount=100.0), Use(name="fork")] # no drop + results = list(Flux(source=_samples(3), ops=ops)) + assert [s.input for s in results] == [0.0, 1.0, 2.0] + + +# --------------------------------------------------------------------------- +# Confluid round-trip (Pipeline Parity) + YAML +# --------------------------------------------------------------------------- + + +class TestSerialization: + def test_every_context_op_dump_load_round_trips(self) -> None: + ops = [ + Save(name="fork"), + Use(name="fork", drop=True), + Drop(names=["a", "b"]), + Apply(op=AddOp(amount=2.0), param="amount", source="cell", drop=True), + Capture(op=DrawOp(), output="drawn", name="snr"), + Mix(input_from="a", target_from="b", metadata_from="m", drop=["a"]), + ] + for op in ops: + text = dump(op) + rebuilt = materialize(load(text)) + assert type(rebuilt) is type(op) + for attr, value in vars(op).items(): + if attr.startswith("_") or attr == "op": + continue # nested op compared structurally below + assert getattr(rebuilt, attr) == value, f"{type(op).__name__}.{attr}" + + rebuilt_apply = materialize(load(dump(ops[3]))) + assert type(rebuilt_apply.op).__name__ == "AddOp" and rebuilt_apply.op.amount == 2.0 + + def test_graph_ops_yaml_executes_via_from_ops_yaml(self, tmp_path: Path) -> None: + yaml_text = """ +ops: + - !class:sampleflux.ops.context.Save(name=fork) + - !class:tests.test_context.AddOp(amount=1.0) + - !class:sampleflux.ops.swap.SwapInputTargetOp() + - !class:sampleflux.ops.context.Save(name=branch_a) + - !class:sampleflux.ops.context.Use(name=fork,drop=true) + - !class:tests.test_context.ScaleOp(factor=2.0) + - !class:sampleflux.ops.context.Mix(target_from=branch_a) + drop: [branch_a] +""" + path = tmp_path / "graph_ops.yaml" + path.write_text(yaml_text) + flux = Flux.from_ops_yaml(str(path), source=_samples(3)) + results = list(flux) + assert [s.input for s in results] == [0.0, 2.0, 4.0] + assert [s.target for s in results] == [1.0, 2.0, 3.0] + + def test_linear_pipeline_metadata_byte_identical(self) -> None: + # A straight sequence (no context ops) — Context threading must be invisible. + flux = Flux(source=_samples(3), ops=[AddOp(amount=1.0)]) + for i, s in enumerate(flux): + assert s.meta == {"idx": i} + assert s.input == float(i) + 1.0 diff --git a/tests/test_coverage_gap.py b/tests/test_coverage_gap.py index 9797105..be0ff0a 100644 --- a/tests/test_coverage_gap.py +++ b/tests/test_coverage_gap.py @@ -4,13 +4,13 @@ import numpy as np import pytest -from dataflux.core import Flux -from dataflux.discovery import get_callable_path, resolve_callable -from dataflux.sample import Sample -from dataflux.storage.base import Storage -from dataflux.storage.directory import DirectorySink -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource +from sampleflux.core import Flux +from sampleflux.discovery import get_callable_path, resolve_callable +from sampleflux.sample import Sample +from sampleflux.storage.base import Storage +from sampleflux.storage.directory import DirectorySink +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource def test_storage_base_close() -> None: @@ -116,7 +116,7 @@ def test_sample_from_any_empty_tuple() -> None: def test_optional_context_manager_direct() -> None: # hits core.py:177 - from dataflux.core import Flux + from sampleflux.core import Flux f = Flux([1]) diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 7c303f6..f0d271b 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -4,7 +4,7 @@ import pytest -from dataflux.discovery import get_callable_path, introspect_callable, resolve_callable, scan_module +from sampleflux.discovery import get_callable_path, introspect_callable, resolve_callable, scan_module def sample_func(a: int, b: str = "default") -> str: @@ -45,7 +45,7 @@ def test_resolve_callable_errors() -> None: # Test AttributeError with pytest.raises(AttributeError): - resolve_callable("dataflux.discovery:nonexistent_func") + resolve_callable("sampleflux.discovery:nonexistent_func") def test_introspect_errors() -> None: @@ -160,7 +160,7 @@ class ClassInScript: assert "ClassInScript" in names # Standard module scan - schemas_self = scan_module("dataflux.discovery") + schemas_self = scan_module("sampleflux.discovery") names_self = [s["name"] for s in schemas_self] assert "scan_module" in names_self assert "get_callable_path" in names_self diff --git a/tests/test_enable.py b/tests/test_enable.py index b80a970..0c2d8f6 100644 --- a/tests/test_enable.py +++ b/tests/test_enable.py @@ -1,4 +1,4 @@ -"""Tests for :class:`dataflux.ops.enable.Enable` and :class:`dataflux.ops.sink.SampleSinkOp`. +"""Tests for :class:`sampleflux.ops.enable.Enable` and :class:`sampleflux.ops.sink.SampleSinkOp`. These modality-neutral compose helpers moved here from ``waivefront.processing`` — they thread any ``Sample`` through any ops and have no signal dependency. @@ -9,9 +9,9 @@ import confluid import pytest -from dataflux.ops.enable import Enable -from dataflux.ops.sink import SampleSinkOp -from dataflux.sample import Sample +from sampleflux.ops.enable import Enable +from sampleflux.ops.sink import SampleSinkOp +from sampleflux.sample import Sample class _CountingOp: @@ -164,10 +164,10 @@ def test_enable_yaml_load_with_cli_style_override(tmp_path: Any) -> None: """Mimic what Liquify's --visualize true override does to Fluid kwargs.""" yaml_text = """\ wrapper: - !class:dataflux.ops.enable.Enable + !class:sampleflux.ops.enable.Enable visualize: false ops: - - !class:dataflux.ops.copy.CopySampleOp {} + - !class:sampleflux.ops.copy.CopySampleOp {} """ cfg = tmp_path / "enable.yaml" cfg.write_text(yaml_text) diff --git a/tests/test_expanding_ops.py b/tests/test_expanding_ops.py new file mode 100644 index 0000000..be965f5 --- /dev/null +++ b/tests/test_expanding_ops.py @@ -0,0 +1,182 @@ +"""Tests for 1→N expanding ops and iterable-only pipeline semantics.""" + +from typing import Iterator, List, Optional + +import pytest +from confluid import configurable + +from sampleflux.core import Flux, _worker_task, _worker_task_multi +from sampleflux.kinds import op_contract +from sampleflux.ops.context import Save, Use +from sampleflux.sample import Sample + +# --------------------------------------------------------------------------- +# Fixture ops (module-level so they pickle for spawn parity) +# --------------------------------------------------------------------------- + + +@configurable +class SplitOp: + """Expand one sample into ``count`` children (index appended to metadata). + + Args: + count: Number of children yielded per incoming sample. + """ + + def __init__(self, count: int = 2) -> None: + self.count = count + + def __call__(self, sample: Sample) -> Iterator[Sample]: + for i in range(self.count): + yield sample._replace(input=sample.input * 10 + i, metadata={**sample.meta, "child": i}) + + +@configurable +class MarkedSplitOp: + """An expansion op detected via the explicit EXPANDS marker (untyped __call__).""" + + EXPANDS = True + + def __call__(self, sample): # type: ignore[no-untyped-def] + return [sample, sample] + + +@configurable +class AddOp: + """Add a constant to the input. + + Args: + amount: Value added to ``sample.input``. + """ + + def __init__(self, amount: float = 1.0) -> None: + self.amount = amount + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + self.amount) + + +@configurable +class DropOddChildOp: + """Filter inside an expansion: drop children with odd input.""" + + def __call__(self, sample: Sample) -> Optional[Sample]: + return None if int(sample.input) % 2 else sample + + +@configurable +class EmptySplitOp: + """An expanding op that yields nothing (drops the sample entirely).""" + + def __call__(self, sample: Sample) -> Iterator[Sample]: + return iter(()) + + +def _samples(n: int = 2) -> List[Sample]: + return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] + + +# --------------------------------------------------------------------------- +# Engine routes +# --------------------------------------------------------------------------- + + +class TestExpansion: + def test_sequential_expansion_depth_first_order(self) -> None: + out = list(Flux(source=_samples(2), ops=[SplitOp(count=2), AddOp(amount=0.5)])) + # sample 0 -> children 0,1 -> +0.5 ; sample 1 -> 10,11 -> +0.5 + assert [s.input for s in out] == [0.5, 1.5, 10.5, 11.5] + assert [s.meta["child"] for s in out] == [0, 1, 0, 1] + + def test_chained_expansions(self) -> None: + out = list(Flux(source=_samples(1), ops=[SplitOp(count=2), SplitOp(count=2)])) + # 0 -> [0, 1] -> [00,01,10,11] depth-first + assert [s.input for s in out] == [0.0, 1.0, 10.0, 11.0] + + def test_none_drop_inside_expansion(self) -> None: + out = list(Flux(source=_samples(1), ops=[SplitOp(count=4), DropOddChildOp()])) + assert [s.input for s in out] == [0.0, 2.0] + + def test_empty_expansion_drops_the_sample(self) -> None: + assert list(Flux(source=_samples(3), ops=[EmptySplitOp()])) == [] + + def test_marked_expansion_via_class_attr(self) -> None: + assert op_contract(MarkedSplitOp()).expands is True + out = list(Flux(source=_samples(1), ops=[MarkedSplitOp()])) + assert len(out) == 2 + + def test_spawn_parallel_parity(self) -> None: + seq = [s.input for s in Flux(source=_samples(3), ops=[SplitOp(count=2), AddOp()])] + par = [s.input for s in Flux(source=_samples(3), ops=[SplitOp(count=2), AddOp()]).parallel(2)] + assert seq == par + + def test_streamed_route_expansion(self) -> None: + from sampleflux.ops.parallel import Parallel + + ops = [SplitOp(count=2), Parallel(ops=[AddOp(amount=0.5)], workers=1)] + out = list(Flux(source=_samples(2), ops=ops)) + assert sorted(s.input for s in out) == [0.5, 1.5, 10.5, 11.5] + + def test_batch_over_expanded_stream(self) -> None: + chunks = list(Flux(source=_samples(2), ops=[SplitOp(count=3)]).batch(4)) + assert [len(c) for c in chunks] == [4, 2] + + def test_expansion_with_context_ops(self) -> None: + # A fork saved BEFORE the expansion is readable by each child (shallow ctx copy). + ops = [Save(name="fork"), SplitOp(count=2), Use(name="fork")] + out = list(Flux(source=_samples(2), ops=ops)) + # Use restores the pre-split fork for every child -> inputs are the originals. + assert [s.input for s in out] == [0.0, 0.0, 1.0, 1.0] + + +class TestIterableOnly: + def test_len_raises_with_actionable_message(self) -> None: + flux = Flux(source=_samples(3), ops=[SplitOp()]) + with pytest.raises(TypeError, match="ITERABLE-ONLY.*SplitOp|SplitOp.*ITERABLE-ONLY"): + len(flux) + + def test_getitem_raises(self) -> None: + flux = Flux(source=_samples(3), ops=[SplitOp()]) + with pytest.raises(TypeError, match="iterable-only|ITERABLE-ONLY"): + _ = flux[0] + + def test_non_expanding_pipeline_keeps_random_access(self) -> None: + flux = Flux(source=_samples(3), ops=[AddOp()]) + assert len(flux) == 3 and flux[1].input == 2.0 + + def test_strict_worker_task_rejects_expansion(self) -> None: + with pytest.raises(TypeError, match="1→N expanding"): + _worker_task(Sample(input=1.0), [SplitOp()]) + + def test_worker_task_multi_returns_all(self) -> None: + results = _worker_task_multi(Sample(input=1.0, metadata={}), [SplitOp(count=3)]) + assert [s.input for s in results] == [10.0, 11.0, 12.0] + + +class TestContextIsolationAcrossChildren: + def test_children_have_independent_cell_sets(self) -> None: + @configurable + class SaveChildIdOp: + def __call__(self, sample: Sample) -> Sample: + from sampleflux.context import require + + require("SaveChildIdOp").put("mine", sample.meta["child"]) + return sample + + @configurable + class ReadBackOp: + def __call__(self, sample: Sample) -> Sample: + from sampleflux.context import require + + return sample._replace(target=require("ReadBackOp").get("mine")) + + out = list(Flux(source=_samples(1), ops=[SplitOp(count=3), SaveChildIdOp(), ReadBackOp()])) + assert [s.target for s in out] == [0, 1, 2] # no cross-child leakage + + +def test_flowgraph_rejects_expanding_step() -> None: + from sampleflux.flow import FlowGraph + + graph = FlowGraph(source=_samples(1), flow={"split": SplitOp()}) + with pytest.raises(NotImplementedError, match="expanding"): + list(graph) diff --git a/tests/test_flow.py b/tests/test_flow.py new file mode 100644 index 0000000..e6ec335 --- /dev/null +++ b/tests/test_flow.py @@ -0,0 +1,396 @@ +"""Tests for the flow document, the FlowGraph engine, and the flow⇄ops converters. + +The load-bearing contract: **execution parity both ways** — a flow document run natively +by FlowGraph equals the same flow lowered (`to_ops`) and run by the serial Flux engine, +and a flat context-ops list run by Flux equals its lifted (`from_ops`) flow run by +FlowGraph. Round-tripping re-lowers to an execution-equivalent list. +""" + +from pathlib import Path +from typing import Callable, Optional + +import pytest +from confluid import configurable, output + +from sampleflux.context import Context, activate +from sampleflux.core import Flux +from sampleflux.flow import FlowGraph, FlowStep, from_ops, parse_flow, to_ops +from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use +from sampleflux.ops.swap import SwapInputTargetOp +from sampleflux.sample import Sample + +# --------------------------------------------------------------------------- +# Test ops (module-level so they pickle for spawn parity) +# --------------------------------------------------------------------------- + + +@configurable +class AddOp: + """Add a constant to the input. + + Args: + amount: Value added to ``sample.input``. + """ + + def __init__(self, amount: float = 1.0) -> None: + self.amount = amount + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input + self.amount) + + +@configurable +class ScaleOp: + """Multiply the input by a factor. + + Args: + factor: Multiplier applied to ``sample.input``. + """ + + def __init__(self, factor: float = 2.0) -> None: + self.factor = factor + + def __call__(self, sample: Sample) -> Sample: + return sample._replace(input=sample.input * self.factor) + + +@configurable +class TenfoldOutputOp: + """Pass-through with a DETERMINISTIC @output (parity tests need reproducibility).""" + + def __init__(self) -> None: + self._last: float = 0.0 + + @property + @output + def tenfold(self) -> float: + """Ten times the last seen input.""" + return self._last + + def __call__(self, sample: Sample) -> Sample: + self._last = float(sample.input) * 10.0 + return sample + + +@configurable +class DropOddOp: + """Filter: drop samples with odd integer input.""" + + def __call__(self, sample: Sample) -> Optional[Sample]: + return None if int(sample.input) % 2 else sample + + +def _samples(n: int = 4) -> list: + return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] + + +def _key(sample: Sample) -> tuple: + target = sample.target + if isinstance(target, Sample): + target = ("sample", target.input, target.target) + return (sample.input, target, tuple(sorted(sample.meta.items()))) + + +def _assert_parity(flow_doc: dict, outputs: str = "", n: int = 4) -> None: + """FlowGraph-native == Flux-over-lowered, sample for sample (fresh ops per engine).""" + import copy + + native = FlowGraph(source=_samples(n), flow=copy.deepcopy(flow_doc), outputs=outputs) + lowered_steps, out = parse_flow(copy.deepcopy(flow_doc), outputs) + serial = Flux(source=_samples(n), ops=to_ops(lowered_steps, out)) + got_native = [_key(s) for s in native] + got_serial = [_key(s) for s in serial] + assert got_native == got_serial, f"engine parity broken:\n native={got_native}\n serial={got_serial}" + + +# --------------------------------------------------------------------------- +# parse_flow validation +# --------------------------------------------------------------------------- + + +class TestParseFlow: + def test_linear_defaults(self) -> None: + steps, out = parse_flow({"a": AddOp(), "b": ScaleOp()}) + assert [s.name for s in steps] == ["a", "b"] + assert steps[1].from_ is None and out == "b" + + def test_forward_reference_rejected(self) -> None: + with pytest.raises(ValueError, match="EARLIER step"): + parse_flow({"a": {"op": AddOp(), "from": "b"}, "b": ScaleOp()}) + + def test_dotted_step_name_rejected(self) -> None: + with pytest.raises(ValueError, match="may not contain"): + parse_flow({"a.b": AddOp()}) + + def test_unknown_step_key_rejected(self) -> None: + with pytest.raises(ValueError, match="unknown step key"): + parse_flow({"a": AddOp(), "b": {"op": ScaleOp(), "sideways": "a"}}) + + def test_bind_requires_known_step_and_op(self) -> None: + with pytest.raises(ValueError, match="does not name an earlier step"): + parse_flow({"a": {"op": AddOp(), "bind": {"amount": "ghost"}}}) + with pytest.raises(ValueError, match="bind requires an op"): + parse_flow({"a": AddOp(), "b": {"from": "a", "bind": {"x": "a"}}}) + + def test_outputs_must_name_a_step(self) -> None: + with pytest.raises(ValueError, match="outputs"): + parse_flow({"a": AddOp()}, outputs="ghost") + + def test_reserved_ctor_param_collision_rejected(self) -> None: + @configurable + class BadOp: + """Op with a reserved-name ctor param. + + Args: + bind: Collides with the reserved flow step key. + """ + + def __init__(self, bind: str = "") -> None: + self.bind = bind + + def __call__(self, sample: Sample) -> Sample: + return sample + + with pytest.raises(ValueError, match="reserved flow step keys"): + parse_flow({"a": BadOp()}) + + def test_duplicate_step_name_rejected(self) -> None: + # dicts dedupe keys silently, so build the parsed list directly + steps = [ + FlowStep("a", AddOp(), None, None, None, {}), + FlowStep("a", ScaleOp(), None, None, None, {}), + ] + graph = FlowGraph(source=_samples(1), flow=steps) + assert graph.steps # duplicate FlowStep lists are the caller's problem; parse_flow guards dicts + + +# --------------------------------------------------------------------------- +# Engine parity (the load-bearing contract) +# --------------------------------------------------------------------------- + + +class TestEngineParity: + def test_linear(self) -> None: + _assert_parity({"a": AddOp(amount=1.0), "b": ScaleOp(factor=3.0)}) + + def test_linear_lowering_is_bare(self) -> None: + ops = to_ops({"a": AddOp(amount=1.0), "b": ScaleOp(factor=3.0)}) + assert [type(o).__name__ for o in ops] == ["AddOp", "ScaleOp"] # zero context ops + + def test_fan_out_fan_in(self) -> None: + _assert_parity( + { + "a": AddOp(amount=1.0), + "b": {"op": SwapInputTargetOp(), "from": "a"}, + "c": {"op": AddOp(amount=5.0), "from": "a"}, + "out": {"from": "c", "target_from": "b"}, + } + ) + + def test_bind_step_result(self) -> None: + _assert_parity( + { + "thresh": ScaleOp(factor=0.5), + "shifted": {"op": AddOp(), "from": "thresh", "bind": {"amount": "thresh"}}, + } + ) + + def test_bind_at_output(self) -> None: + _assert_parity( + { + "probe": TenfoldOutputOp(), + "shifted": {"op": AddOp(), "bind": {"amount": "probe.tenfold"}}, + } + ) + + def test_outputs_earlier_step(self) -> None: + _assert_parity({"a": AddOp(amount=1.0), "b": ScaleOp(factor=3.0)}, outputs="a") + + def test_identity_first_step_source_fork(self) -> None: + _assert_parity( + { + "src": {}, + "a": AddOp(amount=1.0), + "b": {"op": ScaleOp(factor=2.0), "from": "src"}, + "out": {"from": "b", "target_from": "a"}, + } + ) + + def test_filtering_drops_in_both_engines(self) -> None: + flow_doc = {"f": DropOddOp(), "a": AddOp(amount=1.0)} + _assert_parity(flow_doc) + native = FlowGraph(source=_samples(4), flow={"f": DropOddOp(), "a": AddOp(amount=1.0)}) + assert [s.input for s in native] == [1.0, 3.0] + + def test_metadata_from_slot(self) -> None: + _assert_parity( + { + "a": AddOp(amount=1.0), + "b": {"op": ScaleOp(factor=2.0), "from": "a"}, + "out": {"from": "b", "metadata_from": "a"}, + } + ) + + +class TestReverseParity: + """Flux(ops) == FlowGraph(from_ops(ops)) — lifting preserves execution.""" + + def _assert_reverse(self, ops_builder: Callable[[], list], n: int = 4) -> None: + serial = Flux(source=_samples(n), ops=ops_builder()) + flow_doc, out = from_ops(ops_builder()) + native = FlowGraph(source=_samples(n), flow=flow_doc, outputs=out) + assert [_key(s) for s in serial] == [_key(s) for s in native] + + def test_linear_list(self) -> None: + self._assert_reverse(lambda: [AddOp(amount=1.0), ScaleOp(factor=3.0)]) + + def test_hand_written_graph_list(self) -> None: + self._assert_reverse( + lambda: [ + Save(name="fork"), + AddOp(amount=1.0), + SwapInputTargetOp(), + Save(name="branch_a"), + Use(name="fork", drop=True), + ScaleOp(factor=2.0), + Mix(target_from="branch_a", drop=["branch_a"]), + ] + ) + + def test_apply_capture_list(self) -> None: + self._assert_reverse( + lambda: [ + Capture(op=TenfoldOutputOp(), output="tenfold", name="t"), + Apply(op=AddOp(), param="amount", source="t", drop=True), + ] + ) + + def test_drop_ops_vanish_from_lifted_flow(self) -> None: + flow_doc, _ = from_ops([Save(name="x"), AddOp(), Drop(names=["x"])]) + assert all("drop" not in str(v).lower() or "op" in v for v in flow_doc.values()) + + +class TestRoundTrip: + def test_flow_to_ops_to_flow_execution_equivalent(self) -> None: + original = { + "a": AddOp(amount=1.0), + "b": {"op": SwapInputTargetOp(), "from": "a"}, + "c": {"op": AddOp(amount=5.0), "from": "a"}, + "out": {"from": "c", "target_from": "b"}, + } + lowered = to_ops(dict(original)) + lifted, out = from_ops(lowered) + relowered = to_ops(lifted, out) + a = [_key(s) for s in Flux(source=_samples(4), ops=lowered)] + b = [_key(s) for s in Flux(source=_samples(4), ops=relowered)] + assert a == b + + def test_lowered_graph_leaves_context_empty(self) -> None: + ops = to_ops( + { + "a": AddOp(amount=1.0), + "b": {"op": SwapInputTargetOp(), "from": "a"}, + "out": {"from": "a", "target_from": "b"}, + } + ) + ctx = Context() + sample: Sample = Sample(input=1.0, metadata={}) + with activate(ctx): + for op in ops: + result = op(sample) + assert result is not None + sample = result + assert ctx.live() == () # automatic liveness freed every cell + + +# --------------------------------------------------------------------------- +# FlowGraph engine surface +# --------------------------------------------------------------------------- + + +class TestFlowGraphSurface: + def test_zero_arg_construction(self) -> None: + graph = FlowGraph() + with pytest.raises(ValueError, match="flow is not set"): + _ = graph.steps + + def test_len_and_getitem(self) -> None: + graph = FlowGraph(source=_samples(5), flow={"a": AddOp(amount=1.0)}) + assert len(graph) == 5 + assert graph[2].input == 3.0 + + def test_getitem_filtered_raises_indexerror(self) -> None: + graph = FlowGraph(source=_samples(4), flow={"f": DropOddOp()}) + with pytest.raises(IndexError, match="filtered"): + _ = graph[1] + + def test_batch(self) -> None: + graph = FlowGraph(source=_samples(4), flow={"a": AddOp()}).batch(3) + chunks = list(graph) + assert [len(c) for c in chunks] == [3, 1] + + def test_parallel_spawn_parity(self) -> None: + flow_doc = { + "a": AddOp(amount=1.0), + "b": {"op": SwapInputTargetOp(), "from": "a"}, + "c": {"op": AddOp(amount=5.0), "from": "a"}, + "out": {"from": "c", "target_from": "b"}, + } + seq = [_key(s) for s in FlowGraph(source=_samples(4), flow=dict(flow_doc))] + par = [_key(s) for s in FlowGraph(source=_samples(4), flow=dict(flow_doc)).parallel(2)] + assert seq == par + + def test_to_flux_twin(self) -> None: + graph = FlowGraph(source=_samples(3), flow={"a": AddOp(amount=2.0)}) + assert [s.input for s in graph.to_flux()] == [2.0, 3.0, 4.0] + + def test_collect(self) -> None: + graph = FlowGraph(source=_samples(2), flow={"a": AddOp()}) + assert len(graph.collect()) == 2 + + +# --------------------------------------------------------------------------- +# YAML round-trips +# --------------------------------------------------------------------------- + +_FLOW_YAML = """ +flow: + a: !class:tests.test_flow.AddOp(amount=1.0) + b: !class:sampleflux.ops.swap.SwapInputTargetOp() + from: a + c: !class:tests.test_flow.AddOp(amount=5.0) + from: a + out: + from: c + target_from: b +outputs: out +""" + + +class TestYaml: + def test_flowgraph_from_yaml(self, tmp_path: Path) -> None: + path = tmp_path / "graph.yaml" + path.write_text(_FLOW_YAML) + graph = FlowGraph.from_yaml(str(path), source=_samples(3)) + results = list(graph) + # v -> a=v+1 -> c=a+5=v+6 (input); target = b's target = a's swapped input = v+1 + assert [s.input for s in results] == [6.0, 7.0, 8.0] + assert [s.target for s in results] == [1.0, 2.0, 3.0] + + def test_flux_from_flow_yaml_matches_native(self, tmp_path: Path) -> None: + path = tmp_path / "graph.yaml" + path.write_text(_FLOW_YAML) + native = [_key(s) for s in FlowGraph.from_yaml(str(path), source=_samples(3))] + serial = [_key(s) for s in Flux.from_flow_yaml(str(path), source=_samples(3))] + assert native == serial + + def test_flowgraph_from_ops_yaml(self, tmp_path: Path) -> None: + ops_yaml = """ +ops: + - !class:tests.test_flow.AddOp(amount=1.0) + - !class:tests.test_flow.ScaleOp(factor=3.0) +""" + path = tmp_path / "ops.yaml" + path.write_text(ops_yaml) + graph = FlowGraph.from_ops_yaml(str(path), source=_samples(3)) + assert [s.input for s in graph] == [3.0, 6.0, 9.0] diff --git a/tests/test_flux.py b/tests/test_flux.py index 5a9f0a4..29c8420 100644 --- a/tests/test_flux.py +++ b/tests/test_flux.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from dataflux.core import Flux -from dataflux.sample import Sample +from sampleflux.core import Flux +from sampleflux.sample import Sample def test_basic_flux() -> None: @@ -71,7 +71,7 @@ def test_wrapped_op_all() -> None: def test_filter_op() -> None: - from dataflux.core import FilterOp + from sampleflux.core import FilterOp op = FilterOp(lambda s: bool(s.input > 5)) s1 = Sample(input=10) @@ -112,7 +112,7 @@ def test_wrapped_op_fallback() -> None: def test_worker_task_none() -> None: - from dataflux.core import _worker_task + from sampleflux.core import _worker_task # hits line 83 by using two ops, first returning None assert _worker_task(Sample(input=1), [lambda s: None, lambda s: s]) is None diff --git a/tests/test_fourier_ops.py b/tests/test_fourier_ops.py index 9f07cda..14faafa 100644 --- a/tests/test_fourier_ops.py +++ b/tests/test_fourier_ops.py @@ -1,5 +1,5 @@ -"""Tests for the 1-D Fourier-transform ops: ``dataflux.ops.numpy.FourierOp`` and -``dataflux.ops.torch.FourierOp``. +"""Tests for the 1-D Fourier-transform ops: ``sampleflux.ops.numpy.FourierOp`` and +``sampleflux.ops.torch.FourierOp``. Both compute the 1-D DFT (``numpy.fft.fft`` / ``torch.fft.fft``) of ``sample.input`` and ALWAYS yield a complex result — for real and complex inputs alike. The tests pin: the @@ -13,30 +13,30 @@ import torch from pydantic import ValidationError -from dataflux.ops import FftShiftOp as FlatFftShiftOp -from dataflux.ops import FourierOp as FlatFourierOp -from dataflux.ops import IfftShiftOp as FlatIfftShiftOp -from dataflux.ops import InverseFourierOp as FlatInverseFourierOp -from dataflux.ops.numpy import FftShiftOp as NpFftShiftOp -from dataflux.ops.numpy import FourierNorm -from dataflux.ops.numpy import FourierOp as NpFourierOp -from dataflux.ops.numpy import IfftShiftOp as NpIfftShiftOp -from dataflux.ops.numpy import InverseFourierOp as NpInverseFourierOp -from dataflux.ops.numpy import SpectrumScalingOp as NpSpectrumScalingOp -from dataflux.ops.numpy import WindowOp as NpWindowOp -from dataflux.ops.torch import FftShiftOp as TorchFftShiftOp -from dataflux.ops.torch import FourierOp as TorchFourierOp -from dataflux.ops.torch import IfftShiftOp as TorchIfftShiftOp -from dataflux.ops.torch import InverseFourierOp as TorchInverseFourierOp -from dataflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp -from dataflux.ops.torch import WindowOp as TorchWindowOp -from dataflux.sample import Sample -from dataflux.typespec import infer_sample_type -from dataflux.windows import WINDOW_SUM_KEY +from sampleflux.ops import FftShiftOp as FlatFftShiftOp +from sampleflux.ops import FourierOp as FlatFourierOp +from sampleflux.ops import IfftShiftOp as FlatIfftShiftOp +from sampleflux.ops import InverseFourierOp as FlatInverseFourierOp +from sampleflux.ops.numpy import FftShiftOp as NpFftShiftOp +from sampleflux.ops.numpy import FourierNorm +from sampleflux.ops.numpy import FourierOp as NpFourierOp +from sampleflux.ops.numpy import IfftShiftOp as NpIfftShiftOp +from sampleflux.ops.numpy import InverseFourierOp as NpInverseFourierOp +from sampleflux.ops.numpy import SpectrumScalingOp as NpSpectrumScalingOp +from sampleflux.ops.numpy import WindowOp as NpWindowOp +from sampleflux.ops.torch import FftShiftOp as TorchFftShiftOp +from sampleflux.ops.torch import FourierOp as TorchFourierOp +from sampleflux.ops.torch import IfftShiftOp as TorchIfftShiftOp +from sampleflux.ops.torch import InverseFourierOp as TorchInverseFourierOp +from sampleflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp +from sampleflux.ops.torch import WindowOp as TorchWindowOp +from sampleflux.sample import Sample +from sampleflux.typespec import infer_sample_type +from sampleflux.windows import WINDOW_SUM_KEY def test_flat_imports_are_torch_variants() -> None: - """``from dataflux.ops import …`` resolves the FFT ops to their torch variants — the package's + """``from sampleflux.ops import …`` resolves the FFT ops to their torch variants — the package's documented convention that flat data-op imports default to torch (mirrors RescaleOp etc.).""" assert FlatFourierOp is TorchFourierOp assert FlatInverseFourierOp is TorchInverseFourierOp diff --git a/tests/test_from_ops_yaml.py b/tests/test_from_ops_yaml.py index 0dfb547..3aedcd8 100644 --- a/tests/test_from_ops_yaml.py +++ b/tests/test_from_ops_yaml.py @@ -10,14 +10,14 @@ import torch -from dataflux import Flux, Sample -from dataflux.ops.torch import RescaleOp # noqa: F401 - import registers the @configurable for !class: resolution +from sampleflux import Flux, Sample +from sampleflux.ops.torch import RescaleOp # noqa: F401 - import registers the @configurable for !class: resolution OPS_YAML = """ops: -- !class:dataflux.ops.torch.RescaleOp() +- !class:sampleflux.ops.torch.RescaleOp() in_min: 0.0 in_max: 255.0 -- !class:dataflux.ops.torch.RescaleOp() +- !class:sampleflux.ops.torch.RescaleOp() in_min: 0.0 in_max: 1.0 out_max: 10.0 diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py index 4cfd0a2..59e4e4f 100644 --- a/tests/test_image_ops.py +++ b/tests/test_image_ops.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`dataflux.ops.image` — generic value→image conversion. +"""Tests for :mod:`sampleflux.ops.image` — generic value→image conversion. ``ConvertToImageOp`` is the generic image-conversion op (normalize → colormap → optional flip → resize), and ``value_to_image`` / ``sample_to_image`` back it @@ -14,7 +14,7 @@ import torch from PIL import Image -from dataflux.ops.image import ( +from sampleflux.ops.image import ( COLORMAPS, TEXT_POSITIONS, Colormap, @@ -31,7 +31,7 @@ select_channel, value_to_image, ) -from dataflux.sample import Sample +from sampleflux.sample import Sample def _sample(value: object) -> Sample: diff --git a/tests/test_joint.py b/tests/test_joint.py index 45ad729..9fd4d06 100644 --- a/tests/test_joint.py +++ b/tests/test_joint.py @@ -2,8 +2,8 @@ import confluid # type: ignore[import-not-found] -from dataflux.core import Flux -from dataflux.sample import Sample +from sampleflux.core import Flux +from sampleflux.sample import Sample @confluid.configurable diff --git a/tests/test_kinds.py b/tests/test_kinds.py new file mode 100644 index 0000000..0f124af --- /dev/null +++ b/tests/test_kinds.py @@ -0,0 +1,262 @@ +"""Tests for op-kind introspection (`sampleflux.kinds`) and the native multi-type engine.""" + +from typing import Any, Iterable, Iterator, Optional, Tuple + +import numpy as np +import pytest +from confluid import configurable + +from sampleflux.core import Flux +from sampleflux.kinds import SAMPLE_KINDS, OpContract, classify_carrier, op_contract +from sampleflux.sample import Sample + +# --------------------------------------------------------------------------- +# Fixture ops (module-level so they pickle for spawn parity) +# --------------------------------------------------------------------------- + + +@configurable +class SampleOp: + """A classic annotated Sample op.""" + + def __call__(self, sample: Sample) -> Optional[Sample]: + return sample._replace(input=sample.input + 1) + + +@configurable +class PairOp: + """A metadata-free pair op: works on (input, target) tuples.""" + + def __call__(self, pair: Tuple[Any, Any]) -> Tuple[Any, Any]: + data, label = pair + return data * 2, label + + +@configurable +class UntypedOp: + """No annotations at all — works on anything (today's behavior).""" + + def __call__(self, sample): # type: ignore[no-untyped-def] + return sample + + +@configurable +class ExpandingOp: + """A 1→N op, detected from the Iterator return annotation.""" + + def __call__(self, sample: Sample) -> Iterator[Sample]: + yield sample + yield sample + + +@configurable +class ExpandingIterableOp: + """A 1→N op via Iterable[...].""" + + def __call__(self, sample: Sample) -> Iterable[Sample]: + return [sample, sample] + + +@configurable +class OverriddenOp: + """Introspection-opaque op relying on explicit class-attr overrides.""" + + SAMPLE_KIND_IN = "pair" + SAMPLE_KIND_OUT = "pair" + EXPANDS = False + + def __call__(self, *args): # type: ignore[no-untyped-def] + return args[0] + + +class StringAnnotatedOp: + """PEP-563-style string annotations must resolve (get_type_hints).""" + + def __call__(self, sample: "Sample") -> "Sample": + return sample + + +# --------------------------------------------------------------------------- +# classify_carrier / op_contract +# --------------------------------------------------------------------------- + + +class TestClassify: + def test_kinds_taxonomy_is_closed(self) -> None: + assert SAMPLE_KINDS == ("sample", "pair", "value", "any") + + def test_classify_carrier(self) -> None: + assert classify_carrier(Sample(1)) == "sample" + assert classify_carrier((np.zeros(3), 7)) == "pair" + assert classify_carrier(np.zeros(3)) == "value" + assert classify_carrier((1, 2, 3)) == "value" # only 2-tuples are pairs + + +class TestOpContract: + def test_sample_op(self) -> None: + assert op_contract(SampleOp()) == OpContract("sample", "sample", False) + + def test_pair_op(self) -> None: + assert op_contract(PairOp()) == OpContract("pair", "pair", False) + + def test_untyped_op_is_any(self) -> None: + assert op_contract(UntypedOp()) == OpContract("any", "any", False) + + def test_expanding_iterator_and_iterable(self) -> None: + assert op_contract(ExpandingOp()) == OpContract("sample", "sample", True) + assert op_contract(ExpandingIterableOp()) == OpContract("sample", "sample", True) + + def test_class_attr_overrides(self) -> None: + assert op_contract(OverriddenOp()) == OpContract("pair", "pair", False) + + def test_string_annotations_resolve(self) -> None: + assert op_contract(StringAnnotatedOp()).accepts == "sample" + + def test_pair_return_is_not_expansion(self) -> None: + # A Tuple return is a PAIR carrier, never a 1→N expansion. + contract = op_contract(PairOp()) + assert contract.produces == "pair" and contract.expands is False + + def test_introspection_failure_degrades_to_any(self) -> None: + class Broken: + pass + + # Inject unresolvable string annotations dynamically (mypy-safe: no fake name in source). + def _call(self, x): # type: ignore[no-untyped-def] + return x + + _call.__annotations__ = {"x": "NoSuchType", "return": "NoSuchType"} + Broken.__call__ = _call # type: ignore[method-assign, assignment] + assert op_contract(Broken()) == OpContract("any", "any", False) + + +# --------------------------------------------------------------------------- +# Native multi-type engine +# --------------------------------------------------------------------------- + + +class TestNativeFlux: + def test_pair_source_through_pair_op_stays_pairs(self) -> None: + pairs = [(np.full(2, float(i)), i) for i in range(3)] + flux = Flux(source=pairs, ops=[PairOp()], native=True) + out = list(flux) + assert all(isinstance(item, tuple) and len(item) == 2 for item in out) + assert out[1][0][0] == 2.0 and out[1][1] == 1 + + def test_pair_source_promoted_for_sample_op_sticky(self) -> None: + pairs = [(float(i), i) for i in range(3)] + flux = Flux(source=pairs, ops=[SampleOp()], native=True) + out = list(flux) + assert all(isinstance(item, Sample) for item in out) # promotion is sticky + assert [s.input for s in out] == [1.0, 2.0, 3.0] + assert all(s.meta == {} for s in out) + + def test_mixed_chain_pair_then_sample_op(self) -> None: + pairs = [(float(i), i) for i in range(3)] + flux = Flux(source=pairs, ops=[PairOp(), SampleOp()], native=True) + out = list(flux) + # PairOp doubled the value natively, then SampleOp promoted and added 1. + assert [s.input for s in out] == [1.0, 3.0, 5.0] + + def test_pair_op_on_sample_carrier_preserves_metadata(self) -> None: + samples = [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(3)] + flux = Flux(source=samples, ops=[PairOp()], native=True) + out = list(flux) + assert [s.input for s in out] == [0.0, 2.0, 4.0] + assert [s.meta["idx"] for s in out] == [0, 1, 2] # metadata rides through the pair view + + def test_untyped_op_receives_carrier_verbatim(self) -> None: + seen: list = [] + + @configurable + class Probe: + def __call__(self, x): # type: ignore[no-untyped-def] + seen.append(type(x).__name__) + return x + + list(Flux(source=[(1.0, 2)], ops=[Probe()], native=True)) + assert seen == ["tuple"] # NOT coerced + + def test_default_mode_unchanged(self) -> None: + # native=False (the default): 2-tuples coerce to Samples exactly as before. + out = list(Flux(source=[(1.0, 2)], ops=[SampleOp()])) + assert isinstance(out[0], Sample) and out[0].input == 2.0 + + def test_native_spawn_parallel_parity(self) -> None: + pairs = [(float(i), i) for i in range(4)] + seq = list(Flux(source=list(pairs), ops=[PairOp()], native=True)) + par = list(Flux(source=list(pairs), ops=[PairOp()], native=True).parallel(2)) + assert [(a[0], a[1]) for a in seq] == [(b[0], b[1]) for b in par] + + def test_native_getitem(self) -> None: + pairs = [(float(i), i) for i in range(4)] + flux = Flux(source=pairs, ops=[PairOp()], native=True) + item = flux[2] + assert item[0] == 4.0 and item[1] == 2 + + def test_native_filter_drop(self) -> None: + @configurable + class DropEven: + def __call__(self, pair: Tuple[Any, Any]) -> Optional[Tuple[Any, Any]]: + return None if pair[1] % 2 == 0 else pair + + out = list(Flux(source=[(0.0, 0), (1.0, 1), (2.0, 2)], ops=[DropEven()], native=True)) + assert [p[1] for p in out] == [1] + + +# --------------------------------------------------------------------------- +# Collate registry +# --------------------------------------------------------------------------- + + +class TestCollate: + def test_sample_default_list_form_metadata(self) -> None: + import torch + + from sampleflux.collate import collate + + batch = [Sample(input=torch.ones(2) * i, target=torch.tensor(i), metadata={"i": i}) for i in range(3)] + out = collate(batch) + assert isinstance(out, Sample) and out.is_batched + assert out.input.shape == (3, 2) and out.batch_meta[2]["i"] == 2 + + def test_pair_default(self) -> None: + from sampleflux.collate import collate + + data, labels = collate([(np.ones(2), 1), (np.zeros(2), 0)]) + assert data.shape == (2, 2) and list(labels) == [1, 0] + + def test_value_default(self) -> None: + from sampleflux.collate import collate + + out = collate([np.ones(2), np.zeros(2)]) + assert out.shape == (2, 2) + + def test_explicit_key_and_registration(self) -> None: + from sampleflux.collate import collate, get_collate, register_collate, registered_collates + + @register_collate("yolo_test") + def yolo_collate(items): # type: ignore[no-untyped-def] + return list(items) + + assert "yolo_test" in registered_collates() + assert get_collate("yolo_test") is yolo_collate + assert collate([(1, 2)], key="yolo_test") == [(1, 2)] + + def test_unknown_key_names_known(self) -> None: + from sampleflux.collate import get_collate + + with pytest.raises(KeyError, match="known:"): + get_collate("nope_nothing") + + def test_empty_batch_raises(self) -> None: + from sampleflux.collate import collate + + with pytest.raises(ValueError, match="empty"): + collate([]) + + def test_stack_fallback_to_list(self) -> None: + from sampleflux.collate import collate + + out = collate(["a", "b"], key="value") + assert out == ["a", "b"] diff --git a/tests/test_labels.py b/tests/test_labels.py index 6e273a2..203612d 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -1,12 +1,12 @@ -"""Tests for :class:`dataflux.labels.LabelMap` — the fittable name↔id label map.""" +"""Tests for :class:`sampleflux.labels.LabelMap` — the fittable name↔id label map.""" import json import pytest -from dataflux.labels import LabelMap -from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp -from dataflux.sample import Sample +from sampleflux.labels import LabelMap +from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp +from sampleflux.sample import Sample # --------------------------------------------------------------------------- # Construction & lazy validation @@ -78,7 +78,7 @@ def test_from_label_names_empty_raises() -> None: # --------------------------------------------------------------------------- -# encode_op / decode_op produce working dataflux ops +# encode_op / decode_op produce working sampleflux ops # --------------------------------------------------------------------------- diff --git a/tests/test_lazy_construction.py b/tests/test_lazy_construction.py index db18b01..4f65b5c 100644 --- a/tests/test_lazy_construction.py +++ b/tests/test_lazy_construction.py @@ -1,11 +1,11 @@ -"""Pins the "Lazy Initialization & Zero-Arg Construction" convention for ALL dataflux configurables. +"""Pins the "Lazy Initialization & Zero-Arg Construction" convention for ALL sampleflux configurables. -Every ``@configurable`` class in dataflux MUST be constructible with no arguments and do no +Every ``@configurable`` class in sampleflux MUST be constructible with no arguments and do no functional work in ``__init__`` (no I/O, no network, no eager materialization). This walks the whole package, discovers every ``@configurable`` class, and asserts ``Cls()`` succeeds — so a newly-added class that violates the convention (a required ctor arg, or a constructor that opens a file / loads a dataset) fails here. See confluid ``AGENTS.md`` → "Lazy Initialization & Zero-Arg -Construction" and dataflux ``AGENTS.md`` → "Lazy Evaluation". +Construction" and sampleflux ``AGENTS.md`` → "Lazy Evaluation". """ import importlib @@ -14,13 +14,13 @@ import pytest -import dataflux +import sampleflux -def _all_dataflux_configurables() -> List[type]: - """Import every dataflux submodule and collect the ``@configurable`` classes defined in dataflux.""" +def _all_sampleflux_configurables() -> List[type]: + """Import every sampleflux submodule and collect the ``@configurable`` classes defined in sampleflux.""" seen: dict = {} - for modinfo in pkgutil.walk_packages(dataflux.__path__, prefix="dataflux."): + for modinfo in pkgutil.walk_packages(sampleflux.__path__, prefix="sampleflux."): try: module = importlib.import_module(modinfo.name) except Exception: # pragma: no cover - optional/heavy deps absent in some envs @@ -29,18 +29,18 @@ def _all_dataflux_configurables() -> List[type]: if ( isinstance(obj, type) and getattr(obj, "__confluid_configurable__", False) - and getattr(obj, "__module__", "").startswith("dataflux") + and getattr(obj, "__module__", "").startswith("sampleflux") ): seen[f"{obj.__module__}.{obj.__qualname__}"] = obj return list(seen.values()) -_CONFIGURABLES = _all_dataflux_configurables() +_CONFIGURABLES = _all_sampleflux_configurables() def test_discovery_found_the_configurables() -> None: # Guard against the walker silently finding nothing (which would make the parametrized - # test below vacuously pass). dataflux has well over a dozen @configurable classes. + # test below vacuously pass). sampleflux has well over a dozen @configurable classes. assert len(_CONFIGURABLES) >= 20 @@ -55,7 +55,7 @@ def test_zero_arg_construction(cls: type) -> None: def test_sources_do_not_materialize_on_construction() -> None: # The lazy caches stay empty until first use — no dataset load / partition / offset compute # happens in __init__. - from dataflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource + from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource assert HuggingFaceSource()._dataset is None assert DatasetSplit()._views == {} diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index b4c6cb5..f71b73f 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -1,4 +1,4 @@ -"""Guard: every node-facing dataflux Source/Op documents all its constructor params. +"""Guard: every node-facing sampleflux Source/Op documents all its constructor params. These classes surface in FluxStudio (as widget tooltips) and navigaitor (as pydantic ``Field(description=...)`` in the form-spec) purely from their docstring @@ -12,8 +12,8 @@ import pytest from confluid import parse_param_docs # type: ignore[import-not-found] -from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp -from dataflux.ops.numpy import ( +from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp +from sampleflux.ops.numpy import ( ConnectedComponentsOp, FftShiftOp, FourierOp, @@ -24,18 +24,18 @@ ThresholdOp, WindowOp, ) -from dataflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp -from dataflux.ops.tee import Tee -from dataflux.ops.torch import FftShiftOp as TorchFftShiftOp -from dataflux.ops.torch import FourierOp as TorchFourierOp -from dataflux.ops.torch import IfftShiftOp as TorchIfftShiftOp -from dataflux.ops.torch import InverseFourierOp as TorchInverseFourierOp -from dataflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp -from dataflux.ops.torch import StandardizeOp as TorchStandardizeOp -from dataflux.ops.torch import ToTensorOp -from dataflux.ops.torch import WindowOp as TorchWindowOp -from dataflux.ops.transform_chain import TransformChain -from dataflux.sources import HuggingFaceSource +from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp +from sampleflux.ops.tee import Tee +from sampleflux.ops.torch import FftShiftOp as TorchFftShiftOp +from sampleflux.ops.torch import FourierOp as TorchFourierOp +from sampleflux.ops.torch import IfftShiftOp as TorchIfftShiftOp +from sampleflux.ops.torch import InverseFourierOp as TorchInverseFourierOp +from sampleflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp +from sampleflux.ops.torch import StandardizeOp as TorchStandardizeOp +from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.torch import WindowOp as TorchWindowOp +from sampleflux.ops.transform_chain import TransformChain +from sampleflux.sources import HuggingFaceSource _NODE_CLASSES = [ HuggingFaceSource, diff --git a/tests/test_ops.py b/tests/test_ops.py index 85b6689..8a202e4 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -1,4 +1,4 @@ -"""Tests for dataflux.ops: torch and numpy variants.""" +"""Tests for sampleflux.ops: torch and numpy variants.""" import os @@ -7,7 +7,7 @@ import torch from PIL import Image -from dataflux.ops import ( +from sampleflux.ops import ( CaptureOutputOp, ConfigureOp, CopyInputOp, @@ -27,9 +27,9 @@ UnstashInputOp, UnstashTargetOp, ) -from dataflux.ops import numpy as np_ops -from dataflux.ops.numpy import MaxOp, ThresholdOp -from dataflux.sample import Sample +from sampleflux.ops import numpy as np_ops +from sampleflux.ops.numpy import MaxOp, ThresholdOp +from sampleflux.sample import Sample # --------------------------------------------------------------------------- # ToTensorOp @@ -931,10 +931,10 @@ def test_missing_metadata_key_raises(self) -> None: np_ops.resolve_expression("{nope}", sample) def test_missing_env_var_raises(self) -> None: - os.environ.pop("DATAFLUX_TEST_NOPE", None) + os.environ.pop("SAMPLEFLUX_TEST_NOPE", None) sample = Sample(input=None, target=None, metadata={}) - with pytest.raises(KeyError, match="environment variable 'DATAFLUX_TEST_NOPE'"): - np_ops.resolve_expression("$DATAFLUX_TEST_NOPE", sample) + with pytest.raises(KeyError, match="environment variable 'SAMPLEFLUX_TEST_NOPE'"): + np_ops.resolve_expression("$SAMPLEFLUX_TEST_NOPE", sample) # --------------------------------------------------------------------------- @@ -1017,9 +1017,9 @@ def test_metadata_lookup_with_negation(self) -> None: assert out.meta["threshold_low"] == -30.0 def test_env_lookup(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("DATAFLUX_TEST_THRESHOLD", "1.0") + monkeypatch.setenv("SAMPLEFLUX_TEST_THRESHOLD", "1.0") arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(low_level="$DATAFLUX_TEST_THRESHOLD")(Sample(input=arr)) + out = np_ops.ThresholdOp(low_level="$SAMPLEFLUX_TEST_THRESHOLD")(Sample(input=arr)) np.testing.assert_array_equal(out.input, [False, False, True]) def test_high_level_expression(self) -> None: @@ -1322,7 +1322,7 @@ def test_roundtrip_squeeze_unsqueeze(self) -> None: # --------------------------------------------------------------------------- class TestDropMetadataOp: def test_literal_exclude_drops_exact_keys(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp # A pattern with no wildcards is an EXACT key match; a missing key is ignored. sample = Sample(input=np.zeros(2), target=None, metadata={"keep": 1, "drop_me": 2, "also": 3}) @@ -1330,14 +1330,14 @@ def test_literal_exclude_drops_exact_keys(self) -> None: assert out.meta == {"keep": 1} def test_glob_star_drops_all_matching(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp meta = {"real": 1, "__taidal_stash_456:input": [1j], "__taidal_stash_456:target": [2j]} out = DropMetadataOp(exclude=["__taidal_stash*"])(Sample(input=np.zeros(2), metadata=meta)) assert out.meta == {"real": 1} def test_glob_mid_wildcard_is_specific(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp # `__taidal_stash_456:*input` drops ONLY node 456's input stash — keeps its target and # other nodes' inputs. @@ -1350,27 +1350,27 @@ def test_glob_mid_wildcard_is_specific(self) -> None: assert out.meta == {"__taidal_stash_456:target": 2, "__taidal_stash_99:input": 3} def test_multiple_exclude_patterns_any_match(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp meta = {"a": 1, "b": 2, "__t_x": 3, "__t_y": 4} out = DropMetadataOp(exclude=["a", "__t_*"])(Sample(input=np.zeros(2), metadata=meta)) assert out.meta == {"b": 2} def test_question_mark_and_set_globs(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp meta = {"img0": 1, "img1": 2, "imgX": 3, "image": 4} out = DropMetadataOp(exclude=["img[0-9]"])(Sample(input=np.zeros(2), metadata=meta)) assert out.meta == {"imgX": 3, "image": 4} # only single-digit img0/img1 dropped def test_matching_is_case_sensitive(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp out = DropMetadataOp(exclude=["key"])(Sample(input=np.zeros(2), metadata={"Key": 1, "key": 2})) assert out.meta == {"Key": 1} def test_include_protects_keys_from_exclude(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp # include WINS: drop every stash key EXCEPT node 456's (carved out by include). meta = { @@ -1385,21 +1385,21 @@ def test_include_protects_keys_from_exclude(self) -> None: assert out.meta == {"real": 1, "__taidal_stash_456:input": 2, "__taidal_stash_456:target": 3} def test_include_without_exclude_drops_nothing(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp meta = {"a": 1, "b": 2} out = DropMetadataOp(include=["a"])(Sample(input=np.zeros(2), metadata=meta)) assert out.meta == {"a": 1, "b": 2} # include only protects against exclude def test_zero_arg_is_identity_metadata(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp meta = {"a": 1, "b": 2} out = DropMetadataOp()(Sample(input=np.zeros(2), metadata=meta)) assert out.meta == {"a": 1, "b": 2} def test_copy_on_write_does_not_mutate_original(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp original = {"a": 1, "drop": 2} out = DropMetadataOp(exclude=["drop"])(Sample(input=np.zeros(2), metadata=original)) @@ -1407,7 +1407,7 @@ def test_copy_on_write_does_not_mutate_original(self) -> None: assert out.meta == {"a": 1} def test_input_and_target_untouched(self) -> None: - from dataflux.ops.metadata import DropMetadataOp + from sampleflux.ops.metadata import DropMetadataOp arr = np.arange(3) out = DropMetadataOp(exclude=["x"])(Sample(input=arr, target=7, metadata={"x": 1, "y": 2})) @@ -1420,14 +1420,14 @@ def test_input_and_target_untouched(self) -> None: # --------------------------------------------------------------------------- class TestPrintSampleOp: def test_returns_sample_unchanged(self) -> None: - from dataflux.ops.debug import PrintSampleOp + from sampleflux.ops.debug import PrintSampleOp sample = Sample(input=np.zeros(3), target=1, metadata={"a": 1}) out = PrintSampleOp(to_console=False)(sample) assert out is sample def test_prints_to_console(self, capsys: pytest.CaptureFixture) -> None: - from dataflux.ops.debug import PrintSampleOp + from sampleflux.ops.debug import PrintSampleOp PrintSampleOp(label="probe")(Sample(input=np.zeros((2, 3)), target=None, metadata={"k": 1})) captured = capsys.readouterr().out @@ -1436,7 +1436,7 @@ def test_prints_to_console(self, capsys: pytest.CaptureFixture) -> None: assert "'k'" in captured # metadata key def test_summarizes_large_array_metadata_without_dumping(self, capsys: pytest.CaptureFixture) -> None: - from dataflux.ops.debug import PrintSampleOp + from sampleflux.ops.debug import PrintSampleOp big = np.arange(100000, dtype=np.complex64) # would flood / not be reprable in full PrintSampleOp(label="p")(Sample(input=np.zeros(2), metadata={"iq": big})) @@ -1445,14 +1445,14 @@ def test_summarizes_large_array_metadata_without_dumping(self, capsys: pytest.Ca assert "..." in out and "50000" not in out # values elided, not dumped in full def test_prints_small_array_values(self, capsys: pytest.CaptureFixture) -> None: - from dataflux.ops.debug import PrintSampleOp + from sampleflux.ops.debug import PrintSampleOp PrintSampleOp(label="p")(Sample(input=np.array([1, 2, 3]), target=None, metadata={})) out = capsys.readouterr().out assert "values=[1, 2, 3]" in out # actual values shown for a small array def test_limit_caps_emissions_but_passes_all(self, capsys: pytest.CaptureFixture) -> None: - from dataflux.ops.debug import PrintSampleOp + from sampleflux.ops.debug import PrintSampleOp op = PrintSampleOp(label="p", limit=2) for _ in range(5): @@ -1461,7 +1461,7 @@ def test_limit_caps_emissions_but_passes_all(self, capsys: pytest.CaptureFixture assert len(lines) == 2 # only the first 2 printed def test_to_console_false_is_silent_on_stdout(self, capsys: pytest.CaptureFixture) -> None: - from dataflux.ops.debug import PrintSampleOp + from sampleflux.ops.debug import PrintSampleOp PrintSampleOp(to_console=False)(Sample(input=np.zeros(1), metadata={})) assert capsys.readouterr().out == "" diff --git a/tests/test_paired.py b/tests/test_paired.py index ffdba5b..267bc0d 100644 --- a/tests/test_paired.py +++ b/tests/test_paired.py @@ -1,13 +1,13 @@ -"""Tests for dataflux.paired.AnnotationJoinSource.""" +"""Tests for sampleflux.paired.AnnotationJoinSource.""" from typing import Any, Dict, Iterator, Optional import confluid # type: ignore[import-not-found] import pytest -from dataflux.discovery import get_callable_path -from dataflux.paired import AnnotationJoinSource -from dataflux.sample import Sample +from sampleflux.discovery import get_callable_path +from sampleflux.paired import AnnotationJoinSource +from sampleflux.sample import Sample # --------------------------------------------------------------------------- # Test fixtures diff --git a/tests/test_parallel.py b/tests/test_parallel.py index b86f7d2..a9fc68c 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -2,7 +2,7 @@ import numpy as np -from dataflux.core import Flux +from sampleflux.core import Flux def heavy_op(x: np.ndarray) -> np.ndarray: diff --git a/tests/test_parallel_op.py b/tests/test_parallel_op.py index 5f9d7cd..7547ae6 100644 --- a/tests/test_parallel_op.py +++ b/tests/test_parallel_op.py @@ -1,4 +1,4 @@ -"""Tests for :class:`dataflux.ops.parallel.Parallel`.""" +"""Tests for :class:`sampleflux.ops.parallel.Parallel`.""" from __future__ import annotations @@ -7,9 +7,9 @@ import numpy as np -from dataflux.core import Flux -from dataflux.ops.parallel import Parallel -from dataflux.sample import Sample +from sampleflux.core import Flux +from sampleflux.ops.parallel import Parallel +from sampleflux.sample import Sample # Top-level functions/classes — workers must be able to pickle these. diff --git a/tests/test_projection.py b/tests/test_projection.py index ff498d0..cca4468 100644 --- a/tests/test_projection.py +++ b/tests/test_projection.py @@ -7,8 +7,8 @@ import pytest import torch -from dataflux.core import Flux -from dataflux.projection import ( +from sampleflux.core import Flux +from sampleflux.projection import ( _FIELDS, INPUT, TARGET, @@ -20,7 +20,7 @@ num_classes, project, ) -from dataflux.sample import Sample +from sampleflux.sample import Sample # --------------------------------------------------------------------------- # # ProjectionField is a closed Literal a UI / form-spec can enumerate diff --git a/tests/test_random_apply.py b/tests/test_random_apply.py index d3d3c8c..e18e320 100644 --- a/tests/test_random_apply.py +++ b/tests/test_random_apply.py @@ -1,9 +1,9 @@ -"""Tests for dataflux.ops.random_apply.RandomApply.""" +"""Tests for sampleflux.ops.random_apply.RandomApply.""" import pytest -from dataflux.ops.random_apply import RandomApply -from dataflux.sample import Sample +from sampleflux.ops.random_apply import RandomApply +from sampleflux.sample import Sample def _s(v: int = 0) -> Sample: diff --git a/tests/test_sample.py b/tests/test_sample.py index 7763550..0d7df67 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from dataflux.sample import Sample +from sampleflux.sample import Sample def test_sample_from_any() -> None: @@ -76,7 +76,7 @@ def test_describe_falls_back_to_inference_on_a_batch() -> None: def test_with_type_rejects_a_batch() -> None: - from dataflux.typespec import infer_sample_type + from sampleflux.typespec import infer_sample_type single = Sample(input=np.zeros((4,)), target=0, metadata={}) typed = single.with_type(infer_sample_type(single)) # single sample: OK diff --git a/tests/test_sigmf.py b/tests/test_sigmf.py new file mode 100644 index 0000000..c0be33c --- /dev/null +++ b/tests/test_sigmf.py @@ -0,0 +1,175 @@ +"""Tests for the SigMF storage pair (`sampleflux.storage.sigmf`) and the metadata query layer.""" + +import json +from pathlib import Path + +import numpy as np +import pytest + +from sampleflux.sample import Sample +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.query import MetadataFilterSource, SupportsMetadataScan, scan_hdf5_metadata +from sampleflux.storage.sigmf import SigMFSink, SigMFSource +from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource + + +def _iq(n: int = 16, seed: float = 1.0) -> np.ndarray: + return (np.arange(n) * seed + 1j * np.arange(n)).astype(np.complex64) + + +class TestSigMFRoundTrip: + def test_complex64_iq_round_trips(self, tmp_path: Path) -> None: + sink = SigMFSink(path=tmp_path / "recs") + sink.write(Sample(input=_iq(), target=None, metadata={"samplerate": 1e6, "drone": "DJI"})) + sink.write(Sample(input=_iq(seed=2.0), target=3, metadata={"snr_db": 12.5})) + sink.flush() + + source = SigMFSource(path=tmp_path / "recs") + samples = list(source) + assert len(samples) == len(source) == 2 + np.testing.assert_array_equal(samples[0].input, _iq()) + assert samples[0].input.dtype == np.complex64 + assert samples[0].meta["samplerate"] == 1e6 and samples[0].meta["drone"] == "DJI" + assert samples[1].target == 3 # JSON-able target restored + assert source[1].meta["snr_db"] == 12.5 # random access + + def test_float_and_int_dtypes(self, tmp_path: Path) -> None: + for arr in (np.ones(4, dtype=np.float32), np.arange(4, dtype=np.int16)): + sink = SigMFSink(path=tmp_path / str(arr.dtype)) + sink.write(Sample(input=arr, metadata={})) + out = list(SigMFSource(path=tmp_path / str(arr.dtype)))[0] + np.testing.assert_array_equal(out.input, arr) + assert out.input.dtype == arr.dtype + + def test_unsupported_dtype_raises(self, tmp_path: Path) -> None: + sink = SigMFSink(path=tmp_path / "bad") + with pytest.raises(TypeError, match="core:datatype"): + sink.write(Sample(input=np.ones(2, dtype=np.float16), metadata={})) + + def test_meta_file_shape_and_checksum(self, tmp_path: Path) -> None: + sink = SigMFSink(path=tmp_path / "recs", checksum=True) + sink.write(Sample(input=_iq(), metadata={"core:description": "capture"})) + doc = json.loads(next((tmp_path / "recs").glob("*.sigmf-meta")).read_text()) + assert doc["global"]["core:datatype"] == "cf32_le" + assert doc["global"]["core:version"] == "1.0.0" + assert doc["global"]["core:description"] == "capture" # core: keys ride verbatim + assert len(doc["global"]["core:sha512"]) == 128 + assert doc["captures"] == [{"core:sample_start": 0}] + + def test_non_serializable_metadata_skipped_not_fatal(self, tmp_path: Path) -> None: + sink = SigMFSink(path=tmp_path / "recs") + sink.write(Sample(input=_iq(), metadata={"ok": 1, "bad": np.ones(3)})) + out = list(SigMFSource(path=tmp_path / "recs"))[0] + assert out.meta["ok"] == 1 and "bad" not in out.meta + + def test_sink_requires_path(self) -> None: + with pytest.raises(ValueError, match="'path'"): + SigMFSink().write(Sample(input=_iq())) + + def test_source_requires_directory(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="not a directory"): + len(SigMFSource(path=tmp_path / "missing")) + + def test_waivefront_vocab_hooks_round_trip(self, tmp_path: Path) -> None: + meta = { + "samplerate": 2e6, + "center_freq": 2.4e9, + "snr": "clean", + "annotated_regions": [[100.0, 200.0, 0.5, 1.0]], + "annotated_labels": ["wifi"], + "drone": "DJI", + } + sink = SigMFSink(path=tmp_path / "wf", meta_encoder="waivefront.vocab.to_sigmf") + sink.write(Sample(input=_iq(), metadata=meta)) + doc = json.loads(next((tmp_path / "wf").glob("*.sigmf-meta")).read_text()) + assert doc["global"]["core:sample_rate"] == 2e6 + assert doc["captures"][0]["core:frequency"] == 2.4e9 + assert doc["global"]["waivefront:snr_raw"] == "clean" # unparseable snr kept verbatim + annotation = doc["annotations"][0] + assert annotation["core:freq_lower_edge"] == 100.0 + assert annotation["core:sample_start"] == int(0.5 * 2e6) + assert annotation["core:label"] == "wifi" and annotation["waivefront:role"] == "annotated" + + out = list(SigMFSource(path=tmp_path / "wf", meta_decoder="waivefront.vocab.from_sigmf"))[0] + assert out.meta["samplerate"] == 2e6 and out.meta["center_freq"] == 2.4e9 + assert out.meta["snr"] == "clean" and out.meta["drone"] == "DJI" + region = out.meta["annotated_regions"][0] + assert region[0] == 100.0 and region[2] == pytest.approx(0.5) and region[3] == pytest.approx(1.0) + assert out.meta["annotated_labels"] == ["wifi"] + + +class TestMetadataQuery: + def _write_hdf5(self, path: Path) -> Path: + sink = HDF5Sink(path=path) + for i in range(4): + sink.write( + Sample( + input=np.ones(3) * i, + metadata={"snr_db": float(i * 5), "drone": "DJI" if i % 2 else "Parrot", "mask": np.ones((2, 2))}, + ) + ) + sink.flush() + sink.close() + return path + + def test_hdf5_scan_reads_no_arrays(self, tmp_path: Path) -> None: + path = self._write_hdf5(tmp_path / "d.h5") + scanned = list(scan_hdf5_metadata(path)) + assert len(scanned) == 4 + _key, meta = scanned[2] + assert meta["snr_db"] == 10.0 + assert meta["mask"].startswith(" None: + path = self._write_hdf5(tmp_path / "d.h5") + assert isinstance(HDF5Source(path=path), SupportsMetadataScan) + assert isinstance(SigMFSource(path=tmp_path), SupportsMetadataScan) + assert isinstance(ZarrGroupSource(path=str(tmp_path / "z")), SupportsMetadataScan) + + def test_filter_source_where_expression_on_hdf5(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + view = MetadataFilterSource(source=source, where="snr_db >= 10") + assert len(view) == 2 + assert [s.meta["snr_db"] for s in view] == [10.0, 15.0] + assert view[0].meta["snr_db"] == 10.0 # random access into matches + + def test_filter_source_string_and_predicate_compose(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + view = MetadataFilterSource(source=source, where="drone == 'DJI'", predicate=lambda m: m["snr_db"] > 5) + assert [s.meta["snr_db"] for s in view] == [15.0] + + def test_missing_key_is_non_matching_not_fatal(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + assert len(MetadataFilterSource(source=source, where="no_such_key > 1")) == 0 + + def test_malformed_expression_fails_loudly(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + with pytest.raises(ValueError, match="failed"): + len(MetadataFilterSource(source=source, where="snr_db +* 2")) + + def test_empty_filter_is_rejected(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="empty filter"): + len(MetadataFilterSource(source=[Sample(1)])) + + def test_fallback_full_iteration_for_plain_sources(self) -> None: + plain = [Sample(input=i, metadata={"v": i}) for i in range(5)] + view = MetadataFilterSource(source=plain, where="v % 2 == 0") + assert [s.input for s in view] == [0, 2, 4] + + def test_zarr_scan_and_filter(self, tmp_path: Path) -> None: + sink = ZarrGroupSink(path=str(tmp_path / "z")) + for i in range(3): + sink.write(Sample(input=np.ones(2) * i, metadata={"v": i})) + sink.flush() + source = ZarrGroupSource(path=str(tmp_path / "z")) + view = MetadataFilterSource(source=source, where="v == 1") + assert len(view) == 1 and view[0].meta["v"] == 1 + + def test_sigmf_scan_and_filter(self, tmp_path: Path) -> None: + sink = SigMFSink(path=tmp_path / "recs") + for i in range(3): + sink.write(Sample(input=_iq(seed=float(i + 1)), metadata={"snr_db": float(i * 10)})) + source = SigMFSource(path=tmp_path / "recs") + view = MetadataFilterSource(source=source, where="snr_db >= 10") + assert len(view) == 2 + np.testing.assert_array_equal(view[0].input, _iq(seed=2.0)) diff --git a/tests/test_sources.py b/tests/test_sources.py index aa02d69..f5a1381 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1,4 +1,4 @@ -"""Tests for DataFlux sources: DatasetSplit (+ cached split views), RangeSource, ConcatSource.""" +"""Tests for SampleFlux sources: DatasetSplit (+ cached split views), RangeSource, ConcatSource.""" import inspect from typing import Any, Iterator, List @@ -6,9 +6,9 @@ import confluid # type: ignore[import-not-found] import pytest -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.sources import ConcatSource, DatasetSplit, RangeSource +from sampleflux.core import Flux +from sampleflux.sample import Sample +from sampleflux.sources import ConcatSource, DatasetSplit, RangeSource @confluid.configurable @@ -345,10 +345,10 @@ def test_property_refs_share_one_instance_and_partition_cleanly() -> None: val_fraction: 0.2 seed: 9 -train_set: !class:dataflux.core.Flux() +train_set: !class:sampleflux.core.Flux() source: !ref:my_split.train -val_set: !class:dataflux.core.Flux() +val_set: !class:sampleflux.core.Flux() source: !ref:my_split.val """ state: Any = confluid.load(yaml_state) @@ -421,7 +421,7 @@ def test_concat_source_roundtrip() -> None: def test_hf_source_zero_arg_construction_does_no_work() -> None: # Per the lazy / zero-arg convention: building the source must not touch the network and # must succeed with no constructor arguments. Nothing is materialized until first use. - from dataflux.sources import HuggingFaceSource + from sampleflux.sources import HuggingFaceSource src = HuggingFaceSource() assert src._dataset is None # nothing loaded at construction time @@ -434,7 +434,7 @@ def test_hf_source_zero_arg_construction_does_no_work() -> None: def test_hf_source_dataset_without_path_raises() -> None: # The zero-arg constructor allows an unconfigured source, but materializing one without a # dataset id cannot succeed — the error surfaces lazily, at the `dataset` property, not in __init__. - from dataflux.sources import HuggingFaceSource + from sampleflux.sources import HuggingFaceSource src = HuggingFaceSource() with pytest.raises(ValueError, match="path is empty"): @@ -448,7 +448,7 @@ def test_hf_source_dataset_without_path_raises() -> None: def _hf_source_with_count(count: Any, dataset_len: int = 13) -> Any: """Build a HuggingFaceSource and pre-seed its lazy cache so `dataset` never hits the network.""" - from dataflux.sources import HuggingFaceSource + from sampleflux.sources import HuggingFaceSource src: Any = HuggingFaceSource(count=count) src._dataset = list(range(dataset_len)) # short-circuits the lazy load in the `dataset` property @@ -476,7 +476,7 @@ def test_hf_source_len_positive_count_caps() -> None: def test_resolve_metadata_features_none_and_empty_mean_none() -> None: - from dataflux.sources import _resolve_metadata_features + from sampleflux.sources import _resolve_metadata_features cols = ["image", "label", "id", "source_file"] assert _resolve_metadata_features(None, cols, "image", "label") == [] @@ -484,7 +484,7 @@ def test_resolve_metadata_features_none_and_empty_mean_none() -> None: def test_resolve_metadata_features_explicit_list_verbatim() -> None: - from dataflux.sources import _resolve_metadata_features + from sampleflux.sources import _resolve_metadata_features cols = ["image", "label", "id", "source_file"] assert _resolve_metadata_features(["id"], cols, "image", "label") == ["id"] @@ -493,7 +493,7 @@ def test_resolve_metadata_features_explicit_list_verbatim() -> None: def test_resolve_metadata_features_star_is_the_rest() -> None: - from dataflux.sources import _resolve_metadata_features + from sampleflux.sources import _resolve_metadata_features cols = ["image", "label", "id", "source_file"] # the rest = every column except input/target, order preserved @@ -503,7 +503,7 @@ def test_resolve_metadata_features_star_is_the_rest() -> None: def test_resolve_metadata_features_star_plus_extras_union() -> None: - from dataflux.sources import _resolve_metadata_features + from sampleflux.sources import _resolve_metadata_features cols = ["image", "label", "id"] # "*" plus a name already in the rest -> no duplicate; an out-of-columns extra is appended @@ -511,7 +511,7 @@ def test_resolve_metadata_features_star_plus_extras_union() -> None: def test_resolve_metadata_features_star_without_columns_degrades() -> None: - from dataflux.sources import _resolve_metadata_features + from sampleflux.sources import _resolve_metadata_features # No column_names available (e.g. a non-Dataset backing) -> "*" yields just the extras. assert _resolve_metadata_features(["*"], None, "image", "label") == [] @@ -534,7 +534,7 @@ def test_hf_source_iter_metadata_features_star_expands_on_real_dataset() -> None # End-to-end through __iter__: a dataset with extra columns + metadata_features="*" carries # every non-input/target column onto Sample.metadata (plus the synthetic hf_path/hf_split). # The "*" expansion is now lazy (resolved_metadata_features reads dataset.column_names). - from dataflux.sources import HuggingFaceSource + from sampleflux.sources import HuggingFaceSource rows = [{"image": i, "label": i % 2, "id": f"r{i}", "src": "a"} for i in range(3)] src = HuggingFaceSource(path="fake/ds", split="train", metadata_features=["*"]) diff --git a/tests/test_storage.py b/tests/test_storage.py index a8bf800..e784674 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -5,11 +5,11 @@ import numpy as np import torch -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.storage.directory import DirectorySink -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource +from sampleflux.core import Flux +from sampleflux.sample import Sample +from sampleflux.storage.directory import DirectorySink +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource def test_hdf5_storage(tmp_path: Path) -> None: @@ -142,7 +142,7 @@ def test_directory_storage_separate(tmp_path: Path) -> None: def test_hdf5_to_numpy_direct() -> None: - from dataflux.storage.hdf5 import to_numpy + from sampleflux.storage.hdf5 import to_numpy # Hits line 19 assert to_numpy(123) == 123 diff --git a/tests/test_target_ops.py b/tests/test_target_ops.py index 6d8cade..0196c37 100644 --- a/tests/test_target_ops.py +++ b/tests/test_target_ops.py @@ -1,17 +1,17 @@ -"""Tests for the target movers / encoders (``dataflux.ops.target``).""" +"""Tests for the target movers / encoders (``sampleflux.ops.target``).""" import numpy as np import pytest from PIL import Image -from dataflux.ops.target import ( +from sampleflux.ops.target import ( CocoToTorchVisionDetectionOp, DecodeTargetOp, EncodeTargetOp, MasksToDetectionBoxesOp, MetadataToTargetOp, ) -from dataflux.sample import Sample +from sampleflux.sample import Sample # --------------------------------------------------------------------------- # diff --git a/tests/test_transform_chain.py b/tests/test_transform_chain.py index e95dd0a..73fb7a1 100644 --- a/tests/test_transform_chain.py +++ b/tests/test_transform_chain.py @@ -1,9 +1,9 @@ -"""Tests for :class:`dataflux.ops.transform_chain.TransformChain`.""" +"""Tests for :class:`sampleflux.ops.transform_chain.TransformChain`.""" from typing import List, Optional -from dataflux.ops.transform_chain import TransformChain -from dataflux.sample import Sample +from sampleflux.ops.transform_chain import TransformChain +from sampleflux.sample import Sample def _s(v: int = 0) -> Sample: diff --git a/tests/test_typespec.py b/tests/test_typespec.py index 48ed442..85114fd 100644 --- a/tests/test_typespec.py +++ b/tests/test_typespec.py @@ -1,13 +1,13 @@ -"""Exhaustive tests for the dataflux type-spec system (matching, inference, JSON, HF bridge).""" +"""Exhaustive tests for the sampleflux type-spec system (matching, inference, JSON, HF bridge).""" from typing import Any, Iterable, List, Tuple, cast, get_args import numpy as np import pytest -from dataflux.core import Flux -from dataflux.sample import FEATURES_KEY, SPEC_KEY, Sample -from dataflux.typespec import ( +from sampleflux.core import Flux +from sampleflux.sample import FEATURES_KEY, SPEC_KEY, Sample +from sampleflux.typespec import ( _DTYPE_FAMILIES, AnyType, ArrayType, @@ -517,9 +517,9 @@ def test_pipeline_drops_stored_type_when_op_has_no_produces() -> None: # -------------------------------------------------------------------------------------------------- -def test_dataflux_op_spec_conformance() -> None: - import dataflux.ops.numpy as N - import dataflux.ops.torch as T +def test_sampleflux_op_spec_conformance() -> None: + import sampleflux.ops.numpy as N + import sampleflux.ops.torch as T rgb = (np.random.rand(3, 8, 8) * 255).astype(np.float32) cases: List[Tuple[Any, Sample]] = [ diff --git a/tests/test_windows.py b/tests/test_windows.py index cd90efe..42aa2e2 100644 --- a/tests/test_windows.py +++ b/tests/test_windows.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`dataflux.windows` — the window functions + spectral unit-scaling math. +"""Tests for :mod:`sampleflux.windows` — the window functions + spectral unit-scaling math. Pins (1) the closed Literals match their runtime tuples, (2) the pure-numpy windows match scipy (when available) and have the right correction constants (Hann coherent gain 0.5 / ENBW @@ -12,8 +12,8 @@ import numpy as np import pytest -from dataflux import windows as W -from dataflux.windows import WindowName +from sampleflux import windows as W +from sampleflux.windows import WindowName def test_literals_match_runtime_tuples() -> None: From 2afd0b9d6166d629a75891695c47d7b223fe2299 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 18 Jul 2026 05:29:45 +0200 Subject: [PATCH 022/102] chore: re-scaffold Jenkinsfiles + CI from current aisland templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerated via 'aisland jenkins scaffold sampleflux --force' after the rename — picks up the directory-examples runner (examples/*/run.py) the templates gained on 2026-07-16. --- .github/workflows/ci.yml | 9 +++++++++ Jenkinsfile | 9 +++++++++ Jenkinsfile.local | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6ad4bc..175e50d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,15 @@ jobs: python3 "$f" fi done + # Directory examples opt in by shipping a run.py entry point; + # network-dependent / install-only example apps ship no run.py + # and are skipped by construction. + for f in examples/*/run.py; do + if [ -f "$f" ]; then + echo "Verifying $f..." + python3 "$f" + fi + done verify-notebooks: name: Verify Notebooks diff --git a/Jenkinsfile b/Jenkinsfile index 7bd75d0..63be8de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -242,6 +242,15 @@ with open('isort-checkstyle.xml', 'w') as f: ${VENV_BIN}/python3 "$f" fi done + # Directory examples opt in by shipping a run.py entry point; + # network-dependent / install-only example apps ship no run.py + # and are skipped by construction. + for f in examples/*/run.py; do + if [ -f "$f" ]; then + echo "Verifying $f..." + ${VENV_BIN}/python3 "$f" + fi + done ''' } } diff --git a/Jenkinsfile.local b/Jenkinsfile.local index f02e8f1..5bf079d 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -252,6 +252,15 @@ with open('isort-checkstyle.xml', 'w') as f: ${VENV_BIN}/python3 "$f" fi done + # Directory examples opt in by shipping a run.py entry point; + # network-dependent / install-only example apps ship no run.py + # and are skipped by construction. + for f in examples/*/run.py; do + if [ -f "$f" ]; then + echo "Verifying $f..." + ${VENV_BIN}/python3 "$f" + fi + done ''' } } From 41189375ba8d4c04e5f3c7ac6c423cd73e85219b Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 19 Jul 2026 16:52:56 +0200 Subject: [PATCH 023/102] =?UTF-8?q?feat:=20modality-neutral=20engine=20?= =?UTF-8?q?=E2=80=94=20move=20signal-domain=20code=20to=20waivefront;=20tr?= =?UTF-8?q?ansform-taxonomy=20grid;=20op=20consolidation;=20docs=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transform taxonomy (kinds.py): field-scope x call-style GRID with combinable per-parameter bindings — Pair/InputMeta/TargetMeta NamedTuple views, INPUT/TARGET marks, packed/unpacked call styles, metadata-only scope, single-view-return guard. Relocations (no back-compat; new homes in waivefront): - storage/sigmf.py -> waivefront.sigmf (query layer stays; protocol is structural) - windows.py + the 1-D FFT/windowing/scaling op family (numpy+torch) -> waivefront.{windows,fourier,fourier_torch} - paired.py (AnnotationJoinSource/AnnotationStore) -> waivefront.paired Op consolidation (one wiring plane): - DELETE Tee (executionally identical to TransformChain) and CaptureOutputOp (context Capture is the op; Apply(source=cell) replaces ConfigureOp+Unstash idiom) - stash family narrowed to Parallel-boundary crossing + sink persistence; graph wiring is exclusively the context ops Docs: README split into slim landing page + 8 per-topic docs/*.md; all FluxStudio references genericized (UI/engine separation); AGENTS mandates updated (modality-neutral rule, consolidation, narrowed stash charter). --- AGENTS.md | 9 +- README.md | 598 ++------------------------- docs/configure.md | 35 ++ docs/graph.md | 92 +++++ docs/image.md | 33 ++ docs/kinds.md | 103 +++++ docs/projection.md | 48 +++ docs/sources.md | 83 ++++ docs/storage.md | 61 +++ docs/typespec.md | 37 ++ examples/paired_annotations.py | 226 ----------- pyproject.toml | 7 +- sampleflux/__init__.py | 14 +- sampleflux/collate.py | 14 +- sampleflux/core.py | 192 +++++++-- sampleflux/discovery.py | 6 +- sampleflux/flow.py | 6 +- sampleflux/kinds.py | 256 ++++++++++-- sampleflux/ops/__init__.py | 36 +- sampleflux/ops/capture.py | 120 ------ sampleflux/ops/configure.py | 4 +- sampleflux/ops/context.py | 4 +- sampleflux/ops/copy.py | 6 +- sampleflux/ops/debug.py | 2 +- sampleflux/ops/enable.py | 2 +- sampleflux/ops/formula.py | 2 +- sampleflux/ops/image.py | 24 +- sampleflux/ops/metadata.py | 10 +- sampleflux/ops/numpy.py | 333 +-------------- sampleflux/ops/parallel.py | 4 +- sampleflux/ops/random_apply.py | 2 +- sampleflux/ops/stash.py | 31 +- sampleflux/ops/tee.py | 53 --- sampleflux/ops/torch.py | 388 +----------------- sampleflux/ops/transform_chain.py | 8 +- sampleflux/paired.py | 254 ------------ sampleflux/sample.py | 54 ++- sampleflux/sources.py | 4 +- sampleflux/storage/directory.py | 2 +- sampleflux/storage/hdf5.py | 2 +- sampleflux/storage/query.py | 6 +- sampleflux/storage/sigmf.py | 270 ------------ sampleflux/storage/zarr.py | 4 +- sampleflux/typespec.py | 8 +- sampleflux/windows.py | 261 ------------ tests/test_categories.py | 65 +-- tests/test_fourier_ops.py | 655 ------------------------------ tests/test_kinds.py | 326 ++++++++++++++- tests/test_node_docs.py | 32 +- tests/test_ops.py | 177 +------- tests/test_paired.py | 497 ----------------------- tests/test_query.py | 78 ++++ tests/test_sigmf.py | 175 -------- tests/test_windows.py | 157 ------- 54 files changed, 1464 insertions(+), 4412 deletions(-) create mode 100644 docs/configure.md create mode 100644 docs/graph.md create mode 100644 docs/image.md create mode 100644 docs/kinds.md create mode 100644 docs/projection.md create mode 100644 docs/sources.md create mode 100644 docs/storage.md create mode 100644 docs/typespec.md delete mode 100644 examples/paired_annotations.py delete mode 100644 sampleflux/ops/capture.py delete mode 100644 sampleflux/ops/tee.py delete mode 100644 sampleflux/paired.py delete mode 100644 sampleflux/storage/sigmf.py delete mode 100644 sampleflux/windows.py delete mode 100644 tests/test_fourier_ops.py delete mode 100644 tests/test_paired.py create mode 100644 tests/test_query.py delete mode 100644 tests/test_sigmf.py delete mode 100644 tests/test_windows.py diff --git a/AGENTS.md b/AGENTS.md index bd8542a..fb17613 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,24 +1,27 @@ # SampleFlux Mandates +- **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). +- **Op Consolidation (2026-07-18) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases). `Tee` threaded the sample through its branches sequentially, making it executionally identical to `TransformChain(ops=[...])` — use `TransformChain` for grouping and the context ops (`Save`/`Use`/`Mix`) for real, isolated fan-out. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom `ConfigureOp(ops=[UnstashInputOp(key)])` is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-sample side-branch (`ops` chain → `metadata[key]` + setattr) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters (fluxstudio export.py AND graphio.py) emit ONLY context ops for wiring; graphio's legacy `__taidal_stash_*` import replay was removed (pre-2026-07 stash-format ops-docs no longer import — re-export from the canvas). Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. - **`Sample.metadata` Is `dict` (single) OR `list[dict]` (batch) — Narrow via `.meta` / `.batch_meta`:** The `metadata` field is `Metadata = Union[Dict[str, Any], List[Dict[str, Any]]]`. A **single** item carries one `dict` (the normal pipeline form every source/op produces and consumes); a **batch** carries a `list` of per-item dicts (one per stacked item), produced by the collate functions (`marainer.collate.collate_fn_with_metadata`, `sonair.classification.classification_collate_fn`) when N samples are stacked into one Sample for the model/loss/predictions-sinks. `Sample.is_batched` (= `isinstance(metadata, list)`) is the single source of truth for telling them apart. Per-sample code MUST read/mutate metadata through the narrowing accessor **`sample.meta`** (returns the dict, raises `TypeError` on a batch) — `sample.meta[key]` / `sample.meta[key] = v`; batch consumers use **`sample.batch_meta`** (returns the list, raises on a single). NEVER index the raw `sample.metadata` Union directly (mypy rejects `Union[...][str]`). NOTE the batch convention is per-collate: marainer/sonair stack into the **list** form (`is_batched` True); deltaid's `segmentation_collate_fn` instead nests under a **dict** `metadata={"per_sample": [...]}` (so `is_batched` is False there — use `.meta["per_sample"]`). `describe()`/`with_type()` operate on single samples only (a batch infers / raises). Pins: `tests/test_sample.py` (batch vs single, `.meta`/`.batch_meta` guards). - **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane — Never `sample.metadata` (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its `input`, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `Mix` (fan-in; named slots read cells, empty slots keep the incoming sample; metadata merges incoming-first then slot order, `metadata_from` wins last). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches `sample.metadata` — a linear run's metadata is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`), mirroring `UnstashInputOp(copy=True, remove=True)` — the stash family remains the METADATA-bus twin for hand-written configs, the context ops are what `flow:` documents/FluxStudio lower to; (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. +- **The Context Is the Graph Data Plane — Never `sample.metadata` (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its `input`, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `Mix` (fan-in; named slots read cells, empty slots keep the incoming sample; metadata merges incoming-first then slot order, `metadata_from` wins last). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches `sample.metadata` — a linear run's metadata is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`), mirroring `UnstashInputOp(copy=True, remove=True)` — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); the stash family's charter is NARROWED to the two jobs cells cannot do — carrying a snapshot ACROSS a `Parallel` boundary (metadata rides the sample; cells raise at the boundary) and deliberately PERSISTING a snapshot into a sink's metadata (2026-07-18 consolidation); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. - **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `target_from`/`metadata_from` (fan-in slots, Mix field semantics), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **The Transform Taxonomy Is a GRID — field scope × call style, with COMBINABLE per-param bindings (`sampleflux.kinds`, 2026-07-18; EXTENDS the introspection mandate below):** A transform declares WHICH slice of the `Sample(input, target, metadata)` triple it processes and HOW it wants to be called, from its `__call__` signature alone. `SampleKind` is now the closed Literal `sample`/`pair`/`input`/`target`/`input_meta`/`target_meta`/`value`/`any`; `CallStyle` is `packed`/`unpacked`; `OpContract` carries both. **Named views are real NamedTuples in `sampleflux.sample`** — `Pair(input, target)`, `InputMeta(input, metadata)`, `TargetMeta(target, metadata)` (metadata typed the same dict-or-list `Metadata` duality as Sample) — recognised by `Sample.from_any` AND `classify_carrier` BEFORE the generic 2-tuple rule (a view IS a tuple; positional coercion would misread `(input, metadata)` as `(input, target)` — pinned). Bare-field intent uses the PEP-593 marks `INPUT`/`TARGET` (aliases `Input = Annotated[Any, INPUT]`, `Target = …`; mark your own type via `Annotated[np.ndarray, INPUT]`). **Detection (`op_contract`) — per-parameter BINDINGS:** arity counts REQUIRED positional params only (optional extras keep single-arg semantics — the back-compat guard). Arity 2–3 → unpacked with `OpContract.bindings`: each param resolves ANNOTATION-first (`InputMeta`/`TargetMeta`/`Input`/`Target` marks, `dict` → `metadata`), then NAME (`input`/`target`/`metadata`/`meta`), then the POSITIONAL default `(input, target, metadata)` — so `f(input, target)`, `f(input, metadata)`, `f(target, metadata)`, `f(input, target, metadata)` reproduce the classic rules AND every combination works: `f(im: InputMeta, tm: TargetMeta)` (input AND target each WITH metadata), `f(x: Input, tm: TargetMeta)`, name-reordered `f(target, input)`, … `accepts` stays the covered-fields grid SUMMARY (`_bindings_summary`: i+t+m→sample, i+m→input_meta, …). Arity 1 → packed, scope from the annotation (`dict` → the new `metadata`-only scope; the `MetaDict` alias/`METADATA` mark exist for explicitness); untyped single stays `any` (NEVER name-sniff a single param — existing ops are untouched). `CALL_STYLE` joins the class-attr escape hatches. **Binding + merge-back:** unpacked ops route through `core._apply_bindings` — each argument bound per its binding (`_BIND_GET`), the result MUST be a same-arity tuple / a full `Sample` / `None` (a wrong arity OR a single named VIEW from a multi-binding op is a LOUD TypeError — a view IS a 2-tuple and would silently misread as two elements, guarded + pinned); each returned element merges per its binding (a view/2-tuple element on a `*_meta` binding replaces value+metadata, a bare element replaces only the value; metadata-bearing elements merge left-to-right, LAST write wins). Packed scopes in `core._apply_view`: `None` drops; a returned `Sample` takes over; `input`/`target` → bare value replaces the field; `metadata` → the dict in, a dict out (else loud error); `pair` → a `Pair` in, a 2-tuple out replaces input+target (metadata KEPT); the meta views receive the ACTUAL metadata dict (in-place mutation propagates). Packed `pair` binds a `Pair` (it IS a tuple, so plain-tuple-annotated ops index it identically while Pair-annotated ops get named fields). **Native fast lanes** (`_apply_op_native`): pair-op on a pair carrier and input-op on a bare value stay metadata-free native; every other non-any combination PROMOTES via the view-correct `from_any` (sticky). Default collates for `input_meta`/`target_meta` mirror the pair form (stacked value + list-of-dicts metadata). These NAMES are the vocabulary FluxStudio will surface as socket types (a later stage — TASKS.md). Pins: `tests/test_kinds.py::TestGridContracts`/`TestGridEngineBinding`. - **Op Kind Is INTROSPECTED, Never Declared in the Engine (`sampleflux.kinds`, 2026-07-17):** The native multi-type engine (`Flux(native=True)`, OPT-IN — the `native=False` default coerces to `Sample` exactly as before, so all consumers are untouched) carries `Sample` triplets, metadata-free **pairs** (2-tuples), and bare **values** through one pipeline, adapting each op via `op_contract(op)` → `OpContract(accepts, produces, expands)` cached per type: `__call__`'s first-param annotation (`Sample`→`sample`, `tuple[...]`→`pair`, missing/`Any`→`any`) and return annotation (`Iterator[...]`/`Iterable[...]`/`List[...]` → `expands=True` — a `Tuple` return is a PAIR, never an expansion). ANY introspection failure (lazy imports, unresolvable forward refs) degrades to `any` so an untyped/exotic op behaves exactly as today; the class attrs `SAMPLE_KIND_IN`/`SAMPLE_KIND_OUT`/`EXPANDS` are the explicit escape hatch and ALWAYS win. Adaptation rules (`core._apply_op_native`): a pair-op on a Sample gets `sample.to_pair()` and its returned pair merges back via `_replace` (METADATA PRESERVED); a sample-op on a pair/value gets a PROMOTED `Sample.from_any` view — promotion is one-way and STICKY (op-written metadata is never dropped); an any-op gets the carrier verbatim. `SampleKind` is a closed Literal (`sample`/`pair`/`value`/`any`; runtime tuple `SAMPLE_KINDS = get_args(...)` — one source of truth); `classify_carrier` is the runtime classifier (ONLY a 2-tuple is a pair). **Collation is the pluggable registry `sampleflux.collate`** (`register_collate(key)` / `get_collate` / `collate(items, key=None)` — key defaults to the detected kind): sampleflux registers `"sample"` (list-form batched metadata — the `is_batched` convention) / `"pair"` / `"value"` defaults; consumers register task aliases ADDITIVELY and their divergent conventions (deltaid/raidar `{"per_sample": …}`) are deliberately NOT unified (TASKS.md follow-up). Pins: `tests/test_kinds.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op whose return annotation is `Iterator[...]`/`Iterable[...]`/`List[...]` (or that carries `EXPANDS = True`) is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op's contract expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `HDF5WindowSource`/`RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. `_refresh_type` is applied per CHILD (`core._expand`). Pins: `tests/test_expanding_ops.py`. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `sampleflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `SAMPLEFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. -- **SigMF Is the Waveform Recording Carrier; Metadata Is QUERYABLE Without Array Loads (2026-07-17):** `sampleflux.storage.sigmf` holds the `SigMFSink`↔`SigMFSource` pair (one `.sigmf-data` + `.sigmf-meta` recording per sample; hand-rolled JSON — no `sigmf` dependency; dtype↔`core:datatype` via `_DTYPE_TO_SIGMF`, unsupported dtypes raise; optional `core:sha512`). sampleflux stays DOMAIN-NEUTRAL: the default encoder/decoder pass `core:*` keys verbatim and namespace everything else `sampleflux:`; the WAVEFORM vocabulary (`samplerate`↔`core:sample_rate` incl. the torchsig `sample_rate` spelling, `center_freq`→the capture's `core:frequency` — SCOPE resolves the torchsig per-signal collision, string `snr`→`waivefront:snr_raw`/`snr_db`, `{role}_regions`/`labels`↔annotations with `waivefront:role`) lives in **`waivefront.vocab`** and plugs in via the `meta_encoder`/`meta_decoder` hooks (dotted `module.attr` or `module:attr` paths, lazily resolved). A JSON-able target rides `sampleflux:target` (SigMF is input-centric; array targets skip with a debug note). **Queryability**: `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; implemented on `HDF5Source` (attrs + array-metadata shape/dtype STUBS), `ZarrGroupSource` (`.zattrs`), `SigMFSource` (meta JSON) — existing files queryable with NO rewrite) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND/composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration (the projection-module pattern). Entry points `sampleflux-storage-sigmf`/`-query`. No index sidecar in v1 (TASKS.md). Pins: `tests/test_sigmf.py`, `waivefront/tests/test_vocab.py`. +- **Metadata Is QUERYABLE Without Array Loads (`sampleflux.storage.query`, 2026-07-17):** `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; implemented on `HDF5Source` (attrs + array-metadata shape/dtype STUBS) and `ZarrGroupSource` (`.zattrs`) — existing files queryable with NO rewrite; the protocol is STRUCTURAL, so external storage sources (e.g. waivefront's `SigMFSource`) implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration (the projection-module pattern). Entry point `sampleflux-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair (`SigMFSink`/`SigMFSource`) MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; sampleflux keeps ZERO knowledge of it. Pins: `tests/test_query.py`, `waivefront/tests/test_sigmf.py`. - **Field Projection (`sampleflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `sampleflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). - **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`sampleflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing sampleflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a sampleflux dependency for this. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The sampleflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the 1-D FFT family `FourierOp` / `InverseFourierOp` / `FftShiftOp` / `IfftShiftOp` (numpy + torch variants in `sampleflux.ops.{numpy,torch}`; `FourierOp`/`InverseFourierOp` take real OR complex input → always-complex output, with `n`/`axis`(`dim`)/`norm` + a `shift` flag = post-`fftshift` on the forward, pre-`ifftshift` on the inverse so they invert each other; the torch FFT/IFFT ops promote half precision to `float32` first; `FftShiftOp`/`IfftShiftOp` are the same shift logic standalone — pure dtype-preserving bin rearrangements that work on any array, e.g. a 2-D spectrogram), the FFT **windowing + unit-scaling** ops `WindowOp` / `SpectrumScalingOp` (numpy + torch; `sampleflux.ops.{numpy,torch}`) plus `FourierOp(window=…, scaling=…, sample_rate=…)` — `WindowOp` applies a `sampleflux.windows.WindowName` taper (Hann/Hamming/Blackman-Harris/flat-top/Kaiser/…) and stashes the coherent-gain correction (`window_sum` `S1`, `window_sum_sq` `S2`, ENBW) into the metadata; `SpectrumScalingOp` reads it (rectangular `S1=S2=N` if absent) to emit amplitude (V, `X/S1`) / power (V², `|X|²/S1²`) / density (V²/Hz, `|X|²/(Fs·S2)`); the calibration math (`get_window`/`scale_spectrum` + the `WindowName`/`SpectrumScaling` Literals) lives in the **library module `sampleflux.windows`** (pure numpy — scipy is optional; NOT `@configurable`, no entry point, shared by both frameworks). `FourierOp`'s default `window="boxcar"`+`scaling="none"` is byte-identical to the old behaviour (no metadata stamped), and calibrated `scaling` requires the unscaled `norm="backward"` (a non-backward norm with `scaling != "none"` raises). `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, `Tee`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation) + `CaptureOutputOp` (`sampleflux.ops.capture` — applies a wrapped op, then records one or more of its `@output` attribute values into `metadata[key]`, reading THROUGH a `.target` wrapper so it composes with `ConfigureOp`; the capture half of FluxStudio's op-`@output`→param wiring, paired with `ConfigureOp(ops=[UnstashInputOp(key)])`, and STOCHASTIC-correct — it captures the value from the actual run, never recomputes), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots, the building blocks of FluxStudio's DAG→sequential export; the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `sampleflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["__taidal_stash*"]` + `include=["__taidal_stash_456:*"]` clears every auto-stash snapshot EXCEPT node 456's, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Tee`/`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`/`CaptureOutputOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots on the METADATA bus, kept ONLY for crossing a `Parallel` boundary or deliberately persisting a snapshot into a sink (graph wiring uses the context ops); the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `sampleflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["spec_*"]` + `include=["spec_keep"]` clears every bulky snapshot key EXCEPT the protected one, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `sampleflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). diff --git a/README.md b/README.md index 1fb2f4a..8cebd39 100644 --- a/README.md +++ b/README.md @@ -2,36 +2,18 @@ **SampleFlux** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. -Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquify`, and `SampleFlux`. +Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `SampleFlux`. ## 🚀 Key Features - **Functional Purity:** Transforms are simple Python callables. No complex base classes required. -- **Standardized Sample Triplet:** Standardizes on `(input, target, metadata)` for full traceability. -- **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context. -- **Advanced Storage:** Built-in support for high-performance backends: - - **HDF5**: Clean, efficient read/write. - - **Zarr**: Cloud-native, concurrent storage (Group and Batch modes). - - **Directory**: Robust concurrent writing for irregular data lengths. -- **Passive Introspection:** Automatically generates JSON manifests for visual orchestration in **FluxStudio**. +- **Standardized Sample Triplet:** Standardizes on `(input, target, metadata)` for full traceability — while the [transform taxonomy](docs/kinds.md) lets an op process just the slice it cares about (`input`, `pair`, `input_meta`, …) in whichever calling style its signature declares. +- **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-sample `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Flux` engine. +- **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. +- **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-samplefluxstoragequery) — filter stored datasets without loading a single array. +- **Passive Introspection:** ops declare [type contracts](docs/typespec.md) and are discoverable by category for visual editors and schema generators. - **100% Reproducibility:** Entire pipelines are serializable via **Confluid** manifests. -## 🎯 Design Goals & Requirements - -### Stream Engine -- **Functional API:** Provide a lazy, chainable pipeline API (`map`, `filter`, `batch`). -- **Standardized Samples:** Use the `Sample(input, target, metadata)` triplet as the primary data unit. -- **Parallel Execution:** Support high-performance multiprocess execution via `.parallel(workers=N)` using the `spawn` context. - -### Storage -- **High-Performance Sinks:** Native support for HDF5 (sequential), Zarr (concurrent), and Directory (irregular) storage. -- **JointFlux Pattern:** Support aggregating multiple heterogeneous data sources into a single stream, preserving per-source transform chains. - -### Metadata & Discovery -- **Passive Introspection:** Automatically discover available tools and ops for serialized manifests. -- **Discovery Categories:** `@configurable` classes are tagged with a confluid `category` (sources `HuggingFaceSource`/`DatasetSplit` → `source`, engines `Flux`/`JointFlux` → `engine`, concrete `Sample→Sample` ops → `op`, storage **sinks** `HDF5Sink`/`ZarrGroupSink`/`ZarrBatchSink`/`DirectorySink` → `sink` (FluxStudio surfaces them as `DatasetProcessor` sink nodes; their read-back **sources** stay UNcategorised); `FilterOp`/`WrappedOp` are deliberately UNcategorised) so tools like navigaitor's `list_configurable_classes(category=...)` enumerate them by kind. -- **Serialization Symmetry:** Ensure full-pipeline states are serializable and reconstructible via Confluid. - ## 🛠 Quick Start ```python @@ -55,565 +37,33 @@ for sample in flux: print(sample.input.shape) ``` -## 🏷 Type Specs - -`sampleflux.typespec` describes *what flows through a `Sample`* and lets ops declare what they accept/produce, so tools like FluxStudio can filter which nodes may connect. It is flexible by design — N-dimensional arrays across numpy/torch/tensorflow, **per-axis bounded ranges**, dtype families, images, and arbitrary Python types — and anything left unspecified defaults to `Any`. - -```python -from sampleflux.typespec import SampleType, ArrayType, Dim, PythonType, UnionType - -# "a 2-D float array whose first axis is 1–10, second axis any size" -ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="floating") -ArrayType.parse("3 h w", dtype="float32", framework="torch") # jaxtyping-style shorthand -ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") # an image convenience -``` - -`dtype`, `framework`/`frameworks`, and the image `layout` are **closed `Literal`s**, not bare strings — a typo is a type error and a UI / connection-validator enumerates the choices via `typing.get_args(...)`: - -- `Dtype` — concrete names (`"float32"`, `"int64"`, …); `DtypeFamily` — relaxed families (`"floating"`, `"numeric"`, …); `DtypeSpec = Dtype | DtypeFamily` is the `dtype` field type. -- `Framework = Literal["numpy", "torch", "tensorflow"]`, `ImageLayout = Literal["CHW", "HWC"]`. - -Authored dtypes must be canonical names; aliases / casing (`"double"`, `"FLOAT32"`) and exotic platform dtypes (`float128`) are runtime-only conveniences normalized by `canonical_dtype` — the single boundary where arbitrary input crosses into the typed domain. - -**Declare an op's contract** with the class attributes `ACCEPTS` / `PRODUCES` (each a `SampleType`; both default to `Any`, so annotating is optional and backward-compatible). No base class — transforms stay plain callables: - -```python -@configurable -class StandardizeOp: - ACCEPTS = SampleType(input=UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image")))) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - def __call__(self, sample): ... -``` - -Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime against a concrete inferred type); `compatible(consumer, producer)` is permissive (used at edit time — `Any`/unknown on either side passes). A `Sample`'s own type comes from `sample.describe()` — it returns a type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict) + `__spec__` (sidecar refinements), or infers one from the live data; attach a stored type with `sample.with_type(SampleType(...))`. - -## 🌀 Fourier Transform (`FourierOp` / `InverseFourierOp` / shift ops) - -A small **1-D FFT toolkit**, each op in a numpy variant (`sampleflux.ops.numpy`, on `np.ndarray`) and a torch variant (`sampleflux.ops.torch`, on `torch.Tensor`); the flat `from sampleflux.ops import …` resolves to the torch one (the package's torch-default convention, like `RescaleOp`): - -- **`FourierOp`** — the 1-D discrete Fourier transform (`numpy.fft.fft` / `torch.fft.fft`). -- **`InverseFourierOp`** — its inverse (`…fft.ifft`), back to the time domain. -- **`FftShiftOp`** / **`IfftShiftOp`** — center the zero-frequency component, and undo it (`…fft.fftshift` / `ifftshift`). - -`FourierOp` / `InverseFourierOp` accept **real *and* complex** signals; the raw transform is always complex (take `.real` downstream if you started real). With a unit `scaling` (see **Windowing & spectral units** below) `FourierOp` may instead emit a *real* power/density spectrum, so it declares a permissive `PRODUCES` (complex **or** floating). The shift ops are pure, dtype-preserving bin rearrangements (no FFT), so they work on any array — including an already-computed 2-D spectrogram. - -```python -import numpy as np -from sampleflux.sample import Sample -from sampleflux.ops.numpy import FourierOp, InverseFourierOp, FftShiftOp - -x = np.array([1.0, 2.0, 3.0, 4.0]) # real signal -spectrum = FourierOp()(Sample(input=x)).input # complex128, == np.fft.fft(x) - -xc = np.array([1 + 2j, 3 - 1j, 0j, -2 + 1j]) # complex signal — also supported -FourierOp(n=8, axis=-1, norm="ortho")(Sample(input=xc)) # zero-pad to 8, orthonormal scaling - -# Round-trip (forward then inverse recovers the input): -recovered = InverseFourierOp()(FourierOp()(Sample(input=x))).input.real # ≈ x - -# Center the spectrum for display — two equivalent ways: -centered = FftShiftOp()(FourierOp()(Sample(input=x))) # explicit, composable -centered = FourierOp(shift=True)(Sample(input=x)) # the one-node convenience flag -``` - -Parameters mirror `numpy.fft.fft` / `torch.fft.fft`: `n` (output length — zero-pad/truncate), `axis` (numpy) / `dim` (torch) — the single transform axis, default the last, so a `[B, N]` batch transforms per row — and `norm`, a closed `Literal["backward", "ortho", "forward"]` (use the **same** `norm` on the inverse to round-trip). Dtype promotion follows each framework: real `float32`/`complex64` → `complex64`, `float64`/integer/`complex128` → `complex128` (numpy) or `complex64` for integer (torch); the torch FFT/IFFT ops promote half precision (`float16`/`bfloat16`) to `float32` first because torch's FFT rejects it. Both transform ops take a `shift` flag — `FourierOp(shift=True)` applies `fftshift` **after** the transform, `InverseFourierOp(shift=True)` applies `ifftshift` **before** it — so the two invert each other exactly (the standalone `FftShiftOp`/`IfftShiftOp` are the same logic, decoupled, for centering arrays that didn't come from `FourierOp`). - -### Windowing & spectral units (`WindowOp` / `SpectrumScalingOp` / `FourierOp(window=…, scaling=…)`) - -A raw FFT is **uncalibrated** — to read a spectrum in real units you must taper the signal with a *window* (to control spectral leakage) and divide out the window's gain. SampleFlux ships this as two composable ops plus options on `FourierOp` (numpy **and** torch variants). The window + unit math lives in **`sampleflux.windows`** (pure numpy; `get_window` / `scale_spectrum` / the `WindowName` + `SpectrumScaling` Literals). - -- **`WindowOp(window=…)`** — multiplies the signal by a taper and **stashes the correction** (`window_sum` `S1=Σw`, `window_sum_sq` `S2=Σw²`, `window_enbw_bins`, `window_coherent_gain`) into the metadata for a later scaling step. Windows: `boxcar` (rectangular/none), `bartlett`, `hann`, `hamming`, `blackman`, `blackmanharris`, `nuttall`, `flattop`, `kaiser`, `tukey`, `gaussian` — parametrized ones take `window_param` (Kaiser β / Tukey α / Gaussian σ); `periodic=True` (default) is the DFT-even form correct for FFT analysis. -- **`SpectrumScalingOp(scaling=…)`** — turns a spectrum into physical units, reading `S1`/`S2` from the metadata (rectangular `S1=S2=N` if no window was applied): - - | `scaling` | output | formula | units | - |---------------|-------------------|------------------------|----------| - | `"none"` | complex (raw) | `X` | — | - | `"amplitude"` | complex | `X / S1` | V | - | `"power"` | real | `|X|² / S1²` | V² | - | `"density"` | real | `|X|² / (Fs·S2)` | V²/Hz | - - `density` uses `sample_rate` (Hz) → falls back to `metadata["samplerate"]` → `1.0` (per normalized frequency). `one_sided=True` folds a real signal's spectrum to one side (keep `0…N/2`, double the interior bins). - -- **`FourierOp(window=…, scaling=…, sample_rate=…)`** folds all three into one node. The default (`window="boxcar"`, `scaling="none"`) is byte-for-byte the old behaviour. Calibrated `scaling` assumes the unscaled transform, so combining it with a non-`"backward"` `norm` raises. - -```python -from sampleflux.ops.numpy import FourierOp, WindowOp, SpectrumScalingOp - -# one node — Hann-windowed power-spectral density in dBW/Hz-ready units: -psd = FourierOp(window="hann", scaling="density", sample_rate=122.88e6)(sample).input - -# …is exactly the explicit, composable chain: -psd = SpectrumScalingOp(scaling="density", sample_rate=122.88e6)( - FourierOp()(WindowOp(window="hann")(sample)) -).input -``` - -A unit-amplitude tone reads `amplitude` ≈ its amplitude and `power` ≈ amplitude²; `power` and `density` differ by the window's equivalent noise bandwidth in Hz (`Fs·S2/S1²`) — the calibration that makes a windowed FFT match a reference analyzer. - -## 🔎 Field Projection & Class Counting - -Walking a source for a single field (the classic case: counting classes from -*targets*) shouldn't pay to build the fields you don't need. `sampleflux.projection` -adds an opt-in protocol plus lazy helpers: - -```python -from sampleflux import project, iter_targets, num_classes -from sampleflux import ProjectionField # Literal["input", "target", "metadata"] - -# A source MAY implement SupportsProjection (`project(fields)`) to skip building -# unrequested fields — e.g. an image dataset reads only the label column for a -# target-only walk, never decoding an image. -for sample in project(my_source, ("target",)): - ... # sample.input is None; sample.target populated - -labels = list(iter_targets(my_source)) # lazy -n = num_classes(my_source) # max(class_id) + 1 — always walks -``` - -The field set is a **closed `Literal`**, `ProjectionField`, not a bare `str` — -so a typo is a type error, and a UI / form-spec / MCP schema enumerates the -choices straight from the annotation instead of hard-coding a parallel list: - -```python -from typing import get_args -get_args(ProjectionField) # ('input', 'target', 'metadata') -``` - -Sources that don't implement `SupportsProjection` still work via a correct -full-iteration fallback (just without the skip-decode speedup). `num_classes` is -a free function, not a `Flux` method: integer class-id semantics are -classification-specific, so the task-agnostic engine doesn't advertise it. - -### `LabelMap` — fittable name↔id encoding - -When a dataset's `target` is a class **name** rather than an integer id, `LabelMap` turns it into -the pinned encoding the `EncodeTargetOp` / `DecodeTargetOp` need — the *fittable* companion to -those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the -`class_names.json` format, and reload it at eval/predict so every stage shares one ordering: - -```python -from sampleflux import LabelMap, Flux - -lm = LabelMap.fit(iter_targets(train_source)) # {"bird": 0, "cat": 1, "dog": 2} -lm.num_classes # 3 -lm.label_names # ["bird", "cat", "dog"] (id -> name) -lm.save("class_names.json") # marainer's class_names.json format - -encoded = Flux(source=train_source, ops=[lm.encode_op()]) # targets are now ints - -# Later, at eval time — reload the SAME ordering instead of refitting: -lm2 = LabelMap.load("class_names.json") -``` - -`LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the -mapping is pinned, so train / eval / predict never disagree. `scikit-learn` backs `fit` (lazy-imported). - -## 🖼 Image Conversion (`sampleflux.ops.image`) - -The single, modality-agnostic "any value → image" layer — generic so every -project (waivefront spectrograms, any dataset preview, FluxStudio) reuses one -implementation. Domain-specific rendering (overlays, signal plots) stays in the -consuming package. - -```python -from sampleflux.ops.image import ConvertToImageOp, value_to_image - -# Op: sample.input (2-D map / CHW tensor / PIL / bool mask) -> PIL image. -op = ConvertToImageOp( - colormap="viridis", # closed `Colormap` Literal -> dropdown in FluxStudio, enum in navigaitor - width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size - flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top -) -sample = op(sample) # also publishes image_width_px / image_height_px to metadata - -# Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): -rgb = value_to_image(some_value, colormap="magma", max_size=512) - -# NormalizeToUint8Op: the standalone min-max value -> uint8 quantization step -# (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; -# set them to pin a fixed scale across samples (out-of-range values clamp). -from sampleflux.ops.image import NormalizeToUint8Op - -sample = NormalizeToUint8Op()(sample) # auto per-array min/max -sample = NormalizeToUint8Op(vmin=-80.0, vmax=0.0)(sample) # fixed dB window across a dataset -u8 = NormalizeToUint8Op.normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # the backing @staticmethod -``` - -`Colormap` / `COLORMAPS` / `value_to_image` / `sample_to_image` are re-exported -from `waivefront.visualizers` for backward compatibility. Pillow is a runtime -dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). - -## 📦 Storage Integration - -SampleFlux makes it easy to move data between different formats: - -```python -from sampleflux.storage.hdf5 import HDF5Source -from sampleflux.storage.zarr import ZarrGroupSink - -# Stream from HDF5 to Zarr in parallel -Flux.from_source(HDF5Source("input.h5")) \ - .parallel(workers=8) \ - .map(heavy_op) \ - .to_sink(ZarrGroupSink("output.zarr")) -``` - -### Sinks and their matching sources - -Every sink has a source that reads its layout back into `Sample` triplets: - -| Backend | Sink | Source | Round-trips | -|---|---|---|---| -| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | input + target + metadata | -| Zarr group (one group / sample) | `ZarrGroupSink` | `ZarrGroupSource` | input + target + metadata | -| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | input only (uniform shape) | -| Directory (one dir / sample) | `DirectorySink` | — | — | - -```python -from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource - -Flux(samples).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) -for sample in ZarrGroupSource("ds.zarr"): # input/target as before, metadata from .zattrs - ... -``` - -### Array-valued metadata (e.g. segmentation masks) - -`HDF5Sink` stores scalar/string metadata as HDF5 **attributes**, but HDF5 caps -attribute size — a large array (a segmentation mask, a per-sample weight map) put -in `Sample.metadata` would overflow that limit. So **array-valued metadata -(`np.ndarray` / `torch.Tensor`) is written as its own dataset** under a per-sample -group `{prefix}_meta/`, and `HDF5Source` merges it back into `Sample.metadata` -on read. This is fully backward-compatible: files written before this layout (no -`{prefix}_meta` group) read exactly as before. - -```python -sample = Sample(input=iq, target=label, metadata={"mask": mask_2d, "snr": 12.0}) -Flux([sample]).to_sink(HDF5Sink("ds.h5", overwrite=True)) -loaded = next(iter(HDF5Source("ds.h5"))) -loaded.metadata["mask"] # the full array, byte-exact (not a truncated repr) -loaded.metadata["snr"] # scalar, via attributes as before -``` - -## ✂️ Train / Val / Test Splitting - -`DatasetSplit` partitions any indexable source (implementing `__len__` and `__getitem__`) into reproducible **train / val / test** views. It is a `source` (`category="source"`) — it yields `Sample`s and is wired into a trainer's `source:` slot — and it applies no ops, so it's a source, not an engine. - -**Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: - -```python -from sampleflux import DatasetSplit -split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) -split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% -``` - -The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: - -```yaml -hf_train: !class:sampleflux.sources.HuggingFaceSource() - path: mnist - split: train - -my_split: !class:sampleflux.sources.DatasetSplit() - source: !ref:hf_train - val_fraction: 0.1 - test_fraction: 0.1 - seed: 42 - -train_set: !class:sampleflux.core.Flux() { source: !ref:my_split.train } -val_set: !class:sampleflux.core.Flux() { source: !ref:my_split.val } -test_set: !class:sampleflux.core.Flux() { source: !ref:my_split.test } -``` - -Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). - -**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `sampleflux.SplitName`. - -```yaml -val_set: !class:sampleflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 -``` - -### Range & concatenation sources - -- **`RangeSource(source, start, end)`** — a contiguous index slice `[start:end)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. - - ```yaml - first_half: !class:sampleflux.sources.RangeSource() - source: !ref:hf_train - start: 0 - end: 5000 - ``` - -- **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointFlux`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. - - ```yaml - combined: !class:sampleflux.sources.ConcatSource() - sources: - - !ref:train_main - - !ref:extra_shard - ``` - -**HuggingFace native slicing** (alternative, no SampleFlux split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. - -> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. - -> **Lazy & zero-arg construction** — `HuggingFaceSource` follows the workspace lazy-init convention: the constructor does no work (no network), so `HuggingFaceSource()` is valid and building one is free. The dataset is downloaded only on first access to the read-only `.dataset` property (cached thereafter; reset `_dataset` to reload), and `.resolved_metadata_features` (the `"*"` expansion) is derived lazily from the loaded columns. `path` is therefore optional at construction and validated lazily — accessing `.dataset` with an empty `path` raises a clear `ValueError`. - -## 🔁 Reattach an ops-only YAML (`Flux.from_ops_yaml`) - -A `{ops: [!class:…()]}` document — e.g. one exported from a FluxStudio canvas (`fluxstudio export …`) — can be attached to any source: - -```python -from sampleflux import Flux -from sampleflux.sources import HuggingFaceSource - -flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) -``` - -The helper **materializes** the deferred `!class:` markers before attaching (via `confluid.materialize`) — necessary because `confluid.load` leaves markers nested under a mapping key deferred, and a `Flux` rejects deferred markers at iteration by design. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. - -## 🎛 Per-sample op parameters (`ConfigureOp` / `CaptureOutputOp`) - -Some op parameters are only known *per sample*. Two composable ops cover this — both are what FluxStudio emits when you wire a value into an op parameter on the canvas: - -- **`ConfigureOp(ops, target, param, key)`** — runs `ops` on the sample as a side-branch; the chain's final `sample.input` is written to `metadata[key]` and injected as `target.`, then `target` is applied. Use it when the value is *derived from the sample itself* (e.g. a threshold from the sample's own max). -- **`CaptureOutputOp(op, output|captures, key)`** — applies `op`, then records one or more of its `@output` attribute values into `metadata[key]`. The value is captured from the **actual run**, so it works for *stochastic* outputs (a random draw) that can't be recomputed. It reads through a `.target` wrapper, so it composes with `ConfigureOp`. - -Together they express "feed one op's runtime `@output` into a later op's parameter" — capture the output, then unstash it into the parameter per sample: - -```yaml -ops: - # NoiseFloorOp draws an SNR each call; capture it into metadata. - - !class:sampleflux.ops.capture.CaptureOutputOp - op: !class:waivefront.torchsig.processing.NoiseFloorOp {} - output: applied_snr_db - key: __captured_snr - # …then inject the captured value into a later op's `noise_power_db` per sample. - - !class:sampleflux.ops.configure.ConfigureOp - ops: - - !class:sampleflux.ops.stash.UnstashInputOp { key: __captured_snr } - target: !class:waivefront.torchsig.processing.NoiseFloorOp {} - param: noise_power_db -``` - -## 🗺 Flow documents & the FlowGraph engine (`sampleflux.flow`) - -The **readable authoring form** of a graph pipeline is a `flow:` document — named steps where a step's name is how later steps reference its result: - -```yaml -flow: - spec: !class:waivefront.SpectrogramOp() # input: the source sample - rescaled: !class:sampleflux.ops.numpy.RescaleOp() # input: previous step - masked: !class:waivefront.SegmentOp() {from: spec} # 2nd reader of spec = fan-out - thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: masked} - denoised: !class:waivefront.torchsig.processing.NoiseFloorOp() - from: rescaled - bind: {low_level: thresh} # per-sample param := thresh's result - out: {from: denoised, target_from: masked} # pure fan-in (no op) -outputs: out -``` - -Step grammar (four reserved keys, stripped before the op is built): **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document order is the schedule and cycles are inexpressible); **`target_from:`/`metadata_from:`** — fan-in slots (a step result contributes its corresponding field; metadata merges last-write-wins); **`bind:`** — `{param: step}` per-sample parameters (a step name = its result's `input`; `step.attr` = the step op's live `@output`, stochastic-correct). A plain-mapping step with no op (`out: {from: a, target_from: b}`) is a pure fan-in; `{}` is the identity (names the source). Cell lifetimes are **automatic** in both forms. - -Two engines, one contract — **bidirectional conversion with execution parity**: - -```python -from sampleflux import Flux, FlowGraph, to_ops, from_ops - -graph = FlowGraph.from_yaml("graph.yaml", source=src) # native named-step engine -flux = Flux.from_flow_yaml("graph.yaml", source=src) # same graph, LOWERED to the - # flat context-ops list (serial) -ops = to_ops(graph.steps, graph.output_step) # flow -> flat ops -flow2 = from_ops(ops) # flat ops -> flow (lifting) -``` - -`FlowGraph` is a `torch.utils.data.Dataset` like `Flux` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Flux's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. See `examples/flow_graph.py` for the full round-trip. +## 📚 Documentation -## 🕸 Graph pipelines on a flat op list (Context ops) - -A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-sample **`Context`** (a named-cell store, `sampleflux.context`) around each sample's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never touches `sample.metadata` — the metadata bus stays byte-identical to a linear run. - -| Op | Semantics | +| Page | Covers | |---|---| -| `Save(name)` | snapshot the stream sample into a cell (pass-through) — the fork point | -| `Use(name, drop=False)` | stream := the cell's value; deep-copies unless `drop` frees the cell (move) | -| `Drop(names)` | free cells explicitly | -| `Apply(op, param, source, drop=False)` | set `op.` from a cell's value, then apply `op` | -| `Capture(op, output, name)` | apply `op`, record its live `@output` into a cell (stochastic-correct) | -| `Mix(input_from, target_from, metadata_from, drop)` | fan-in: compose a sample from cells + the incoming sample | - -```yaml -ops: - - !class:sampleflux.ops.context.Save(name=fork) # fork the stream - - !class:waivefront.SpectrogramOp() # branch A rides the stream - - !class:sampleflux.ops.context.Save(name=branch_a) - - !class:sampleflux.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork - - !class:waivefront.SegmentOp() - - !class:sampleflux.ops.context.Mix(target_from=branch_a) # fan-in - drop: [branch_a] -``` - -A straight sequence needs none of this — a bare `ops:` list stays exactly as before. Outside an engine (a hand-rolled loop), activate a Context explicitly: - -```python -from sampleflux.context import Context, activate - -with activate(Context()): - for op in ops: - sample = op(sample) -``` - -Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's `input`, `Mix` reads each cell's corresponding field. Copy discipline mirrors the stash family: stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to (see `sampleflux.flow`). - -## 🎭 Multi-type carriers & the collate registry (`sampleflux.kinds` / `sampleflux.collate`) - -Pipelines can carry more than `Sample` triplets: **`Flux(native=True)`** (opt-in) keeps each carrier's own kind — a metadata-free **pair** (`(image, label)`, `(tensor, mask)`, `(tensor, coco_dict)`) or a bare **value** — and adapts every op via its introspected contract: +| [docs/kinds.md](docs/kinds.md) | The transform taxonomy (field scope × call style), multi-type carriers (`Flux(native=True)`), the collate registry, 1→N expanding ops | +| [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Flux.from_ops_yaml` | +| [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | +| [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources, array-valued metadata, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | +| [docs/typespec.md](docs/typespec.md) | The flexible array/type system, `ACCEPTS` / `PRODUCES` op contracts, closed `Literal` vocabularies | +| [docs/projection.md](docs/projection.md) | Field projection (`SupportsProjection`), lazy target walks, `num_classes`, the fittable `LabelMap` | +| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImageOp`, `NormalizeToUint8Op`), array introspection helpers | +| [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | -```python -from confluid import configurable -from sampleflux import Flux, Sample, op_contract - -@configurable -class NormalizePair: # a pair-native op — no metadata anywhere - def __call__(self, pair: tuple) -> tuple: - img, label = pair - return img / 255.0, label - -@configurable -class StampOp: # a classic Sample op — unchanged - def __call__(self, sample: Sample) -> Sample: ... - -flux = Flux(source=[(img_a, 3), (img_b, 7)], ops=[NormalizePair(), StampOp()], native=True) -# NormalizePair receives the raw pair; StampOp receives a PROMOTED Sample view -# (promotion is one-way and sticky, so op-written metadata is never dropped). - -op_contract(NormalizePair()) # OpContract(accepts='pair', produces='pair', expands=False) -``` - -Detection reads the `__call__` annotations (`Sample` → sample-op, `tuple[...]` → pair-op, untyped → works-on-anything — **untyped ops behave exactly as today**); the class attrs `SAMPLE_KIND_IN` / `SAMPLE_KIND_OUT` / `EXPANDS` override detection where introspection can't see. `native=False` (the default) coerces everything to `Sample` exactly as before — no consumer changes. - -**Collation** is a pluggable registry keyed by representation: - -```python -from sampleflux import collate, get_collate, register_collate - -batch = collate(list(flux)) # dispatches on the detected kind -@register_collate("yolo") # task aliases are additive -def yolo_collate(items): ... -loader = DataLoader(flux, collate_fn=get_collate("yolo")) -``` - -Defaults: `"sample"` (stacked input/target + list-form batched metadata — the `is_batched` convention), `"pair"` (`(stacked_inputs, stacked_targets)`), `"value"`. Consumer collates (classification/segmentation/detection) register additively and keep their own conventions. - -## 🌱 1→N expanding ops (iterable-only pipelines) - -An op may return **several** carriers — a windowing op splitting one capture into N windows is just a generator-returning op: - -```python -from typing import Iterator - -@configurable -class WindowOp: - def __call__(self, sample: Sample) -> Iterator[Sample]: - for w in sliding_windows(sample.input, self.size, self.stride): - yield sample._replace(input=w) -``` - -Expansion is detected from the return annotation (`Iterator[...]` / `Iterable[...]` / `List[...]`; or the explicit `EXPANDS = True` marker) and flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. - -A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(flux)` / `flux[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(flux)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Flux` engine. - -## 📡 SigMF recordings & queryable metadata (`sampleflux.storage.sigmf` / `.query`) - -**SigMF** ([sigmf.org](https://sigmf.org)) is the open Signal Metadata Format — a raw binary sample file (`.sigmf-data`) plus a JSON metadata file (`.sigmf-meta`) with `global`/`captures`/`annotations` sections. `SigMFSink` ↔ `SigMFSource` are the sampleflux carrier pair (siblings of HDF5/Zarr — additive, no migration): - -```python -from sampleflux.storage.sigmf import SigMFSink, SigMFSource +## 🧭 Scope: a modality-neutral engine -sink = SigMFSink(path="recordings/", meta_encoder="waivefront.vocab.to_sigmf", checksum=True) -sink.write(sample) # complex64 IQ -> cf32_le + JSON metadata -source = SigMFSource(path="recordings/", meta_decoder="waivefront.vocab.from_sigmf") -``` - -sampleflux stays domain-neutral (unrecognised keys ride the namespaced `sampleflux:` extension); the *waveform vocabulary* — `samplerate` ↔ `core:sample_rate`, `center_freq` ↔ the capture's `core:frequency`, `{role}_regions`/`{role}_labels` ↔ SigMF annotations, the torchsig naming collisions — plugs in from the domain package via the `meta_encoder`/`meta_decoder` hooks. - -**Queryable metadata** — filter stored samples by metadata predicates *without loading arrays*: sources implementing the `SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs/`.zattrs`/meta-JSON — `HDF5Source`, `ZarrGroupSource`, and `SigMFSource` all do, so **existing HDF5/Zarr files are queryable with no rewrite**: - -```python -from sampleflux.storage.query import MetadataFilterSource - -view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="snr_db > 10 and drone == 'DJI'") -len(view) # matches counted from a metadata-only scan -flux = Flux(source=view, ops=[...]) # arrays load ONLY for matching samples -``` +SampleFlux deliberately contains **no domain-specific code** — every op, source and sink in this package is meaningful for any modality (arrays, tensors, images, generic metadata). Domain packages build on it and keep their own vocabulary: -`where` uses the FormulaOp restricted namespace with metadata keys as variables (a missing key = non-matching, a malformed expression fails loudly); a programmatic `predicate=` composes with AND; sources without the protocol fall back to full-iteration filtering. - -## 🔗 Paired Join (Binary ↔ Annotations) - -`AnnotationJoinSource` joins a data `DataSource` (e.g. raw binary samples) with a sidecar mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern — typically re-attaching a LabelStudio export back onto the raw samples for training. Three join policies cover the scenarios we actually see in ML research: - -| Policy | Iterates | Use case | -|---|---|---| -| `left_outer` (default) | Every data sample; attaches annotation when the key matches | Process everything, use labels where available | -| `inner` | Only data samples whose key is in the store | Train/evaluate on the labeled subset | -| `right_driven` | Every key in the annotation store; resolves the data sample via `data_resolver(key, data)` | Very sparse labels where full-data enumeration is costly | - -```yaml -data: !class:waivefront.rfuav.data.source.RFUAVSource() - root: /Volumes/Data/RFUAV - window_samples: 1000000 - -labels: !class:annotaide.store.JSONFileAnnotationStore() - path: /Volumes/Data/RFUAV-labels - -paired: !class:sampleflux.paired.AnnotationJoinSource() - data: !ref:data - annotations: !ref:labels - key_fn: "waivefront.rfuav.keys:sample_window_key" - policy: left_outer -``` - -Annotation records are **flattened into `Sample.metadata`**, so a detection record `{bboxes, labels, scores}` shows up as three independent metadata keys. Two `metadata` keys are always populated: `annotated: bool` and `annotation_key: str`. Optional `prefix` and `store_full_under` parameters shape the layout. The parameters are typed, not `Any`: `data` is an `Iterable[Any]` (any source), `annotations` is an `AnnotationStore` (a read-mapping `key → record` — a `dict` or annotaide's `JSONFileAnnotationStore` both qualify), and `policy` is a fixed `Literal["left_outer", "inner", "right_driven"]`. Both the store shape and the policy are validated at construction. - -### Coarser-granularity keys (broadcast and slicing) - -`key_fn` is free to return a coarser key than the sample granularity. When multiple data samples map to the same key, they all look up the same record: - -- **Without `extract_fn`** — the record is broadcast identically into every matching sample's metadata (e.g. a scalar pack-level class label inherited by every window of that pack). -- **With `extract_fn`** — the record is projected per sample. The callable is invoked as `extract_fn(record, sample) -> dict | None`; returning `None` marks the sample unannotated (and filters it under `policy="inner"`). Use this when a pack-level annotation carries time-ranged content that must be trimmed to each window's bounds. - -Multi-granularity joins (e.g. pack-level + window-level annotations merged together) compose by chaining `AnnotationJoinSource` instances — the output of one is itself a `DataSource` that the next can consume. - -### Callable resolution - -`key_fn`, `extract_fn`, and `data_resolver` all accept either a callable **or** a `"module:function"` string path resolved through `sampleflux.discovery.resolve_callable`. The string form is what survives YAML round-trip via Confluid. - -See [`examples/paired_annotations.py`](examples/paired_annotations.py) for a runnable end-to-end walkthrough of all four scenarios. +- Signal/waveform work (1-D FFT + windowing ops, SigMF recording storage, spectrograms, the annotation-join source) lives in the **waivefront** package. +- Task-specific trainers, collates and models live in their consuming projects. ## 🌐 Ecosystem Integration -SampleFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines. - -### Hugging Face (Community & Standardized Datasets) -- **Use Hugging Face for:** Accessing community datasets and leveraging the `datasets` library for efficient Arrow/Parquet loading. -- **Integration:** Use SampleFlux to transform `datasets.Dataset` objects into standardized `Sample` triplets, ensuring metadata traceability that often goes missing in simple dictionary-based records. -- **`metadata_features` (which columns ride along on `Sample.metadata`):** `None` / `[]` keep none (the default); an explicit list keeps exactly those columns; and the sentinel **`"*"`** (or `["*"]`) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load. It stays opt-in so existing configs are unchanged. - -```yaml -hf_train: !class:sampleflux.sources.HuggingFaceSource() - path: mnist - input_feature: image - target_feature: label - metadata_features: ["*"] # keep every other column as metadata (here: none extra beyond hf_path/hf_split) -``` +SampleFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: -### SampleFlux (The Functional Engine) -- **Use SampleFlux for:** The "inner loop" of your experiment. When you need high-performance multiprocess streaming, per-sample metadata preservation, and 100% reproducible pipelines via **Confluid** serialization. +- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into `Sample` triplets with full metadata traceability (see [docs/sources.md](docs/sources.md)). +- **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node, every run reproducible. +- **PyTorch**: `Flux` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#multi-type-carriers--the-collate-registry-samplefluxcollate). ## 🔧 Installation diff --git a/docs/configure.md b/docs/configure.md new file mode 100644 index 0000000..2864db9 --- /dev/null +++ b/docs/configure.md @@ -0,0 +1,35 @@ +# Per-sample op parameters (`ConfigureOp` / `Apply` / `Capture`) + +Some op parameters are only known *per sample*. Two mechanisms cover this: + +- **`ConfigureOp(ops, target, param, key)`** — runs `ops` on the sample as a side-branch; the chain's final `sample.input` is written to `metadata[key]` and injected as `target.`, then `target` is applied. Use it when the value is *derived from the sample itself* (e.g. a threshold from the sample's own max) — the whole derivation reads as one node/YAML block. +- **`Capture` + `Apply`** (`sampleflux.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. + +```yaml +ops: + # AugmentOp draws a random level each call; capture it into a cell. + - !class:sampleflux.ops.context.Capture + op: !class:mypackage.ops.AugmentOp {} # any op exposing a confluid @output + output: applied_level + name: __captured_level + # …then inject the captured value into a later op's parameter per sample. + - !class:sampleflux.ops.context.Apply + op: !class:mypackage.ops.CompensateOp {} + param: level + source: __captured_level +``` + +A self-contained `ConfigureOp` example — derive a per-sample threshold from the sample's own statistics: + +```yaml +ops: + - !class:sampleflux.ops.configure.ConfigureOp + ops: + - !class:sampleflux.ops.numpy.MaxOp {} + - !class:sampleflux.ops.formula.FormulaOp { formula: "a * 0.5" } + target: !class:sampleflux.ops.numpy.ThresholdOp {} + param: low_level + key: derived_threshold +``` + +`ConfigureOp` also stamps the derived value into `metadata[key]` (traceability — it persists into a sink); `Capture`/`Apply` move values through the per-sample Context, which never touches `sample.metadata`. diff --git a/docs/graph.md b/docs/graph.md new file mode 100644 index 0000000..ad248f8 --- /dev/null +++ b/docs/graph.md @@ -0,0 +1,92 @@ +# Graph pipelines — flow documents, the FlowGraph engine and Context ops + +## Flow documents & the FlowGraph engine (`sampleflux.flow`) + +The **readable authoring form** of a graph pipeline is a `flow:` document — named steps where a step's name is how later steps reference its result: + +```yaml +flow: + scaled: !class:sampleflux.ops.numpy.RescaleOp() # input: the source sample + norm: !class:sampleflux.ops.numpy.StandardizeOp() # input: previous step + mask: !class:sampleflux.ops.numpy.ThresholdOp(low_level=0.5) {from: scaled} # 2nd reader of `scaled` = fan-out + thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: norm} + gated: !class:sampleflux.ops.numpy.ThresholdOp() + from: scaled + bind: {low_level: thresh} # per-sample param := thresh's result + out: {from: gated, target_from: mask} # pure fan-in (no op) +outputs: out +``` + +Step grammar (four reserved keys, stripped before the op is built): + +- **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document order is the schedule and cycles are inexpressible). +- **`target_from:` / `metadata_from:`** — fan-in slots (a step result contributes its corresponding field; metadata merges last-write-wins). +- **`bind:`** — `{param: step}` per-sample parameters (a step name = its result's `input`; `step.attr` = the step op's live `@output`, stochastic-correct). + +A plain-mapping step with no op (`out: {from: a, target_from: b}`) is a pure fan-in; `{}` is the identity (names the source). Cell lifetimes are **automatic** in both forms. + +Two engines, one contract — **bidirectional conversion with execution parity**: + +```python +from sampleflux import Flux, FlowGraph, to_ops, from_ops + +graph = FlowGraph.from_yaml("graph.yaml", source=src) # native named-step engine +flux = Flux.from_flow_yaml("graph.yaml", source=src) # same graph, LOWERED to the + # flat context-ops list (serial) +ops = to_ops(graph.steps, graph.output_step) # flow -> flat ops +flow2 = from_ops(ops) # flat ops -> flow (lifting) +``` + +`FlowGraph` is a `torch.utils.data.Dataset` like `Flux` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Flux's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. See `examples/flow_graph.py` for the full round-trip. + +## Graph pipelines on a flat op list (Context ops) + +A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-sample **`Context`** (a named-cell store, `sampleflux.context`) around each sample's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never touches `sample.metadata` — the metadata bus stays byte-identical to a linear run. + +| Op | Semantics | +|---|---| +| `Save(name)` | snapshot the stream sample into a cell (pass-through) — the fork point | +| `Use(name, drop=False)` | stream := the cell's value; deep-copies unless `drop` frees the cell (move) | +| `Drop(names)` | free cells explicitly | +| `Apply(op, param, source, drop=False)` | set `op.` from a cell's value, then apply `op` | +| `Capture(op, output, name)` | apply `op`, record its live `@output` into a cell (stochastic-correct) | +| `Mix(input_from, target_from, metadata_from, drop)` | fan-in: compose a sample from cells + the incoming sample | + +```yaml +ops: + - !class:sampleflux.ops.context.Save(name=fork) # fork the stream + - !class:sampleflux.ops.numpy.StandardizeOp() # branch A rides the stream + - !class:sampleflux.ops.context.Save(name=branch_a) + - !class:sampleflux.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork + - !class:sampleflux.ops.numpy.ThresholdOp + low_level: 0.5 + - !class:sampleflux.ops.context.Mix(target_from=branch_a) # fan-in + drop: [branch_a] +``` + +A straight sequence needs none of this — a bare `ops:` list stays exactly as before. Outside an engine (a hand-rolled loop), activate a Context explicitly: + +```python +from sampleflux.context import Context, activate + +with activate(Context()): + for op in ops: + sample = op(sample) +``` + +Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's `input`, `Mix` reads each cell's corresponding field. Copy discipline mirrors the stash family: stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to. + +> **What about the stash family?** `sampleflux.ops.stash` (`StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp`) snapshots a field into `sample.metadata` instead of a cell. It is NOT a wiring mechanism — the context ops are — and remains only for the two jobs cells cannot do: carrying a snapshot **across a `Parallel` boundary** (metadata rides the sample through the stream split; cells deliberately raise there) and deliberately **persisting a snapshot into a sink**. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. + +## Reattach an ops-only YAML (`Flux.from_ops_yaml`) + +A `{ops: [!class:…()]}` document — e.g. one exported by an external pipeline-authoring tool — can be attached to any source: + +```python +from sampleflux import Flux +from sampleflux.sources import HuggingFaceSource + +flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) +``` + +The helper **materializes** the deferred `!class:` markers before attaching (via `confluid.materialize`) — necessary because `confluid.load` leaves markers nested under a mapping key deferred, and a `Flux` rejects deferred markers at iteration by design. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. diff --git a/docs/image.md b/docs/image.md new file mode 100644 index 0000000..41ab683 --- /dev/null +++ b/docs/image.md @@ -0,0 +1,33 @@ +# Image conversion (`sampleflux.ops.image`) + +The single, modality-agnostic "any value → image" layer — generic so every consuming project (spectrogram previews, dataset browsers, GUI viewers) reuses one implementation. Domain-specific rendering (overlays, signal plots) stays in the consuming package. + +```python +from sampleflux.ops.image import ConvertToImageOp, value_to_image + +# Op: sample.input (2-D map / CHW tensor / PIL / bool mask) -> PIL image. +op = ConvertToImageOp( + colormap="viridis", # closed `Colormap` Literal -> enumerable in GUIs / schemas + width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size + flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top +) +sample = op(sample) # also publishes image_width_px / image_height_px to metadata + +# Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): +rgb = value_to_image(some_value, colormap="magma", max_size=512) + +# NormalizeToUint8Op: the standalone min-max value -> uint8 quantization step +# (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; +# set them to pin a fixed scale across samples (out-of-range values clamp). +from sampleflux.ops.image import NormalizeToUint8Op + +sample = NormalizeToUint8Op()(sample) # auto per-array min/max +sample = NormalizeToUint8Op(vmin=-80.0, vmax=0.0)(sample) # fixed dB window across a dataset +u8 = NormalizeToUint8Op.normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # the backing @staticmethod +``` + +`Colormap` / `COLORMAPS` / `value_to_image` / `sample_to_image` are re-exported from `waivefront.visualizers` for backward compatibility. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). + +## Introspection helpers + +Pure library functions (not ops) also live here, backing viewer tooling: `select_channel` (reduce an array/tensor to a 2-D float32 map for one channel; negative = mean across channels), `channel_count`, `array_histogram` (finite-only binning + summary stats, JSON-safe), `confusion_matrix_payload` / `confusion_matrices_payload` (render payloads for every confusion-matrix-shaped entry in a metrics result), and `draw_text` (text → `(H, W, 3)` uint8 image with word-wrap and 9-grid anchoring, plus the closed `TextPosition` Literal). diff --git a/docs/kinds.md b/docs/kinds.md new file mode 100644 index 0000000..42d6f08 --- /dev/null +++ b/docs/kinds.md @@ -0,0 +1,103 @@ +# The transform taxonomy, multi-type carriers & expanding ops (`sampleflux.kinds`) + +## What an op processes, how it's called + +A **sample** is the triple `(input, target, metadata)`; the classic AI tuple is the **pair** `(input, target)`. A transform declares — via its `__call__` signature alone — exactly which *slice* of the triple it processes, and the engine binds that view and merges the result back (untouched fields preserved): + +| scope | without metadata | with metadata | +|---|---|---| +| input only | `input` — the bare value | `input_meta` — `InputMeta(input, metadata)` | +| target only | `target` — the bare value | `target_meta` — `TargetMeta(target, metadata)` | +| both | `pair` — `(input, target)` / `Pair` | `sample` — the full `Sample` | +| metadata only | — | `metadata` — the bare dict (`m: dict` / `MetaDict`) | + +Each scope works in **two calling styles** — packed (one argument) or unpacked (the fields as separate arguments) — and unpacked arguments COMBINE freely: each parameter binds its own view (annotation first, then the name, then the classic `f(input, target, metadata)` positional defaults): + +```python +class A: # bare input value — any array/tensor/dict; target+metadata pass through + def __call__(self, x: Input): return x / 255.0 # Annotated[T, INPUT] keeps a real T + +class B: # the classic AI signature, unpacked + def __call__(self, input, target): return aug(input), target + +class C: # input with its metadata, unpacked (2nd arg named `metadata`/`meta`) + def __call__(self, input, metadata): return crop(input, metadata["roi"]), metadata + +class D: # packed named view + def __call__(self, v: TargetMeta) -> TargetMeta: return TargetMeta(encode(v.target), v.metadata) + +class E: # the full triple, unpacked + def __call__(self, input, target, metadata): return input, target, {**metadata, "seen": True} + +class F: # today's classic — completely unchanged + def __call__(self, sample: Sample) -> Sample: ... + +class G: # COMBINED views: input WITH its metadata + target WITH its metadata + def __call__(self, im: InputMeta, tm: TargetMeta): + return InputMeta(aug(im.input), im.metadata), TargetMeta(remap(tm.target), tm.metadata) + +class H: # metadata-only transform + def __call__(self, m: dict) -> dict: return {**m, "canonical": True} +``` + +Detection rules: arity counts **required** parameters (optional extras don't change anything); 3 args → unpacked `sample`; 2 args → `input_meta`/`target_meta` when the 2nd is named `metadata`/`meta` (or annotated `dict`), first-arg name `target` selects the target side, else the `pair`; 1 arg → the annotation (`Sample`, `tuple`/`Pair`, `InputMeta`/`TargetMeta`, `Input`/`Target` marks; untyped = **any** — exactly today's behavior). `op_contract(op)` exposes the result — `OpContract(accepts, produces, expands, style, bindings)`, where `bindings` lists each unpacked parameter's scope in order (e.g. `("input_meta", "target_meta")`) and `accepts` is the grid summary of the covered fields — the vocabulary a visual editor can surface as socket types. Escape hatches: `SAMPLE_KIND_IN`/`SAMPLE_KIND_OUT`/`CALL_STYLE`/`EXPANDS` class attrs. + +Merge-back: `None` drops the sample; a returned `Sample` takes over; otherwise only the declared fields update (a `pair` op keeps metadata; an `input` op keeps target+metadata; the meta variants receive the *actual* metadata dict, so in-place mutation propagates). The views are real NamedTuples (`Pair`/`InputMeta`/`TargetMeta`), recognized by `Sample.from_any`/`classify_carrier` *before* the generic tuple rule, flow natively under `Flux(native=True)`, and have default collates. + +## Multi-type carriers & the collate registry (`sampleflux.collate`) + +Pipelines can carry more than `Sample` triplets: **`Flux(native=True)`** (opt-in) keeps each carrier's own kind — a metadata-free **pair** (`(image, label)`, `(tensor, mask)`, `(tensor, coco_dict)`) or a bare **value** — and adapts every op via its introspected contract: + +```python +from confluid import configurable +from sampleflux import Flux, Sample, op_contract + +@configurable +class NormalizePair: # a pair-native op — no metadata anywhere + def __call__(self, pair: tuple) -> tuple: + img, label = pair + return img / 255.0, label + +@configurable +class StampOp: # a classic Sample op — unchanged + def __call__(self, sample: Sample) -> Sample: ... + +flux = Flux(source=[(img_a, 3), (img_b, 7)], ops=[NormalizePair(), StampOp()], native=True) +# NormalizePair receives the raw pair; StampOp receives a PROMOTED Sample view +# (promotion is one-way and sticky, so op-written metadata is never dropped). + +op_contract(NormalizePair()) # OpContract(accepts='pair', produces='pair', expands=False) +``` + +Detection reads the `__call__` annotations (`Sample` → sample-op, `tuple[...]` → pair-op, untyped → works-on-anything — **untyped ops behave exactly as today**); the class attrs `SAMPLE_KIND_IN` / `SAMPLE_KIND_OUT` / `EXPANDS` override detection where introspection can't see. `native=False` (the default) coerces everything to `Sample` exactly as before — no consumer changes. + +**Collation** is a pluggable registry keyed by representation: + +```python +from sampleflux import collate, get_collate, register_collate + +batch = collate(list(flux)) # dispatches on the detected kind +@register_collate("yolo") # task aliases are additive +def yolo_collate(items): ... +loader = DataLoader(flux, collate_fn=get_collate("yolo")) +``` + +Defaults: `"sample"` (stacked input/target + list-form batched metadata — the `is_batched` convention), `"pair"` (`(stacked_inputs, stacked_targets)`), `"value"`, and the view forms `"input_meta"`/`"target_meta"`. Consumer collates (classification/segmentation/detection) register additively and keep their own conventions. + +## 1→N expanding ops (iterable-only pipelines) + +An op may return **several** carriers — a windowing op splitting one capture into N windows is just a generator-returning op: + +```python +from typing import Iterator + +@configurable +class SlidingWindowOp: + def __call__(self, sample: Sample) -> Iterator[Sample]: + for w in sliding_windows(sample.input, self.size, self.stride): + yield sample._replace(input=w) +``` + +Expansion is detected from the return annotation (`Iterator[...]` / `Iterable[...]` / `List[...]`; or the explicit `EXPANDS = True` marker) and flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. + +A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(flux)` / `flux[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(flux)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Flux` engine. diff --git a/docs/projection.md b/docs/projection.md new file mode 100644 index 0000000..6441c15 --- /dev/null +++ b/docs/projection.md @@ -0,0 +1,48 @@ +# Field projection, class counting & label maps (`sampleflux.projection` / `sampleflux.labels`) + +## Field projection + +Walking a source for a single field (the classic case: counting classes from *targets*) shouldn't pay to build the fields you don't need. `sampleflux.projection` adds an opt-in protocol plus lazy helpers: + +```python +from sampleflux import project, iter_targets, num_classes +from sampleflux import ProjectionField # Literal["input", "target", "metadata"] + +# A source MAY implement SupportsProjection (`project(fields)`) to skip building +# unrequested fields — e.g. an image dataset reads only the label column for a +# target-only walk, never decoding an image. +for sample in project(my_source, ("target",)): + ... # sample.input is None; sample.target populated + +labels = list(iter_targets(my_source)) # lazy +n = num_classes(my_source) # max(class_id) + 1 — always walks +``` + +The field set is a **closed `Literal`**, `ProjectionField`, not a bare `str` — so a typo is a type error, and a UI / form-spec / MCP schema enumerates the choices straight from the annotation instead of hard-coding a parallel list: + +```python +from typing import get_args +get_args(ProjectionField) # ('input', 'target', 'metadata') +``` + +Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup). `num_classes` is a free function, not a `Flux` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. + +## `LabelMap` — fittable name↔id encoding + +When a dataset's `target` is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTargetOp` / `DecodeTargetOp` need — the *fittable* companion to those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: + +```python +from sampleflux import LabelMap, Flux + +lm = LabelMap.fit(iter_targets(train_source)) # {"bird": 0, "cat": 1, "dog": 2} +lm.num_classes # 3 +lm.label_names # ["bird", "cat", "dog"] (id -> name) +lm.save("class_names.json") # marainer's class_names.json format + +encoded = Flux(source=train_source, ops=[lm.encode_op()]) # targets are now ints + +# Later, at eval time — reload the SAME ordering instead of refitting: +lm2 = LabelMap.load("class_names.json") +``` + +`LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the mapping is pinned, so train / eval / predict never disagree. `scikit-learn` backs `fit` (lazy-imported). diff --git a/docs/sources.md b/docs/sources.md new file mode 100644 index 0000000..78c3861 --- /dev/null +++ b/docs/sources.md @@ -0,0 +1,83 @@ +# Sources — HuggingFace, splits, ranges, concatenation (`sampleflux.sources`) + +## Hugging Face datasets + +`HuggingFaceSource` turns any `datasets.Dataset` (a Hub repo id or a local imagefolder path) into standardized `Sample` triplets, preserving metadata traceability that often goes missing in simple dictionary-based records. + +- **`metadata_features` (which columns ride along on `Sample.metadata`):** `None` / `[]` keep none (the default); an explicit list keeps exactly those columns; and the sentinel **`"*"`** (or `["*"]`) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load. It stays opt-in so existing configs are unchanged. + +```yaml +hf_train: !class:sampleflux.sources.HuggingFaceSource() + path: mnist + input_feature: image + target_feature: label + metadata_features: ["*"] # keep every other column as metadata +``` + +> **Lazy & zero-arg construction** — `HuggingFaceSource` follows the workspace lazy-init convention: the constructor does no work (no network), so `HuggingFaceSource()` is valid and building one is free. The dataset is downloaded only on first access to the read-only `.dataset` property (cached thereafter; reset `_dataset` to reload), and `.resolved_metadata_features` (the `"*"` expansion) is derived lazily from the loaded columns. `path` is therefore optional at construction and validated lazily — accessing `.dataset` with an empty `path` raises a clear `ValueError`. + +## Train / val / test splitting (`DatasetSplit`) + +`DatasetSplit` partitions any indexable source (implementing `__len__` and `__getitem__`) into reproducible **train / val / test** views. It is a `source` (`category="source"`) — it yields `Sample`s and is wired into a trainer's `source:` slot — and it applies no ops, so it's a source, not an engine. + +**Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: + +```python +from sampleflux import DatasetSplit +split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) +split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% +``` + +The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: + +```yaml +hf_train: !class:sampleflux.sources.HuggingFaceSource() + path: mnist + split: train + +my_split: !class:sampleflux.sources.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + +train_set: !class:sampleflux.core.Flux() { source: !ref:my_split.train } +val_set: !class:sampleflux.core.Flux() { source: !ref:my_split.val } +test_set: !class:sampleflux.core.Flux() { source: !ref:my_split.test } +``` + +Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). + +**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `sampleflux.SplitName`. + +```yaml +val_set: !class:sampleflux.sources.DatasetSplit() + source: !ref:hf_train + split: val + val_fraction: 0.1 + seed: 42 +``` + +## Range & concatenation sources + +- **`RangeSource(source, start, end)`** — a contiguous index slice `[start:end)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. + + ```yaml + first_half: !class:sampleflux.sources.RangeSource() + source: !ref:hf_train + start: 0 + end: 5000 + ``` + +- **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointFlux`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. + + ```yaml + combined: !class:sampleflux.sources.ConcatSource() + sources: + - !ref:train_main + - !ref:extra_shard + ``` + +**HuggingFace native slicing** (alternative, no SampleFlux split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. + +> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..a03677c --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,61 @@ +# Storage — sinks, sources and queryable metadata (`sampleflux.storage`) + +SampleFlux makes it easy to move data between different formats: + +```python +from sampleflux.storage.hdf5 import HDF5Source +from sampleflux.storage.zarr import ZarrGroupSink + +# Stream from HDF5 to Zarr in parallel +Flux.from_source(HDF5Source("input.h5")) \ + .parallel(workers=8) \ + .map(heavy_op) \ + .to_sink(ZarrGroupSink("output.zarr")) +``` + +## Sinks and their matching sources + +Every sink has a source that reads its layout back into `Sample` triplets: + +| Backend | Sink | Source | Round-trips | +|---|---|---|---| +| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | input + target + metadata | +| Zarr group (one group / sample) | `ZarrGroupSink` | `ZarrGroupSource` | input + target + metadata | +| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | input only (uniform shape) | +| Directory (one dir / sample) | `DirectorySink` | — | — | + +```python +from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource + +Flux(samples).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) +for sample in ZarrGroupSource("ds.zarr"): # input/target as before, metadata from .zattrs + ... +``` + +> Domain-specific storage formats implement the same `DataSink`/`DataSource` protocols in their own package — e.g. the SigMF waveform-recording pair (`SigMFSink`/`SigMFSource`) lives in `waivefront.sigmf`, not here. The engine never couples to a specific format. + +## Array-valued metadata (e.g. segmentation masks) + +`HDF5Sink` stores scalar/string metadata as HDF5 **attributes**, but HDF5 caps attribute size — a large array (a segmentation mask, a per-sample weight map) put in `Sample.metadata` would overflow that limit. So **array-valued metadata (`np.ndarray` / `torch.Tensor`) is written as its own dataset** under a per-sample group `{prefix}_meta/`, and `HDF5Source` merges it back into `Sample.metadata` on read. This is fully backward-compatible: files written before this layout (no `{prefix}_meta` group) read exactly as before. + +```python +sample = Sample(input=data, target=label, metadata={"mask": mask_2d, "snr": 12.0}) +Flux([sample]).to_sink(HDF5Sink("ds.h5", overwrite=True)) +loaded = next(iter(HDF5Source("ds.h5"))) +loaded.metadata["mask"] # the full array, byte-exact (not a truncated repr) +loaded.metadata["snr"] # scalar, via attributes as before +``` + +## Queryable metadata (`sampleflux.storage.query`) + +Filter stored samples by metadata predicates *without loading arrays*: sources implementing the `SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs / `.zattrs` / sidecar JSON — `HDF5Source` and `ZarrGroupSource` both do (and external storage sources can implement the structural protocol without importing this module), so **existing HDF5/Zarr files are queryable with no rewrite**: + +```python +from sampleflux.storage.query import MetadataFilterSource + +view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="snr_db > 10 and drone == 'DJI'") +len(view) # matches counted from a metadata-only scan +flux = Flux(source=view, ops=[...]) # arrays load ONLY for matching samples +``` + +`where` uses the FormulaOp restricted namespace with metadata keys as variables (a missing key = non-matching, a malformed expression fails loudly); a programmatic `predicate=` composes with AND; sources without the protocol fall back to full-iteration filtering. Array-valued HDF5 metadata appears in the scan as shape/dtype stub strings (`""`), so queries can test presence without a single array read. diff --git a/docs/typespec.md b/docs/typespec.md new file mode 100644 index 0000000..58cbbf2 --- /dev/null +++ b/docs/typespec.md @@ -0,0 +1,37 @@ +# Type specs — `ACCEPTS` / `PRODUCES` (`sampleflux.typespec`) + +`sampleflux.typespec` describes *what flows through a `Sample`* and lets ops declare what they accept/produce, so tools (connection validators, visual config editors, schema generators) can filter which ops may connect. It is flexible by design — N-dimensional arrays across numpy/torch/tensorflow, **per-axis bounded ranges**, dtype families, images, and arbitrary Python types — and anything left unspecified defaults to `Any`. + +```python +from sampleflux.typespec import SampleType, ArrayType, Dim, PythonType, UnionType + +# "a 2-D float array whose first axis is 1–10, second axis any size" +ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="floating") +ArrayType.parse("3 h w", dtype="float32", framework="torch") # jaxtyping-style shorthand +ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") # an image convenience +``` + +`dtype`, `framework`/`frameworks`, and the image `layout` are **closed `Literal`s**, not bare strings — a typo is a type error and a UI / connection-validator enumerates the choices via `typing.get_args(...)`: + +- `Dtype` — concrete names (`"float32"`, `"int64"`, …); `DtypeFamily` — relaxed families (`"floating"`, `"numeric"`, …); `DtypeSpec = Dtype | DtypeFamily` is the `dtype` field type. +- `Framework = Literal["numpy", "torch", "tensorflow"]`, `ImageLayout = Literal["CHW", "HWC"]`. + +Authored dtypes must be canonical names; aliases / casing (`"double"`, `"FLOAT32"`) and exotic platform dtypes (`float128`) are runtime-only conveniences normalized by `canonical_dtype` — the single boundary where arbitrary input crosses into the typed domain. + +## Declaring an op's contract + +Use the class attributes `ACCEPTS` / `PRODUCES` (each a `SampleType`; both default to `Any`, so annotating is optional and backward-compatible). No base class — transforms stay plain callables: + +```python +@configurable +class StandardizeOp: + ACCEPTS = SampleType(input=UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image")))) + PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) + def __call__(self, sample): ... +``` + +Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime against a concrete inferred type); `compatible(consumer, producer)` is permissive (used at edit time — `Any`/unknown on either side passes). + +## A Sample's own type + +A `Sample`'s type comes from `sample.describe()` — it returns a type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict) + `__spec__` (sidecar refinements), or infers one from the live data; attach a stored type with `sample.with_type(SampleType(...))`. Default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. diff --git a/examples/paired_annotations.py b/examples/paired_annotations.py deleted file mode 100644 index 7bcddd2..0000000 --- a/examples/paired_annotations.py +++ /dev/null @@ -1,226 +0,0 @@ -"""AnnotationJoinSource walkthrough: binary-first, annotation-first, broadcast, slicing. - -Runs standalone with no external data. Demonstrates the four scenarios the -``AnnotationJoinSource`` primitive is designed for, using a tiny in-memory data -source and a dict-shaped annotation store. -""" - -from typing import Any, Dict, Iterator, Optional - -import confluid # type: ignore[import-not-found] - -from sampleflux.paired import AnnotationJoinSource -from sampleflux.sample import Sample - - -@confluid.configurable -class WindowedSource: - """Primary source yielding N windows over a mock pack.""" - - def __init__(self, pack_id: str = "demo:pack1", n_windows: int = 6, samples_per_window: int = 100) -> None: - self.pack_id = pack_id - self.n_windows = n_windows - self.samples_per_window = samples_per_window - - def __len__(self) -> int: - return self.n_windows - - def __getitem__(self, idx: int) -> Sample: - return Sample( - input=f"iq_window_{idx}", - target=None, - metadata={ - "pack_id": self.pack_id, - "window_start_sample": idx * self.samples_per_window, - "window_end_sample": (idx + 1) * self.samples_per_window, - "samplerate": 1_000_000.0, - }, - ) - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.n_windows): - yield self[i] - - -@confluid.configurable -class DictStore: - """Mapping-shaped annotations for demos.""" - - def __init__(self, records: Optional[Dict[str, Dict[str, Any]]] = None) -> None: - self.records = records or {} - - def __contains__(self, key: str) -> bool: - return key in self.records - - def __getitem__(self, key: str) -> Dict[str, Any]: - return self.records[key] - - def keys(self) -> Any: - return self.records.keys() - - -# Module-level callables so they survive Confluid YAML round-trip via -# sampleflux.discovery.resolve_callable("examples.paired_annotations:"). -def window_key(sample: Sample) -> str: - return f"{sample.meta['pack_id']}:win{sample.meta['window_start_sample']:08d}" - - -def pack_key(sample: Sample) -> str: - return str(sample.meta["pack_id"]) - - -def resolve_by_window_key(key: str, data: WindowedSource) -> Sample: - for item in data: - if window_key(item) == key: - return item - raise KeyError(key) - - -def slice_intervals(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - """Trim per-pack time intervals down to each window's range.""" - samplerate = sample.meta["samplerate"] - win_start = sample.meta["window_start_sample"] / samplerate - win_end = sample.meta["window_end_sample"] / samplerate - trimmed = [] - for iv in record.get("intervals", []): - s, e = max(iv["start_s"], win_start), min(iv["end_s"], win_end) - if e > s: - trimmed.append({**iv, "start_s": s, "end_s": e}) - if not trimmed: - return None - out = {k: v for k, v in record.items() if k != "intervals"} - out["intervals"] = trimmed - return out - - -def scenario_a_binary_first() -> None: - """Scenario A: iterate all samples; attach annotation when available.""" - print("\n=== Scenario A: binary-first, annotations optional ===") - data = WindowedSource(n_windows=4) - store = DictStore({"demo:pack1:win00000100": {"label": "dji_mavic", "score": 0.92}}) - - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key) - - for s in paired: - flag = "ANNOTATED" if s.meta["annotated"] else " -" - label = s.meta.get("label", "") - print(f" [{flag}] window_start={s.meta['window_start_sample']:>4} label={label!r}") - - -def scenario_b_annotation_first() -> None: - """Scenario B: only emit samples that have an annotation.""" - print("\n=== Scenario B: annotation-first, curated labeled subset ===") - data = WindowedSource(n_windows=6) - store = DictStore( - { - "demo:pack1:win00000000": {"label": "wifi"}, - "demo:pack1:win00000300": {"label": "lora"}, - } - ) - - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key, policy="inner") - - for s in paired: - print(f" key={s.meta['annotation_key']:<30} label={s.meta['label']!r}") - print(f" -> {len(list(paired))} samples (out of {len(data)} in data)") - - -def scenario_c1_broadcast() -> None: - """Scenario C1: one annotation per pack, broadcast to every window.""" - print("\n=== Scenario C1: pack-level broadcast ===") - data = WindowedSource(n_windows=4) - store = DictStore({"demo:pack1": {"drone": "DJI Mavic 3 Pro", "operator": "alice"}}) - - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=pack_key) - - for s in paired: - print( - f" win={s.meta['window_start_sample']:>4} " f"drone={s.meta['drone']!r} operator={s.meta['operator']!r}" - ) - - -def scenario_c2_slicing() -> None: - """Scenario C2: pack-level time-ranged annotation, sliced per window.""" - print("\n=== Scenario C2: pack-level time intervals, sliced per window ===") - data = WindowedSource(n_windows=6, samples_per_window=100) - # Samplerate is 1 MHz and windows are 100 samples = 100 us each, so: - # win0 = [0, 100us], win1 = [100us, 200us], ..., win5 = [500us, 600us]. - # An interval at [150us, 470us] overlaps windows 1, 2, 3, 4. - store = DictStore( - { - "demo:pack1": { - "drone": "mavic", - "intervals": [{"start_s": 150e-6, "end_s": 470e-6, "label": "active_emission"}], - } - } - ) - - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=pack_key, - extract_fn=slice_intervals, - ) - - for s in paired: - win = s.meta["window_start_sample"] - if s.meta["annotated"]: - iv = s.meta["intervals"][0] - print( - f" win_start={win:>4} drone={s.meta['drone']!r} " - f"active=[{iv['start_s']*1e6:.1f}us, {iv['end_s']*1e6:.1f}us]" - ) - else: - print(f" win_start={win:>4} no overlap") - - -def scenario_d_right_driven() -> None: - """Scenario D: iterate the annotation store, resolve data on demand.""" - print("\n=== Scenario D: right-driven (sparse labels, large data) ===") - data = WindowedSource(n_windows=1000) # pretend this is huge - store = DictStore( - { - "demo:pack1:win00000000": {"label": "wifi"}, - "demo:pack1:win00050000": {"label": "lora"}, - "demo:pack1:win00099900": {"label": "dji_ocusync"}, - } - ) - - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=window_key, - policy="right_driven", - data_resolver=resolve_by_window_key, - ) - - for s in paired: - print(f" key={s.meta['annotation_key']:<30} label={s.meta['label']!r}") - - -def scenario_e_confluid_roundtrip() -> None: - """Show that the pipeline survives YAML serialization via Confluid.""" - print("\n=== Scenario E: Confluid YAML round-trip ===") - data = WindowedSource(n_windows=3) - store = DictStore({"demo:pack1:win00000000": {"label": "wifi"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=window_key) - - yaml_state = confluid.dump(paired) - print(yaml_state) - restored = confluid.load(yaml_state) - print(f" restored.policy = {restored.policy!r}") - print(f" restored.key_fn = {restored.key_fn!r}") - - -def main() -> None: - scenario_a_binary_first() - scenario_b_annotation_first() - scenario_c1_broadcast() - scenario_c2_slicing() - scenario_d_right_driven() - scenario_e_confluid_roundtrip() - print("\nOK — paired_annotations.py finished.") - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 957e762..d5dea04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,23 +53,18 @@ sampleflux = "sampleflux" sampleflux-core = "sampleflux.core" sampleflux-sources = "sampleflux.sources" sampleflux-ops-parallel = "sampleflux.ops.parallel" -sampleflux-ops-tee = "sampleflux.ops.tee" sampleflux-ops-enable = "sampleflux.ops.enable" sampleflux-ops-random-apply = "sampleflux.ops.random_apply" # ConfigureOp (per-sample parameter injection — the helios Configure pattern); entry-point # changes need an editable reinstall before FluxStudio/navigaitor discovery sees the module. sampleflux-ops-configure = "sampleflux.ops.configure" sampleflux-ops-formula = "sampleflux.ops.formula" -# CaptureOutputOp (records an op's @output value into metadata — the capture half of -# FluxStudio's op-@output -> param wiring, paired with ConfigureOp(UnstashInputOp)). -sampleflux-ops-capture = "sampleflux.ops.capture" sampleflux-ops-transform-chain = "sampleflux.ops.transform_chain" # Context ops (Save/Use/Drop/Apply/Capture/Mix) — the graph-plane building blocks lowered from flow: docs sampleflux-ops-context = "sampleflux.ops.context" # The FlowGraph engine (flow: named-step documents + the flow<->ops converters) sampleflux-flow = "sampleflux.flow" -# SigMF recordings (SigMFSink <-> SigMFSource) + the queryable-metadata view source -sampleflux-storage-sigmf = "sampleflux.storage.sigmf" +# The queryable-metadata scan protocol + MetadataFilterSource view source sampleflux-storage-query = "sampleflux.storage.query" sampleflux-ops-sink = "sampleflux.ops.sink" sampleflux-ops-stash = "sampleflux.ops.stash" diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py index 6df4566..ab083ae 100644 --- a/sampleflux/__init__.py +++ b/sampleflux/__init__.py @@ -6,12 +6,11 @@ from sampleflux.context import Context from sampleflux.core import Flux, JointFlux, WrappedOp from sampleflux.flow import FlowGraph, from_ops, to_ops -from sampleflux.kinds import OpContract, SampleKind, classify_carrier, op_contract +from sampleflux.kinds import INPUT, TARGET, Input, OpContract, SampleKind, Target, classify_carrier, op_contract from sampleflux.labels import LabelMap from sampleflux.ops import RescaleOp, StandardizeOp, ToTensorOp -from sampleflux.paired import AnnotationJoinSource, AnnotationStore from sampleflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project -from sampleflux.sample import Sample +from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName from sampleflux.typespec import ( AnyType, @@ -32,8 +31,6 @@ ) __all__ = [ - "AnnotationJoinSource", - "AnnotationStore", "AnyType", "ArrayType", "ConcatSource", @@ -41,7 +38,14 @@ "DatasetSplit", "Dim", "FlowGraph", + "INPUT", + "Input", + "InputMeta", "OpContract", + "Pair", + "TARGET", + "Target", + "TargetMeta", "SampleKind", "classify_carrier", "collate", diff --git a/sampleflux/collate.py b/sampleflux/collate.py index 3f393cf..6ffd2a3 100644 --- a/sampleflux/collate.py +++ b/sampleflux/collate.py @@ -25,7 +25,7 @@ from loggair import get_logger from sampleflux.kinds import classify_carrier -from sampleflux.sample import Sample +from sampleflux.sample import InputMeta, Sample, TargetMeta logger = get_logger(__name__) @@ -125,3 +125,15 @@ def pair_collate(items: Sequence[Any]) -> Tuple[Any, Any]: def value_collate(items: Sequence[Any]) -> Any: """Default value collate: the bare values stacked.""" return _stack(list(items)) + + +@register_collate("input_meta") +def input_meta_collate(items: Sequence[Any]) -> InputMeta: + """Default InputMeta collate: stacked inputs + the per-item metadata dicts as a list.""" + return InputMeta(_stack([item.input for item in items]), [dict(item.metadata) for item in items]) + + +@register_collate("target_meta") +def target_meta_collate(items: Sequence[Any]) -> TargetMeta: + """Default TargetMeta collate: stacked targets + the per-item metadata dicts as a list.""" + return TargetMeta(_stack([item.target for item in items]), [dict(item.metadata) for item in items]) diff --git a/sampleflux/core.py b/sampleflux/core.py index 9be7279..9ff9391 100644 --- a/sampleflux/core.py +++ b/sampleflux/core.py @@ -28,7 +28,7 @@ from sampleflux.context import Context, activate from sampleflux.projection import ProjectionField -from sampleflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, Sample +from sampleflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, InputMeta, Pair, Sample, TargetMeta if TYPE_CHECKING: # pragma: no cover - typing only from sampleflux.typespec import SampleType @@ -63,11 +63,142 @@ def _refresh_type(sample: Sample, op: Any) -> Sample: def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: """Apply one op and refresh the stored type. The single op-application chokepoint shared by the - sequential, parallel (via :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths.""" - result = op(sample) + sequential, parallel (via :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths. + + The op's introspected contract (:func:`sampleflux.kinds.op_contract`) picks the BINDING: + a classic sample/untyped op receives the Sample verbatim (today's fast path); a + field-scoped op (``input`` / ``target`` / ``pair`` / ``input_meta`` / ``target_meta`` — + packed views or unpacked separate arguments) receives exactly its declared view and the + result merges back with the untouched fields preserved (:func:`_apply_view`). + """ + from sampleflux.kinds import op_contract + + contract = op_contract(op) + if contract.accepts in ("sample", "any", "value") and contract.style == "packed": + result = op(sample) + if result is None: + return None + return _refresh_type(result, op) + return _apply_view(sample, op, contract) + + +def _view_error(op: Any, scope: str, result: Any) -> TypeError: + return TypeError( + f"{type(op).__name__}: a {scope!r}-scope op must return the matching view/tuple, a full " + f"Sample, or None — got {type(result).__name__}" + ) + + +# One argument of an unpacked op, bound from the sample per its declared field scope. +_BIND_GET: Dict[str, Callable[[Sample], Any]] = { + "input": lambda s: s.input, + "target": lambda s: s.target, + "metadata": lambda s: s.meta, + "input_meta": lambda s: s.input_meta(), + "target_meta": lambda s: s.target_meta(), +} + + +def _apply_bindings(sample: Sample, op: Any, bindings: Tuple[str, ...]) -> Optional[Sample]: + """Apply an UNPACKED op — each argument bound per its declared field scope — and merge back. + + Handles EVERY combination the binding resolver produces: the classic + ``f(input, target)`` / ``f(input, target, metadata)``, the meta forms + ``f(input, metadata)`` / ``f(target, metadata)``, and mixed VIEW arguments like + ``f(im: InputMeta, tm: TargetMeta)`` or ``f(x: Input, tm: TargetMeta)``. The result + must be ``None`` (drop), a full ``Sample`` (takes over), or a tuple of the SAME arity + — each element merged per its binding (a view/2-tuple element for a ``*_meta`` binding + replaces value + metadata; a bare element replaces only the value). Metadata-bearing + elements merge left-to-right (the LAST metadata write wins — they usually share the + one live dict anyway, which the op may also mutate in place). + """ + args = [_BIND_GET[b](sample) for b in bindings] + result = op(*args) if result is None: return None - return _refresh_type(result, op) + if isinstance(result, Sample): + return _refresh_type(result, op) + # A NAMED view is itself a tuple — returning ONE view from a multi-binding op would be + # silently misread as two elements, so it only counts as the whole result at arity 1. + is_single_view = isinstance(result, (InputMeta, TargetMeta, Pair)) + if (is_single_view and len(bindings) != 1) or not (isinstance(result, tuple) and len(result) == len(bindings)): + raise TypeError( + f"{type(op).__name__}: an unpacked op bound as {bindings!r} must return a tuple of the " + f"same arity, a full Sample, or None — got {type(result).__name__}" + ) + updates: Dict[str, Any] = {} + for binding, element in zip(bindings, result): + if binding in ("input", "target", "metadata"): + updates[binding] = element + else: # input_meta / target_meta + field = "input" if binding == "input_meta" else "target" + if isinstance(element, tuple) and len(element) == 2: + updates[field] = element[0] + updates["metadata"] = element[1] + else: # bare value: only the field changes (in-place meta mutation is already live) + updates[field] = element + return _refresh_type(sample._replace(**updates), op) + + +def _apply_view(sample: Sample, op: Any, contract: Any) -> Optional[Sample]: + """Bind a field-scoped op's declared view from ``sample``, apply, and merge the result back. + + Unpacked ops route through :func:`_apply_bindings` (per-argument scopes). Packed + single-view scopes (``None`` always drops; a returned ``Sample`` always takes over; + metadata dicts are handed live, so in-place mutation propagates): + + - ``input`` / ``target`` — the bare value in, the new value out (other fields kept); + - ``metadata`` — the dict in, the (new) dict out; + - ``pair`` — a `Pair` in (a plain-tuple-annotated op indexes it identically), a + 2-tuple out replaces input+target (metadata kept); + - ``input_meta`` / ``target_meta`` — the named view in; a view/2-tuple out replaces + value + metadata; a bare value out replaces only the value. + """ + if contract.style == "unpacked" and contract.bindings: + return _apply_bindings(sample, op, contract.bindings) + scope = contract.accepts + + if scope == "input" or scope == "target": + field = scope + result = op(getattr(sample, field)) + if result is None: + return None + return _refresh_type(sample._replace(**{field: result}), op) + + if scope == "metadata": + result = op(sample.meta) + if result is None: + return None + if isinstance(result, Sample): + return _refresh_type(result, op) + if isinstance(result, dict): + return _refresh_type(sample._replace(metadata=result), op) + raise _view_error(op, scope, result) + + if scope == "pair": + result = op(Pair(sample.input, sample.target)) + if result is None: + return None + if isinstance(result, Sample): + return _refresh_type(result, op) + if isinstance(result, tuple) and len(result) == 2: + return _refresh_type(sample._replace(input=result[0], target=result[1]), op) + raise _view_error(op, scope, result) + + if scope in ("input_meta", "target_meta"): + field = "input" if scope == "input_meta" else "target" + result = op(sample.input_meta() if scope == "input_meta" else sample.target_meta()) + if result is None: + return None + if isinstance(result, Sample): + return _refresh_type(result, op) + if isinstance(result, tuple) and len(result) == 2: + return _refresh_type(sample._replace(**{field: result[0], "metadata": result[1]}), op) + return _refresh_type(sample._replace(**{field: result}), op) + + # A packed "sample"-scope op took the _apply_op fast path; anything else is defensive. + result = op(sample) # pragma: no cover + return None if result is None else _refresh_type(result, op) # pragma: no cover def _describe_deferred_source(source: Any) -> str: @@ -185,34 +316,37 @@ class _Carried(NamedTuple): def _apply_op_native(carrier: Any, op: Any) -> Any: - """Apply one op to a NATIVE carrier (Sample / metadata-free pair / bare value). - - The op's introspected contract (:func:`sampleflux.kinds.op_contract`) picks the - adaptation: - - - a **pair-op** on a Sample carrier receives ``(input, target)`` and its returned - pair merges back via ``_replace`` (metadata preserved); - - a **sample-op** on a pair/value carrier receives a PROMOTED Sample view - (``Sample.from_any`` — promotion is one-way and sticky, so op-written metadata is - never dropped); - - an **any-op** receives the carrier verbatim (untyped ops behave exactly as today). + """Apply one op to a NATIVE carrier (Sample / pair / bare value / a field view). + + Adaptation rules (the op's contract via :func:`sampleflux.kinds.op_contract`): + + - a **Sample** carrier routes through :func:`_apply_op` (which binds every scope); + - an **any-op** receives the carrier verbatim (untyped ops behave exactly as today); + - two NATIVE fast lanes keep metadata-free data metadata-free: a pair-scope op on a + pair carrier (result stays a pair) and an input-scope op on a bare value (result + stays a bare value); + - everything else PROMOTES the carrier to a Sample view (``Sample.from_any`` — view + types like ``InputMeta`` coerce field-correctly) — promotion is one-way and sticky, + so op-written metadata is never dropped. """ - from sampleflux.kinds import op_contract + from sampleflux.kinds import classify_carrier, op_contract contract = op_contract(op) if isinstance(carrier, Sample): - if contract.accepts == "pair": - result = op(carrier.to_pair()) - if result is None: - return None - if isinstance(result, tuple) and len(result) == 2: - return carrier._replace(input=result[0], target=result[1]) - return result return _apply_op(carrier, op) - if contract.accepts == "sample": - return _apply_op(Sample.from_any(carrier), op) # promotion is sticky - result = op(carrier) - return result + if contract.accepts == "any": + return op(carrier) + kind = classify_carrier(carrier) + if contract.accepts == "pair" and kind == "pair": + result = op(carrier[0], carrier[1]) if contract.style == "unpacked" else op(tuple(carrier)) + if result is None: + return None + if isinstance(result, (Sample, tuple)): + return result + raise _view_error(op, "pair", result) + if contract.accepts == "input" and kind == "value": + return op(carrier) + return _apply_op(Sample.from_any(carrier), op) # promotion is sticky def _expand(op: Any, carrier: Any) -> List[Any]: @@ -392,10 +526,10 @@ def joint(cls, fluxes: List["Flux"]) -> "Flux": @classmethod def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": - """Attach an ops-only Confluid YAML (e.g. exported from FluxStudio) to ``source``. + """Attach an ops-only Confluid YAML (e.g. one exported by a pipeline-authoring tool) to ``source``. ``path`` is the ``{ops: [!class:...()]}`` document produced by - :func:`fluxstudio.export.export_ops_yaml` (the ``fluxstudio export`` CLI or the + an external graph exporter's ops-export (the CLI or the canvas Export button). It also accepts an inline YAML string (``confluid.load`` handles both). diff --git a/sampleflux/discovery.py b/sampleflux/discovery.py index 16af5e3..8992cff 100644 --- a/sampleflux/discovery.py +++ b/sampleflux/discovery.py @@ -12,7 +12,7 @@ * **Discovery** (callable -> JSON schema): :func:`introspect_callable` reflects a single callable into a schema (signature + docstring + the ``ACCEPTS`` / ``PRODUCES`` typespec contract), and :func:`scan_module` does the same for - every callable *defined in* a module. FluxStudio reads these to auto-generate + every callable *defined in* a module. visual editors read these to auto-generate ComfyUI nodes and their property panels; navigaitor builds its MCP form-spec from the same data. """ @@ -109,7 +109,7 @@ def introspect_callable(func: Callable) -> Dict[str, Any]: ``*args`` / ``**kwargs``), plus the declared ``ACCEPTS`` / ``PRODUCES`` typespec contract when present. - Use: FluxStudio reads this to render a node and its property-panel widgets, + Use: a visual editor reads this to render a node and its property-panel widgets, and it feeds navigaitor's MCP form-spec. """ try: @@ -154,7 +154,7 @@ def scan_module(path_or_name: Union[str, Path]) -> List[Dict[str, Any]]: module — names merely imported into it are filtered out by checking ``member.__module__ == mod_name``. - Use: the entry point for whole-module discovery — FluxStudio's ``bridge`` + Use: the entry point for whole-module discovery — a visual editor's bridge calls this to auto-generate one node per source/op, fulfilling the "never require manual tool definitions" mandate. """ diff --git a/sampleflux/flow.py b/sampleflux/flow.py index 226ba9b..139e6de 100644 --- a/sampleflux/flow.py +++ b/sampleflux/flow.py @@ -2,7 +2,7 @@ A **flow document** is the readable, named-step form of a graph-shaped pipeline: a mapping of ``step-name → op``, where a step's name is also the name later steps use to -reference its result. It is the authoring format (humans and the FluxStudio exporter +reference its result. It is the authoring format (humans and graph exporters write it); the flat context-ops form (:mod:`sampleflux.ops.context`) is the serial execution format the plain :class:`~sampleflux.core.Flux` engine runs. The two convert **bidirectionally**: :func:`to_ops` lowers a flow into a flat op list, :func:`from_ops` @@ -493,7 +493,7 @@ def to_ops(steps: Union[Sequence[FlowStep], Dict[str, Any]], outputs: str = "") """Lower a flow (parsed steps or a raw flow mapping) into a flat context-ops list. The result runs on the plain serial :class:`~sampleflux.core.Flux` engine and is the - serialization form FluxStudio's ``--serial`` export emits. Cell names are the step + serialization form a graph exporter's serial mode emits. Cell names are the step names (deterministic, diffable); liveness is compiled into ``drop`` flags so a well-formed graph leaves the Context empty. A purely linear flow lowers to the bare op list — zero context ops. @@ -669,7 +669,7 @@ def from_ops(ops: Sequence[Any], outputs: str = "") -> Tuple[Dict[str, Any], str execution-equivalent to ``ops``. Accepts LIVE ops or confluid ``Instance``/``Class`` MARKERS interchangeably (the - FluxStudio exporter lifts compiled marker lists without materializing them, keeping + graph exporter lifts compiled marker lists without materializing them, keeping hoisted-constant ``!ref:``\\ s intact); a real op arrives in the flow mapping verbatim (marker in, marker out). """ diff --git a/sampleflux/kinds.py b/sampleflux/kinds.py index f6afb5f..d726c01 100644 --- a/sampleflux/kinds.py +++ b/sampleflux/kinds.py @@ -1,34 +1,81 @@ -"""Op-kind introspection — what carrier an op accepts/produces, detected from its annotations. - -The native multi-type engine (``Flux(native=True)``) lets carriers other than -:class:`~sampleflux.sample.Sample` flow through a pipeline — metadata-free **pairs** like -``(image, label)`` / ``(tensor, mask)`` / ``(tensor, coco_dict)``, or bare **values**. -Ops can process everything: the engine detects each op's contract by INTROSPECTING the -``__call__`` type annotations (``__call__(self, sample: Sample)`` vs -``__call__(self, pair: tuple[np.ndarray, int])`` vs untyped = works-on-anything) and -adapts the carrier per op. Explicit class attributes (``SAMPLE_KIND_IN`` / -``SAMPLE_KIND_OUT`` / ``EXPANDS``) override detection for cases introspection can't see -(C-extension callables, wrappers around raw functions). +"""Op-kind introspection — WHAT a transform processes and HOW it wants to be called. + +The taxonomy is a grid over two axes, detected from ``__call__``'s signature so ops stay +plain callables (no base classes) and a visual editor can surface the names later: + +**Field scope** (:data:`SampleKind`) — which part of the ``Sample(input, target, +metadata)`` triple the transform processes: + +======================== ========================== ============================ +scope without metadata with metadata +======================== ========================== ============================ +input only ``input`` (bare value) ``input_meta`` (`InputMeta`) +target only ``target`` (bare value) ``target_meta`` (`TargetMeta`) +both ``pair`` (`(input,target)`) ``sample`` (the full triple) +======================== ========================== ============================ + +plus ``value`` (a bare carrier of unknown role, runtime classification only) and ``any`` +(untyped — receives whatever flows, exactly today's behavior). + +**Call style** (:data:`CallStyle`) — packed (ONE argument: the ``Sample`` / a tuple / a +view) or unpacked (the fields as SEPARATE arguments): + +- ``__call__(self, sample: Sample)`` → sample, packed +- ``__call__(self, input, target, metadata)`` → sample, unpacked (3 required args) +- ``__call__(self, pair: tuple)`` / ``(p: Pair)`` → pair, packed +- ``__call__(self, input, target)`` → pair, unpacked (2 required args) +- ``__call__(self, v: InputMeta)`` → input_meta, packed +- ``__call__(self, input, metadata)`` → input_meta, unpacked (2nd arg named ``metadata``/``meta``) +- ``__call__(self, target, metadata)`` → target_meta, unpacked (1st arg named ``target``) +- ``__call__(self, x: Input)`` → input, bare value (`Input`/`Target` Annotated aliases, + or mark your own type: ``Annotated[np.ndarray, INPUT]``) +- untyped single argument → any (unchanged) + +The ENGINE binds the declared view from whatever carrier flows and merges the result +back, preserving untouched fields (see ``core._apply_op``). Arity counts REQUIRED +parameters only, so an existing op with optional extras keeps today's behavior. Explicit +class attributes (``SAMPLE_KIND_IN`` / ``SAMPLE_KIND_OUT`` / ``EXPANDS`` / +``CALL_STYLE``) override detection for callables introspection can't read. The same introspection powers 1→N detection: a ``-> Iterator[Sample]`` / -``-> Iterable[Sample]`` return annotation (or ``EXPANDS = True``) marks an EXPANDING op — -one carrier in, several out — which makes the pipeline iterable-only (see -``Flux.__len__``/``__getitem__``). +``-> Iterable[Sample]`` return annotation (or ``EXPANDS = True``) marks an EXPANDING op, +which makes the pipeline iterable-only (see ``Flux.__len__``/``__getitem__``). """ import collections.abc import inspect from dataclasses import dataclass -from typing import Any, Dict, Literal, Tuple, Union, get_args, get_origin, get_type_hints +from typing import Annotated, Any, Dict, Literal, Tuple, Union, get_args, get_origin, get_type_hints + +from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta -from sampleflux.sample import Sample +SampleKind = Literal["sample", "pair", "input", "target", "metadata", "input_meta", "target_meta", "value", "any"] +"""The field-scope taxonomy — see the module docstring grid.""" -SampleKind = Literal["sample", "pair", "value", "any"] -"""The carrier taxonomy: a full Sample triplet, a metadata-free 2-tuple, a bare value, or anything.""" +CallStyle = Literal["packed", "unpacked"] +"""How the op wants its view: one packed argument, or the fields as separate arguments.""" SAMPLE_KINDS: Tuple[str, ...] = get_args(SampleKind) +CALL_STYLES: Tuple[str, ...] = get_args(CallStyle) -__all__ = ["OpContract", "SAMPLE_KINDS", "SampleKind", "classify_carrier", "op_contract"] +_META_PARAM_NAMES = frozenset({"metadata", "meta"}) +_TARGET_PARAM_NAMES = frozenset({"target"}) + +__all__ = [ + "CALL_STYLES", + "CallStyle", + "INPUT", + "Input", + "METADATA", + "MetaDict", + "OpContract", + "SAMPLE_KINDS", + "SampleKind", + "TARGET", + "Target", + "classify_carrier", + "op_contract", +] _EXPANDING_ORIGINS = ( list, @@ -41,17 +88,47 @@ ) +class _KindMark: + """PEP-593 marker naming the field a bare-value annotation binds (``Annotated[T, INPUT]``).""" + + __slots__ = ("kind",) + + def __init__(self, kind: str) -> None: + self.kind = kind + + def __repr__(self) -> str: # pragma: no cover - debug aid + return f"KindMark({self.kind})" + + +INPUT = _KindMark("input") +TARGET = _KindMark("target") +METADATA = _KindMark("metadata") + +Input = Annotated[Any, INPUT] +"""Annotation alias: the op processes the BARE input value (``Annotated[T, INPUT]`` keeps a real T).""" + +Target = Annotated[Any, TARGET] +"""Annotation alias: the op processes the BARE target value (``Annotated[T, TARGET]`` keeps a real T).""" + +MetaDict = Annotated[Any, METADATA] +"""Annotation alias: the op processes the metadata DICT (a plain ``dict`` annotation works too).""" + + @dataclass(frozen=True) class OpContract: - """What an op consumes and produces. + """What an op consumes/produces, how it is called, and whether it expands 1→N. - ``accepts``/``produces`` are :data:`SampleKind` members; ``expands`` marks a 1→N op - (returns an iterable of carriers instead of one). + For an UNPACKED op, ``bindings`` lists each required parameter's field scope in + order (e.g. ``("input_meta", "target_meta")`` for ``f(im: InputMeta, tm: TargetMeta)``) + — the engine binds each argument independently and merges each returned element back. + ``accepts`` stays the grid SUMMARY of the covered fields (what a visual editor surfaces). """ accepts: SampleKind = "any" produces: SampleKind = "any" expands: bool = False + style: CallStyle = "packed" + bindings: Tuple[str, ...] = () _ANY_CONTRACT = OpContract() @@ -59,9 +136,19 @@ class OpContract: def classify_carrier(obj: Any) -> SampleKind: - """The carrier kind of a runtime object: Sample -> ``sample``, 2-tuple -> ``pair``, else ``value``.""" + """The carrier kind of a runtime object. + + The named views are checked BEFORE the generic tuple rule — an ``InputMeta`` IS a + 2-tuple and would otherwise misclassify as a pair. + """ if isinstance(obj, Sample): return "sample" + if isinstance(obj, InputMeta): + return "input_meta" + if isinstance(obj, TargetMeta): + return "target_meta" + if isinstance(obj, Pair): + return "pair" if isinstance(obj, tuple) and len(obj) == 2: return "pair" return "value" @@ -76,20 +163,48 @@ def _unwrap_optional(anno: Any) -> Any: return anno +def _kind_mark_of(anno: Any) -> Any: + """The ``_KindMark`` on an ``Annotated[...]`` layer, or None.""" + if get_origin(anno) is Annotated: + for meta in get_args(anno)[1:]: + if isinstance(meta, _KindMark): + return meta + return None + + def _kind_of(anno: Any) -> SampleKind: - """The carrier kind an annotation names; unknown/absent/Any -> ``any``.""" + """The field scope an annotation names; unknown/absent/Any -> ``any``.""" anno = _unwrap_optional(anno) + mark = _kind_mark_of(anno) + if mark is not None: + return mark.kind # type: ignore[no-any-return] + if get_origin(anno) is Annotated: + return _kind_of(get_args(anno)[0]) if anno is inspect.Parameter.empty or anno is Any or anno is None: return "any" if anno is Sample: return "sample" + if anno is InputMeta: + return "input_meta" + if anno is TargetMeta: + return "target_meta" + if anno is Pair: + return "pair" if anno is tuple or get_origin(anno) is tuple: return "pair" + if anno is dict or get_origin(anno) is dict: + return "metadata" if isinstance(anno, type) and issubclass(anno, Sample): return "sample" return "any" +def _is_meta_annotation(anno: Any) -> bool: + """True when an annotation names a metadata dict (``Dict[str, ...]`` / ``dict``).""" + anno = _unwrap_optional(anno) + return anno is dict or get_origin(anno) is dict + + def _return_contract(anno: Any) -> Tuple[SampleKind, bool]: """(produced kind, expands) from a return annotation.""" anno = _unwrap_optional(anno) @@ -101,14 +216,66 @@ def _return_contract(anno: Any) -> Tuple[SampleKind, bool]: return _kind_of(anno), False +# Per-parameter binding vocabulary: which field scope one argument of an UNPACKED op binds. +_PARAM_BINDINGS = ("input", "target", "metadata", "input_meta", "target_meta") +# Positional defaults — the classic AI convention: f(input, target[, metadata]). +_POSITIONAL_DEFAULTS = ("input", "target", "metadata") +_INPUT_PARAM_NAMES = frozenset({"input"}) + +# Which fields each binding covers, for the grid summary. +_BINDING_FIELDS: Dict[str, frozenset] = { + "input": frozenset({"i"}), + "target": frozenset({"t"}), + "metadata": frozenset({"m"}), + "input_meta": frozenset({"i", "m"}), + "target_meta": frozenset({"t", "m"}), +} +_FIELDS_TO_KIND: Dict[frozenset, str] = { + frozenset({"i", "t", "m"}): "sample", + frozenset({"i", "t"}): "pair", + frozenset({"i", "m"}): "input_meta", + frozenset({"t", "m"}): "target_meta", + frozenset({"i"}): "input", + frozenset({"t"}): "target", + frozenset({"m"}): "metadata", +} + + +def _param_binding(param: Any, anno: Any, position: int) -> str: + """One required parameter's field binding: annotation wins, then the name, then position. + + Positional defaults are the classic ``f(input, target, metadata)`` convention, so an + unannotated/unnamed multi-arg op keeps the old behavior; a view annotation + (``InputMeta``/``TargetMeta``/``Input``/``Target``/``dict``) or a recognised name + (``input``/``target``/``metadata``/``meta``) overrides its slot. + """ + kind = _kind_of(anno) + if kind in _PARAM_BINDINGS: + return kind + if param.name in _INPUT_PARAM_NAMES: + return "input" + if param.name in _TARGET_PARAM_NAMES: + return "target" + if param.name in _META_PARAM_NAMES: + return "metadata" + return _POSITIONAL_DEFAULTS[position] + + +def _bindings_summary(bindings: Tuple[str, ...]) -> SampleKind: + """The grid-summary kind of a binding list (the covered fields).""" + covered: frozenset = frozenset().union(*(_BINDING_FIELDS[b] for b in bindings)) + return _FIELDS_TO_KIND.get(covered, "sample") # type: ignore[return-value] + + def op_contract(op: Any) -> OpContract: - """The introspected (cached per type) carrier contract of an op. + """The introspected (cached per type) contract of an op — scope, call style, expansion. Explicit class attributes win: ``SAMPLE_KIND_IN`` / ``SAMPLE_KIND_OUT`` (a - :data:`SampleKind` string) and ``EXPANDS`` (bool) override whatever the annotations - say — the escape hatch for callables introspection can't read. Annotation resolution - failures (lazy imports, unresolvable forward refs) degrade to ``any`` so an untyped or - exotic op behaves exactly as today. + :data:`SampleKind`), ``CALL_STYLE`` (a :data:`CallStyle`), and ``EXPANDS`` (bool) — + the escape hatch for callables introspection can't read. Annotation resolution + failures degrade to ``any``/packed so an untyped or exotic op behaves exactly as + today. Arity counts REQUIRED parameters (no default) only, so an op with optional + extras after its sample argument keeps single-argument semantics. """ cls = type(op) cached = _contract_cache.get(cls) @@ -118,34 +285,49 @@ def op_contract(op: Any) -> OpContract: accepts: SampleKind = "any" produces: SampleKind = "any" expands = False + style: CallStyle = "packed" + bindings: Tuple[str, ...] = () call = getattr(cls, "__call__", None) if call is not None: try: signature = inspect.signature(call) - hints = get_type_hints(call) + hints = get_type_hints(call, include_extras=True) except Exception: # noqa: BLE001 - degrade to "any" on ANY introspection failure signature, hints = None, {} if signature is not None: - params = [p for name, p in signature.parameters.items() if name != "self"] - if params: - first = params[0] - accepts = _kind_of(hints.get(first.name, first.annotation)) + params = [ + p + for name, p in signature.parameters.items() + if name != "self" and p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) + ] + required = [p for p in params if p.default is inspect.Parameter.empty] + arity = len(required) if required else min(len(params), 1) + if arity == 1 and params: + accepts = _kind_of(hints.get(params[0].name, params[0].annotation)) + elif 2 <= arity <= 3: + bindings = tuple(_param_binding(p, hints.get(p.name, p.annotation), i) for i, p in enumerate(required)) + accepts, style = _bindings_summary(bindings), "unpacked" + # arity 0 or > 3: leave "any"/packed — the op is called with the carrier verbatim. produces, expands = _return_contract(hints.get("return", inspect.Parameter.empty)) - contract = OpContract(accepts=accepts, produces=produces, expands=expands) + bindings = bindings if style == "unpacked" else () + contract = OpContract(accepts=accepts, produces=produces, expands=expands, style=style, bindings=bindings) _contract_cache[cls] = contract return _explicit_overrides(op, contract) def _explicit_overrides(op: Any, base: OpContract) -> OpContract: - """Apply the ``SAMPLE_KIND_IN``/``SAMPLE_KIND_OUT``/``EXPANDS`` class-attr escape hatches.""" + """Apply the ``SAMPLE_KIND_IN``/``SAMPLE_KIND_OUT``/``EXPANDS``/``CALL_STYLE`` escape hatches.""" kind_in = getattr(op, "SAMPLE_KIND_IN", None) kind_out = getattr(op, "SAMPLE_KIND_OUT", None) expands = getattr(op, "EXPANDS", None) - if kind_in is None and kind_out is None and expands is None: + call_style = getattr(op, "CALL_STYLE", None) + if kind_in is None and kind_out is None and expands is None and call_style is None: return base return OpContract( accepts=kind_in if kind_in in SAMPLE_KINDS else base.accepts, produces=kind_out if kind_out in SAMPLE_KINDS else base.produces, expands=bool(expands) if expands is not None else base.expands, + style=call_style if call_style in CALL_STYLES else base.style, + bindings=base.bindings, ) diff --git a/sampleflux/ops/__init__.py b/sampleflux/ops/__init__.py index ca2f11a..87508f8 100644 --- a/sampleflux/ops/__init__.py +++ b/sampleflux/ops/__init__.py @@ -4,19 +4,13 @@ Submodules: - sampleflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp, SqueezeOp, - UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, - WindowOp, SpectrumScalingOp (ndarray) + UnsqueezeOp, MinOp, MaxOp, MedianOp, PercentileOp, StatsOp (ndarray) - sampleflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp, SqueezeOp, - UnsqueezeOp, FourierOp, InverseFourierOp, FftShiftOp, IfftShiftOp, - WindowOp, SpectrumScalingOp (tensor) - - sampleflux.windows: get_window / scale_spectrum + the WindowName / - SpectrumScaling Literals — the window + unit-scaling math the FFT ops share - - sampleflux.ops.tee: Tee (fan-out branching) + UnsqueezeOp (tensor) - sampleflux.ops.parallel: Parallel (worker-pool sub-pipeline) - sampleflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - sampleflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - sampleflux.ops.configure: ConfigureOp (per-sample parameter injection — the helios Configure pattern) - - sampleflux.ops.capture: CaptureOutputOp (record an op's @output value into metadata) - sampleflux.ops.formula: FormulaOp (math formula over sample.input — the Math node's op form) - sampleflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - sampleflux.ops.transform_chain: TransformChain (sequential op-chain grouping) @@ -25,13 +19,14 @@ - sampleflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - sampleflux.ops.swap: SwapInputTargetOp - sampleflux.ops.stash: StashInputOp, UnstashInputOp, StashTargetOp, UnstashTargetOp + (metadata-bus snapshots — only for crossing a Parallel boundary or persisting + a snapshot into a sink; graph wiring uses sampleflux.ops.context) - sampleflux.ops.target: MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp (target field) Flat imports default to torch variants for the data ops; flow / copy / swap / stash / target utilities are field-agnostic. """ -from sampleflux.ops.capture import CaptureOutputOp from sampleflux.ops.configure import ConfigureOp from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use from sampleflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp @@ -49,24 +44,10 @@ MasksToDetectionBoxesOp, MetadataToTargetOp, ) -from sampleflux.ops.tee import Tee -from sampleflux.ops.torch import ( - FftShiftOp, - FourierOp, - IfftShiftOp, - InverseFourierOp, - RescaleOp, - SpectrumScalingOp, - SqueezeOp, - StandardizeOp, - ToTensorOp, - UnsqueezeOp, - WindowOp, -) +from sampleflux.ops.torch import RescaleOp, SqueezeOp, StandardizeOp, ToTensorOp, UnsqueezeOp from sampleflux.ops.transform_chain import TransformChain __all__ = [ - "CaptureOutputOp", "ConfigureOp", "CopyInputOp", "CopyMetadataOp", @@ -80,11 +61,7 @@ "Mix", "Save", "Use", - "FftShiftOp", "FormulaOp", - "FourierOp", - "IfftShiftOp", - "InverseFourierOp", "EncodeTargetOp", "MetadataToTargetOp", "CocoToTorchVisionDetectionOp", @@ -93,17 +70,14 @@ "RandomApply", "RescaleOp", "SampleSinkOp", - "SpectrumScalingOp", "SqueezeOp", "StandardizeOp", "StashInputOp", "StashTargetOp", "SwapInputTargetOp", - "Tee", "TransformChain", "ToTensorOp", "UnstashInputOp", "UnstashTargetOp", "UnsqueezeOp", - "WindowOp", ] diff --git a/sampleflux/ops/capture.py b/sampleflux/ops/capture.py deleted file mode 100644 index 29bb7e3..0000000 --- a/sampleflux/ops/capture.py +++ /dev/null @@ -1,120 +0,0 @@ -"""``CaptureOutputOp`` — record an op's declared ``@output`` value into sample metadata. - -Wraps a target op: applies it to the sample (so the op's ``@output`` properties take their -post-call values), then copies one or more of those ``@output`` attributes off the *live op -instance* into ``metadata[key]``, and returns the applied sample. - -This is the capture half of FluxStudio's "wire one op's runtime ``@output`` into a LATER op's -parameter" feature. A canvas wire from e.g. ``NoiseFloorOp.applied_snr_db`` into another op's -parameter compiles to a ``CaptureOutputOp`` (records the producer's ACTUAL drawn value) followed -by a ``ConfigureOp(ops=[UnstashInputOp(key)], target=…, param=…)`` that injects it per sample. -The value MUST be captured from the real run — many ``@output``\\ s are stochastic -(``applied_snr_db`` is a random SNR draw) and so cannot be re-derived by re-running the op. - -Modality-neutral — it threads any ``Sample`` through any op — so it lives in core sampleflux -(``compose`` group, alongside ``ConfigureOp`` / ``FormulaOp`` / the stash family). -""" - -from typing import Any, Dict, Optional, cast - -from confluid import configurable, flow -from confluid.fluid import Fluid - -from sampleflux.sample import Sample - -_MISSING = object() - - -@configurable(category="op", group="compose") -class CaptureOutputOp: - """Apply an op, then copy its ``@output`` attribute(s) into the sample metadata. - - The wrapped ``op`` is applied to the incoming sample (its input/target transformations are - KEPT — the returned sample is ``op(sample)``), then each requested ``@output`` attribute is - read off the live ``op`` instance and written to ``metadata[]``. Use ``captures`` to - record SEVERAL outputs from ONE application (so a stochastic op runs exactly once); ``output`` - / ``key`` are the single-output convenience form. - - Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call (like - ``ConfigureOp``), so a ``CaptureOutputOp()`` built from YAML costs nothing. - - YAML — capture ``NoiseFloorOp``'s drawn SNR so a later op can read it back: - - .. code-block:: yaml - - - !class:sampleflux.ops.capture.CaptureOutputOp - op: !class:waivefront.torchsig.processing.NoiseFloorOp {} - output: applied_snr_db - key: __captured_snr - - Args: - op: The op to apply; its ``@output`` attributes are read after it runs. Required at call time, validated lazily. - output: A single ``@output`` attribute name to capture. Blank = capture only the ``captures`` entries. - key: Metadata key for the ``output`` value. Blank (default) = the ``output`` name itself. - captures: Mapping of ``@output`` attribute name -> metadata key, for capturing several outputs in one apply. - """ - - def __init__( - self, - op: Optional[object] = None, - output: str = "", - key: str = "", - captures: Optional[Dict[str, str]] = None, - ) -> None: - # Lazy / zero-arg: store config only; op/outputs are validated at first call. - self.op = op - self.output = str(output) - self.key = str(key) - self.captures = dict(captures) if captures else {} - - def _items(self) -> Dict[str, str]: - """The full ``{output_name: metadata_key}`` map — ``captures`` plus the single-output form.""" - items = dict(self.captures) - if self.output: - items.setdefault(self.output, self.key or self.output) - return items - - @staticmethod - def _read_output(op: Any, name: str) -> Any: - """Read the ``@output`` attribute ``name`` off ``op``, looking THROUGH a ``target`` chain. - - The op may be wrapped (e.g. by a ``ConfigureOp``, which exposes the configured op as - ``.target``) when its own params are also configured — so the ``@output`` lives on the - innermost wrapped op. Walk ``.target`` to the first level that declares ``name``; returns - ``_MISSING`` if no level has it. - """ - cur, seen = op, set() - while cur is not None and id(cur) not in seen: - seen.add(id(cur)) - value = getattr(cur, name, _MISSING) - if value is not _MISSING: - return value - cur = getattr(cur, "target", None) - return _MISSING - - def __call__(self, sample: Sample) -> Optional[Sample]: - if self.op is None: - raise ValueError("CaptureOutputOp: an 'op' to apply is required") - items = self._items() - if not items: - raise ValueError("CaptureOutputOp: nothing to capture — set 'output' (and 'key') or 'captures'") - if isinstance(self.op, Fluid): - self.op = flow(self.op) - op = cast(Any, self.op) - result = op(sample) - if result is None: - return None # the wrapped op filtered the sample (FilterOp semantics) - for name, meta_key in items.items(): - value = self._read_output(op, name) - if value is _MISSING: - raise AttributeError( - f"CaptureOutputOp: {type(op).__name__!r} has no @output attribute {name!r} to capture" - ) - result.meta[meta_key] = value - return cast(Optional[Sample], result) - - def close(self) -> None: - """Propagate close() to the wrapped op if it owns resources.""" - close_fn = getattr(self.op, "close", None) - if callable(close_fn): - close_fn() diff --git a/sampleflux/ops/configure.py b/sampleflux/ops/configure.py index be44321..ed32370 100644 --- a/sampleflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -9,7 +9,7 @@ to the ORIGINAL sample. Modality-neutral — it threads any ``Sample`` through any ops — so it lives in core -sampleflux (compose group, alongside ``Tee`` / ``Enable`` / ``RandomApply``). +sampleflux (compose group, alongside ``TransformChain`` / ``Enable`` / ``RandomApply``). """ from typing import Any, List, Optional, cast @@ -31,7 +31,7 @@ class ConfigureOp: attribute of ``target``, then ``target`` is applied to the original sample. Confluid ``!class:`` / ``!lazy:`` markers in ``ops`` / ``target`` are flowed lazily at - first call (like ``Tee``), so a ``ConfigureOp()`` built from YAML costs nothing. + first call (like ``TransformChain``), so a ``ConfigureOp()`` built from YAML costs nothing. YAML — a per-sample threshold (the helios ``Configure(TimeInSamples, CropToSize)`` shape, here deriving ``ThresholdOp.low_level`` from the sample's own statistics): diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py index 6affe1f..2d104ef 100644 --- a/sampleflux/ops/context.py +++ b/sampleflux/ops/context.py @@ -33,7 +33,7 @@ def _flow_if_fluid(value: Any) -> Any: def _read_output(op: Any, name: str) -> Any: """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. - Mirrors ``CaptureOutputOp._read_output`` but also descends our own ``Apply.op`` slot so + Reads a live ``@output`` attribute through wrapper chains (incl. our own ``Apply.op`` slot) so ``Capture(op=Apply(op=X, …))`` reaches X's ``@output``. Returns ``_MISSING`` when absent. """ cur, seen = op, set() @@ -192,7 +192,7 @@ def close(self) -> None: class Capture: """Apply an op, then record its ``@output`` attribute(s) into Context cells. - The Context twin of ``CaptureOutputOp``: the wrapped op runs once (stochastic-correct + Records a wrapped op's live ``@output``: the wrapped op runs once (stochastic-correct — the value is read from the actual run, never recomputed) and each requested ``@output`` is stored as a raw cell value for a later ``Apply``/``Mix`` to read. The returned sample is ``op(sample)`` — transformations are kept. diff --git a/sampleflux/ops/copy.py b/sampleflux/ops/copy.py index 197c6e5..a4b342d 100644 --- a/sampleflux/ops/copy.py +++ b/sampleflux/ops/copy.py @@ -1,9 +1,9 @@ """Defensive deepcopy ops. Use these when a downstream op mutates ``sample.input`` / ``sample.target`` -in place and you want subsequent branches (or external references) to see -the pre-mutation value. ``Tee`` shares a single ``Sample`` across all -branches by design, so isolation is opt-in via these ops. +in place and you want later readers (or external references) to see the +pre-mutation value — e.g. before handing a sample to an in-place library +call, or to decouple a snapshot from the live stream. """ import copy diff --git a/sampleflux/ops/debug.py b/sampleflux/ops/debug.py index 5f341ce..af56a9e 100644 --- a/sampleflux/ops/debug.py +++ b/sampleflux/ops/debug.py @@ -61,7 +61,7 @@ class PrintSampleOp: A pipeline probe: emits a compact description of the sample — ``input`` / ``target`` shape+dtype plus a length-capped value preview (large arrays elided), and the ``metadata`` (values summarised the same way) — to the Loggair logger (the LOG file + console) and, by default, to stdout via - ``print`` (so it shows in a terminal / the FluxStudio node output panel regardless of log + ``print`` (so it shows in a terminal / a GUI node output panel regardless of log level). The sample is returned UNCHANGED. Args: diff --git a/sampleflux/ops/enable.py b/sampleflux/ops/enable.py index f532686..291fa5b 100644 --- a/sampleflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -1,6 +1,6 @@ """``Enable`` — toggle one or more ops on/off via a single named CLI flag. -A compose-group op (alongside ``Tee`` / ``Parallel``): wrap an inner op-list +A compose-group op (alongside ``TransformChain`` / ``Parallel``): wrap an inner op-list so the whole chain can be switched on or off from one boolean attribute whose name becomes the CLI flag. Modality-neutral — it threads any ``Sample`` through any ops — so it lives in core sampleflux, not a domain package. diff --git a/sampleflux/ops/formula.py b/sampleflux/ops/formula.py index 8463e86..82da10f 100644 --- a/sampleflux/ops/formula.py +++ b/sampleflux/ops/formula.py @@ -1,6 +1,6 @@ """``FormulaOp`` — evaluate a math formula over ``sample.input``. -The op-form of FluxStudio's canvas *Math* node: a restricted Python expression over one +The op-form of a visual canvas *Math* node: a restricted Python expression over one named variable bound to the incoming ``sample.input`` (plus the stdlib ``math`` namespace and the scalar helpers ``abs``/``min``/``max``/``round``/``pow`` — no builtins, so ``__import__``/``open``/``exec`` are unavailable). Its main consumer is the ops-export's diff --git a/sampleflux/ops/image.py b/sampleflux/ops/image.py index 393e943..eff0486 100644 --- a/sampleflux/ops/image.py +++ b/sampleflux/ops/image.py @@ -2,11 +2,11 @@ This is the single home for "turn an arbitrary value into an image": the :class:`ConvertToImageOp` op plus the library functions -(:func:`value_to_image` / :func:`sample_to_image`) that back it and FluxStudio's +(:func:`value_to_image` / :func:`sample_to_image`) that back it and the GUI sample preview. It lives in sampleflux (not waivefront) because the conversion is fully generic — a 2-D map, a CHW tensor, a PIL image, a boolean mask all render the same way regardless of domain — so every project (waivefront's spectrogram -render, any image dataset preview, FluxStudio nodes) reuses ONE implementation. +render, any image dataset preview, GUI viewer nodes) reuses ONE implementation. Domain-specific rendering stays in the consuming package: waivefront's ``RenderOverlaysOp`` draws signal-region rectangles on top of the PIL image this @@ -35,9 +35,9 @@ # Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob # across the workspace (``value_to_image`` / ``sample_to_image`` / ``ConvertToImageOp`` and, via -# re-export, waivefront's renderers) AND for FluxStudio's colormap dropdown (which reads ``COLORMAPS``). +# re-export, waivefront's renderers) AND for GUI colormap dropdowns (which read ``COLORMAPS``). # A closed ``Literal`` (never a bare ``str``) makes the choice self-documenting and machine- -# introspectable: the FluxStudio palette, navigaitor's form-spec, and MCP tool schemas enumerate the +# introspectable: visual-editor palettes, navigaitor's form-spec, and MCP tool schemas enumerate the # options straight from the annotation via ``typing.get_args`` instead of hard-coding a parallel list # that silently drifts. ``"gray"`` is the greyscale path (special-cased in ``_apply_colormap``); every # other name resolves through ``matplotlib.colormaps[name]``. Per the workspace "closed Literal" @@ -148,7 +148,7 @@ def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 5 """Render an arbitrary value (a Sample's ``input`` OR ``target``) to an ``(H, W, 3)`` uint8 RGB image. A generic, modality-agnostic preview usable from any SampleFlux pipeline (and - by FluxStudio's sample extractor, which renders the selected field). Handles: + by a GUI sample extractor, which renders the selected field). Handles: * ``PIL.Image`` — converted to RGB; * ``torch.Tensor`` — detached to numpy (CHW collapsed to HWC below); @@ -189,12 +189,12 @@ def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: in # --------------------------------------------------------------------------- # # Array introspection helpers — channel selection + histogram. # -# These back FluxStudio's "Array / Tensor Histogram" viewer node (and are usable +# These back GUI "Array / Tensor Histogram" viewer nodes (and are usable # from any pipeline / notebook): a generic, modality-agnostic way to look at the # RAW numeric values of an array/tensor — pick a channel, render it, and bin its # values. Pure functions (NOT @configurable ops): they measure/derive, they don't # transform a Sample, so they're library helpers like value_to_image — not canvas -# nodes. They live here (not in the FluxStudio node) so the computation is reusable +# nodes. They live here (not in the GUI node) so the computation is reusable # and unit-tested, per the workspace "rendering/analysis lives in sampleflux" mandate. # --------------------------------------------------------------------------- # @@ -238,7 +238,7 @@ def _channel_axis(shape: Tuple[int, ...]) -> int: Deliberately distinct from the other two channel heuristics in this workspace, each scoped to a narrower job: :func:`_render_rgb`'s ``{1,3,4}``-membership test is RGB-render-specific (it only - recognises 1/3/4-channel *images*), and ``fluxstudio.nodes.SampleExtractorNode._as_2d`` is + recognises 1/3/4-channel *images*), and a GUI extractor's float-only mask rule is mask-specific (float-only). For a general N-channel feature map (e.g. an 8-channel tensor) the smallest-axis rule is the most defensible default; documented here so the three never look like an accidental disagreement. @@ -363,7 +363,7 @@ def confusion_matrix_payload( ) -> Dict[str, Any]: """Structure a confusion matrix + class names into a JSON-safe payload for a GUI viewer. - Backs FluxStudio's *Confusion Matrix* viewer node (``fluxstudio.nodes.ConfusionMatrixViewerNode``). + Backs GUI *Confusion Matrix* viewer nodes. The MATH that lives here is the three normalizations (the viewer toggles between them WITHOUT a re-run — the JS only colours + labels + hovers): ``true`` (each row / actual-class sums to 1), ``pred`` (each column / predicted-class sums to 1) and ``all`` (the whole matrix sums to 1). Every @@ -440,7 +440,7 @@ def confusion_matrices_payload(metrics: Any, class_names: Optional[Sequence[Any] ``_is_square_2d``, by SHAPE not name), returning one :func:`confusion_matrix_payload` per match (each tagged with its metric ``name``) in dict order, or ``[]`` when none. A bare square-2D ``metrics`` (not a dict) is treated as a single matrix named ``"confusion_matrix"``. This is what - lets FluxStudio's *Confusion Matrix* viewer render ALL matrices from one all-metrics output (there + lets a GUI *Confusion Matrix* viewer render ALL matrices from one all-metrics output (there can be several). ``class_names`` labels every matrix the same way (they share the class set). Args: @@ -469,7 +469,7 @@ def _sanitize_finite(x: float) -> Optional[float]: # --------------------------------------------------------------------------- # # Closed 9-grid set of text anchor positions (a closed Literal per the workspace mandate, so the -# choice is a dropdown in FluxStudio / navigaitor enumerated from one source of truth). +# choice is a dropdown in GUIs / navigaitor enumerated from one source of truth). TextPosition = Literal[ "top-left", "top", @@ -532,7 +532,7 @@ def draw_text( ) -> np.ndarray: """Render ``text`` onto ``image`` (or a fresh ``background`` canvas) → an ``(H, W, 3)`` uint8 RGB array. - The single, modality-agnostic "draw text on an image" renderer (FluxStudio's *Draw Text to Image* + The single, modality-agnostic "draw text on an image" renderer (a GUI *Draw Text to Image* node is thin glue over it). When ``image`` is ``None`` a blank ``(height, width)`` canvas of color ``background`` is created; otherwise the value is coerced to an RGB image (via :func:`_render_rgb`, so PIL / ndarray / tensor / 2-D maps all work) and drawn on a copy. The text is word-wrapped to the diff --git a/sampleflux/ops/metadata.py b/sampleflux/ops/metadata.py index 48f1253..16057cf 100644 --- a/sampleflux/ops/metadata.py +++ b/sampleflux/ops/metadata.py @@ -20,19 +20,19 @@ class DropMetadataOp: A key is DROPPED when it matches an ``exclude`` pattern AND does NOT match any ``include`` pattern — so ``include`` PROTECTS keys and takes priority over ``exclude`` (the rsync / gitignore include-wins model). Strips bookkeeping you don't want a downstream sink to - serialise — e.g. the internal ``__taidal_stash_*`` snapshots FluxStudio's DAG -> sequential - export leaves on the metadata bus (a stashed complex signal). The replacement metadata is a + serialise — e.g. a bulky ``Stash*Op`` snapshot (a stashed complex signal) kept on the + metadata bus for a ``Parallel`` crossing. The replacement metadata is a fresh dict (copy-on-write); ``input`` / ``target`` are untouched. Single-sample only (reads ``sample.meta``), like ``CopyMetadataOp`` — drop keys before collation. Args: exclude: Glob patterns (``fnmatch``: ``*`` = any run, ``?`` = one char, ``[seq]`` = a set) for keys to REMOVE. A pattern with NO wildcards matches that key exactly. Case-sensitive. - E.g. ``__taidal_stash*`` removes every auto-stash snapshot; with no ``exclude`` nothing + E.g. ``spec_*`` removes every spec-prefixed snapshot; with no ``exclude`` nothing is dropped. include: Glob patterns for keys to KEEP even when they match ``exclude`` — higher priority, - so it carves exceptions out of ``exclude``. E.g. ``exclude=["__taidal_stash*"]`` + - ``include=["__taidal_stash_456:*"]`` drops every stash key EXCEPT node 456's. ``include`` + so it carves exceptions out of ``exclude``. E.g. ``exclude=["spec_*"]`` + + ``include=["spec_keep"]`` drops every ``spec_*`` key EXCEPT the protected one. ``include`` only ever protects against ``exclude`` (with no ``exclude`` it has no effect). """ diff --git a/sampleflux/ops/numpy.py b/sampleflux/ops/numpy.py index a4293f5..c4a4626 100644 --- a/sampleflux/ops/numpy.py +++ b/sampleflux/ops/numpy.py @@ -9,24 +9,10 @@ from sampleflux.sample import Sample from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType -from sampleflux.windows import ( - WINDOW_SUM_KEY, - WINDOW_SUMSQ_KEY, - SpectrumScaling, - WindowName, - get_window, - scale_spectrum, - window_metadata, - window_sums, -) # Common shorthands for the numpy ops' declared types. _NDARRAY = ArrayType(frameworks={"numpy"}) _NUMERIC_OR_PIL = UnionType((ArrayType(dtype="numeric", frameworks={"numpy"}), PythonType("PIL.Image.Image"))) -# Spectrum-scaling ops emit complex (none/amplitude) OR real (power/density) — a permissive union. -_COMPLEX_OR_FLOAT = UnionType( - (ArrayType(dtype="complex", frameworks={"numpy"}), ArrayType(dtype="floating", frameworks={"numpy"})) -) logger = get_logger(__name__) @@ -311,7 +297,7 @@ def __call__(self, sample: Sample) -> Sample: # ThresholdOp comparison selectors. Closed ``Literal``s (workspace "prefer closed -# Literals over bare strings" mandate) so FluxStudio / navigaitor render the choice +# Literals over bare strings" mandate) so GUIs / schema generators render the choice # as a dropdown and the allowed operators stay machine-introspectable via # ``typing.get_args(...)``. Two distinct types because the lower bound only sensibly # uses ``>`` / ``>=`` and the upper bound only ``<`` / ``<=``. @@ -668,320 +654,3 @@ def __call__(self, sample: Sample) -> Sample: # validates min_area_bins / connectivity and raises the scipy ImportError. bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) return sample._replace(input=bboxes) - - -def _apply_window(arr: np.ndarray, window: np.ndarray, axis: int) -> np.ndarray: - """Multiply ``arr`` by the 1-D ``window`` broadcast along ``axis`` (dtype-preserving). - - The window is cast to the real dtype matching ``arr`` so a ``complex64`` / ``float32`` signal - keeps its precision (a raw ``float64`` window would otherwise upcast it). - """ - if np.issubdtype(arr.dtype, np.floating) or np.issubdtype(arr.dtype, np.complexfloating): - window = window.astype(arr.real.dtype, copy=False) - moved = np.swapaxes(arr, axis, -1) - return np.swapaxes(moved * window, axis, -1) - - -def _resolve_fft_sample_rate(sample: Sample, explicit: Optional[float]) -> Optional[float]: - """Resolve the density-scaling sample rate: explicit arg → ``metadata['samplerate']`` → ``None``.""" - if explicit is not None: - return explicit - if sample.is_batched: - return None - raw = sample.meta.get("samplerate") - return float(raw) if raw else None - - -# FFT normalization mode. Closed ``Literal`` (workspace "prefer closed Literals over bare strings" -# mandate) so FluxStudio / navigaitor render the choice as a dropdown and the allowed modes stay -# machine-introspectable via ``typing.get_args(...)``. These three strings are EXACTLY what -# ``numpy.fft.fft`` accepts for its ``norm=`` argument (the same set ``torch.fft.fft`` uses), passed -# straight through with no parallel runtime tuple to drift. -FourierNorm = Literal["backward", "ortho", "forward"] - - -@configurable(category="op", group="numpy") -class FourierOp: - """Compute the 1-D discrete Fourier transform of ``sample.input`` (``numpy.fft.fft``). - - Accepts real **and** complex arrays; the raw output is always complex — ``complex64`` for - ``float32``/``complex64`` input, ``complex128`` for ``float64``/integer/``complex128`` input - (numpy's promotion rule). This is the **1-D** transform (``numpy.fft.fft``), not the 2-D / N-D - one: for an N-D array it runs along a single ``axis`` (default the last), so a ``[B, N]`` batch - of signals transforms per row. :class:`InverseFourierOp` is the inverse; set ``shift=True`` to - center the zero-frequency bin (the standalone :class:`FftShiftOp` does the same independently). - - **Windowing & units.** ``window`` applies a :func:`sampleflux.windows.get_window` taper before the - transform (default ``"boxcar"`` = no taper = unchanged behaviour) and stashes the window - correction into the metadata; ``scaling`` then returns the spectrum in real units — - ``"amplitude"`` (V), ``"power"`` (V²) or ``"density"`` (V²/Hz, using ``sample_rate``) — dividing - out the window's coherent gain / noise bandwidth. ``scaling="none"`` (default) leaves the raw - complex spectrum. The one-node ``FourierOp(window="hann", scaling="density", sample_rate=…)`` is - equivalent to the explicit chain ``WindowOp(window="hann") → FourierOp() → - SpectrumScalingOp(scaling="density", sample_rate=…)``. Calibrated ``scaling`` requires the - unscaled transform (``norm="backward"``); any other ``norm`` with ``scaling != "none"`` raises. - - Args: - n: Output length along ``axis`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. - axis: Axis to transform over. Default ``-1`` (the last axis — the natural choice for a 1-D signal). - norm: Normalization — ``"backward"`` (default, unscaled forward), ``"ortho"`` (1/sqrt(n) both ways), - or ``"forward"`` (1/n on the forward transform). Calibrated ``scaling`` requires ``"backward"``. - shift: When True, ``fftshift`` along ``axis`` after transforming (centers the zero bin). Default False. - window: Taper applied before the FFT — a ``WindowName`` (``"boxcar"`` default = no taper). - window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. - periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. - scaling: Units — ``"none"`` (complex, default), ``"amplitude"`` V, ``"power"`` V², ``"density"`` V²/Hz. - sample_rate: Hz, for ``"density"``. ``None`` reads ``metadata["samplerate"]``, else ``1.0`` (normalized). - one_sided: Fold to a one-sided spectrum (real signals). Default ``False``; exclusive with ``shift``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_COMPLEX_OR_FLOAT) - - def __init__( - self, - n: Optional[int] = None, - axis: int = -1, - norm: FourierNorm = "backward", - shift: bool = False, - window: WindowName = "boxcar", - window_param: Optional[float] = None, - periodic: bool = True, - scaling: SpectrumScaling = "none", - sample_rate: Optional[float] = None, - one_sided: bool = False, - ) -> None: - # Lazy / zero-arg: store config only. ``n`` and window params are validated lazily in __call__. - self.n = n - self.axis = axis - self.norm = norm - self.shift = bool(shift) - self.window: WindowName = window - self.window_param = window_param - self.periodic = bool(periodic) - self.scaling: SpectrumScaling = scaling - self.sample_rate = sample_rate - self.one_sided = bool(one_sided) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "FourierOp") - if self.scaling != "none" and self.norm != "backward": - raise ValueError( - f"FourierOp: calibrated scaling={self.scaling!r} requires norm='backward' " - f"(the unscaled transform); got norm={self.norm!r}" - ) - if self.shift and self.one_sided: - raise ValueError("FourierOp: shift and one_sided are mutually exclusive (one_sided is a half-spectrum)") - window = None - signal = arr - if self.window != "boxcar": - window = get_window( - self.window, arr.shape[self.axis], window_param=self.window_param, periodic=self.periodic - ) - signal = _apply_window(arr, window, self.axis) - out = np.fft.fft(signal, n=self.n, axis=self.axis, norm=self.norm) - if self.scaling != "none": - s1, s2 = window_sums(window) if window is not None else (float(arr.shape[self.axis]),) * 2 - out = scale_spectrum( - out, - self.scaling, - s1=s1, - s2=s2, - sample_rate=_resolve_fft_sample_rate(sample, self.sample_rate), - one_sided=self.one_sided, - axis=self.axis, - ) - if self.shift: - out = np.fft.fftshift(out, axes=self.axis) - if window is not None and not sample.is_batched: - new_meta = dict(sample.meta) - new_meta.update(window_metadata(self.window, window)) - return sample._replace(input=out, metadata=new_meta) - return sample._replace(input=out) - - -@configurable(category="op", group="numpy") -class InverseFourierOp: - """Compute the 1-D inverse discrete Fourier transform of ``sample.input`` (``numpy.fft.ifft``). - - The sibling of :class:`FourierOp`: it maps a spectrum back to the time domain. The output is - always complex (``numpy.fft.ifft`` always returns complex; take ``.real`` downstream if the - original signal was real). ``InverseFourierOp(norm=…)`` must use the **same** ``norm`` as the - forward transform to round-trip. With ``shift=True`` an ``ifftshift`` is applied to the input - **before** the inverse transform, exactly undoing a prior ``FourierOp(shift=True)`` (the correct - pairing even for odd-length axes). - - Args: - n: Output length along ``axis`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. - axis: Axis to transform over. Default ``-1`` (the last axis — the natural choice for a 1-D signal). - norm: Normalization — must match the forward transform: ``"backward"`` (default), ``"ortho"``, or ``"forward"``. - shift: When True, ``ifftshift`` along ``axis`` before inverting (undoes a prior ``fftshift``). Default False. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=ArrayType(dtype="complex", frameworks={"numpy"})) - - def __init__( - self, n: Optional[int] = None, axis: int = -1, norm: FourierNorm = "backward", shift: bool = False - ) -> None: - # Lazy / zero-arg: store config only. ``n`` (if set) is validated lazily by numpy in __call__. - self.n = n - self.axis = axis - self.norm = norm - self.shift = bool(shift) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "InverseFourierOp") - if self.shift: - arr = np.fft.ifftshift(arr, axes=self.axis) - out = np.fft.ifft(arr, n=self.n, axis=self.axis, norm=self.norm) - return sample._replace(input=out) - - -@configurable(category="op", group="numpy") -class FftShiftOp: - """Shift the zero-frequency component to the center of the spectrum (``numpy.fft.fftshift``). - - A pure bin-rearrangement — no FFT is computed, so it is dtype- AND shape-preserving and works - on **any** array (real, complex, or integer). Chain it after :class:`FourierOp` to center a - spectrum for display (the ``FourierOp(shift=True)`` flag is the one-node convenience), or use it - standalone to center an already-computed spectrum such as a 2-D spectrogram. :class:`IfftShiftOp` - is its exact inverse (they differ only for odd-length axes). - - Args: - axis: Axis to shift. Default ``-1`` (last axis, matches :class:`FourierOp`); ``None`` shifts every axis. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: Optional[int] = -1) -> None: - self.axis = axis - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "FftShiftOp") - return sample._replace(input=np.fft.fftshift(arr, axes=self.axis)) - - -@configurable(category="op", group="numpy") -class IfftShiftOp: - """Undo an :class:`FftShiftOp` — move the center frequency back to index 0 (``numpy.fft.ifftshift``). - - The exact inverse of :class:`FftShiftOp` (the two coincide for even-length axes but differ for - odd-length ones, which is why both exist). Like its sibling it is a pure, dtype- and - shape-preserving rearrangement that accepts any array. Apply it before :class:`InverseFourierOp` - to recover the natural FFT bin order (``InverseFourierOp(shift=True)`` folds it in). - - Args: - axis: Axis to shift. Default ``-1`` (last axis, matches :class:`InverseFourierOp`); ``None`` shifts every axis. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: Optional[int] = -1) -> None: - self.axis = axis - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "IfftShiftOp") - return sample._replace(input=np.fft.ifftshift(arr, axes=self.axis)) - - -@configurable(category="op", group="numpy") -class WindowOp: - """Apply a window taper to ``sample.input`` and record the unit-scaling correction. - - Multiplies the signal by a :func:`sampleflux.windows.get_window` taper (broadcast along ``axis``) - — the standard first step of spectral analysis, controlling FFT spectral leakage — and stashes - the window's correction factors into ``sample.metadata`` (``window`` / ``window_sum`` ``S1`` / - ``window_sum_sq`` ``S2`` / ``window_enbw_bins`` / ``window_coherent_gain``) so a later - :class:`SpectrumScalingOp` can divide them out and return the spectrum in real units. Chain - ``WindowOp → FourierOp → SpectrumScalingOp``, or fold all three into one node via - ``FourierOp(window=…, scaling=…)``. Shape-preserving; real input stays real, complex stays - complex (the taper is cast to the input's real dtype so precision is preserved). - - Args: - window: Which taper — a ``WindowName`` (default ``"hann"``; ``"boxcar"`` is the rectangular identity). - window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. - periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. - axis: Axis the window is applied along. Default ``-1`` (the last axis — the 1-D signal). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__( - self, - window: WindowName = "hann", - window_param: Optional[float] = None, - periodic: bool = True, - axis: int = -1, - ) -> None: - # Lazy / zero-arg: store config only; window params are validated lazily by get_window. - self.window: WindowName = window - self.window_param = window_param - self.periodic = bool(periodic) - self.axis = axis - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "WindowOp") - window = get_window(self.window, arr.shape[self.axis], window_param=self.window_param, periodic=self.periodic) - out = _apply_window(arr, window, self.axis) - if sample.is_batched: - return sample._replace(input=out) - new_meta = dict(sample.meta) - new_meta.update(window_metadata(self.window, window)) - return sample._replace(input=out, metadata=new_meta) - - -@configurable(category="op", group="numpy") -class SpectrumScalingOp: - """Scale a (complex) FFT spectrum to physical units using the window correction. - - The calibration half of the FFT chain: turns the raw :class:`FourierOp` output into an amplitude - (V), power (V²) or power-spectral-density (V²/Hz) spectrum, dividing out the window's coherent - gain ``S1`` / noise bandwidth ``S2`` — read from the ``window_*`` metadata stashed by - :class:`WindowOp` or ``FourierOp(window=…)``; if absent it assumes a rectangular/boxcar window - (``S1=S2=N``). Assumes the spectrum came from the **unscaled** forward transform - (``norm="backward"``, the FourierOp default). Output dtype follows the mode — complex for - ``"none"``/``"amplitude"`` (phase preserved), real for ``"power"``/``"density"``. - - Args: - scaling: Units — ``"none"`` (unchanged), ``"amplitude"`` V, ``"power"`` V² (default), ``"density"`` V²/Hz. - sample_rate: Hz, for ``"density"``. ``None`` (default) reads ``metadata["samplerate"]``, else ``1.0``. - one_sided: Fold to one-sided (real-signal convention: keep 0…N/2, double interior bins). Default ``False``. - axis: Spectrum axis. Default ``-1``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_COMPLEX_OR_FLOAT) - - def __init__( - self, - scaling: SpectrumScaling = "power", - sample_rate: Optional[float] = None, - one_sided: bool = False, - axis: int = -1, - ) -> None: - # Lazy / zero-arg: store config only. - self.scaling: SpectrumScaling = scaling - self.sample_rate = sample_rate - self.one_sided = bool(one_sided) - self.axis = axis - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "SpectrumScalingOp") - n = arr.shape[self.axis] - if sample.is_batched: - s1 = s2 = float(n) - else: - meta = sample.meta - raw_s1, raw_s2 = meta.get(WINDOW_SUM_KEY), meta.get(WINDOW_SUMSQ_KEY) - s1, s2 = ( - (float(raw_s1), float(raw_s2)) if raw_s1 is not None and raw_s2 is not None else (float(n), float(n)) - ) - fs = _resolve_fft_sample_rate(sample, self.sample_rate) - if self.scaling == "density" and not fs: - logger.debug("SpectrumScalingOp: no sample_rate for density; using normalized frequency (Fs=1.0)") - out = scale_spectrum(arr, self.scaling, s1=s1, s2=s2, sample_rate=fs, one_sided=self.one_sided, axis=self.axis) - return sample._replace(input=out) diff --git a/sampleflux/ops/parallel.py b/sampleflux/ops/parallel.py index a42a2bf..21898ff 100644 --- a/sampleflux/ops/parallel.py +++ b/sampleflux/ops/parallel.py @@ -11,7 +11,7 @@ Note: Do not nest a ``Parallel`` op inside another ``Parallel.ops`` — workers - must not themselves spawn workers. ``Tee``, ``Enable``, and any + must not themselves spawn workers. ``TransformChain``, ``Enable``, and any pickle-safe per-sample op are fine inside. """ @@ -45,7 +45,7 @@ def __init__(self, ops: Optional[List[Any]] = None, workers: int = 4) -> None: def _materialize_ops(self) -> None: # Confluid post-construction paradigm leaves nested ops as Fluid - # markers; resolve them in-place on first use, mirroring Tee. + # markers; resolve them in-place on first use, mirroring TransformChain. for i, op in enumerate(self.ops): if isinstance(op, Fluid): self.ops[i] = flow(op) diff --git a/sampleflux/ops/random_apply.py b/sampleflux/ops/random_apply.py index 3b23901..891ab7e 100644 --- a/sampleflux/ops/random_apply.py +++ b/sampleflux/ops/random_apply.py @@ -1,6 +1,6 @@ """``RandomApply`` — apply an op with a given probability. -A compose-group op (alongside ``Enable`` / ``Tee`` / ``Parallel``): +A compose-group op (alongside ``Enable`` / ``TransformChain`` / ``Parallel``): wrap any single ``Sample → Sample`` op so it fires only *p* fraction of the time. Samples that are skipped pass through unchanged. diff --git a/sampleflux/ops/stash.py b/sampleflux/ops/stash.py index f30d88d..bad1adc 100644 --- a/sampleflux/ops/stash.py +++ b/sampleflux/ops/stash.py @@ -1,19 +1,24 @@ """Stash / unstash ``sample.input`` / ``sample.target`` to / from ``metadata``. Use ``StashInputOp(key)`` to snapshot the current ``sample.input`` under a -metadata key without changing ``sample.input``. Use ``UnstashInputOp(key)`` -later (e.g. inside another ``Tee`` branch) to restore that value into -``sample.input``. ``StashTargetOp`` / ``UnstashTargetOp`` are the exact -``sample.target`` counterparts — together the family is what lets a branchy -canvas graph compile to ONE sequential op-list (FluxStudio's DAG→sequential -ops-export restores the fork-point input/target between branches and -translates a Mix-style fan-in into unstashes). - -The ``Unstash*Op``\\ s default to ``copy=True`` (deepcopy) so two branches -that both unstash the same key are independent — each gets its own array -to mutate. Without the copy, an in-place op like ``ClipPercentilesOp`` -in the first branch would silently corrupt the stashed value seen by the -second branch. +metadata key without changing ``sample.input``; ``UnstashInputOp(key)`` +restores it later. ``StashTargetOp`` / ``UnstashTargetOp`` are the exact +``sample.target`` counterparts. + +Graph WIRING is the job of the context ops (``sampleflux.ops.context`` — +``Save``/``Use``/``Mix`` over per-sample Context cells, see ``docs/graph.md``). +The stash family remains for the two jobs cells cannot do, because the +snapshot rides ``sample.metadata`` WITH the sample: + +* crossing a ``Parallel`` boundary — metadata travels through the stream + split/join; Context cells deliberately raise there; +* deliberately PERSISTING a snapshot into a sink (the metadata key is + serialised alongside the sample unless an ``Unstash*Op`` removes it). + +The ``Unstash*Op``\\ s default to ``copy=True`` (deepcopy) so two readers +of the same key are independent — each gets its own array to mutate. +Without the copy, an in-place op like ``ClipPercentilesOp`` after the first +restore would silently corrupt the stashed value seen by the second. """ import copy as _copy diff --git a/sampleflux/ops/tee.py b/sampleflux/ops/tee.py deleted file mode 100644 index 076f998..0000000 --- a/sampleflux/ops/tee.py +++ /dev/null @@ -1,53 +0,0 @@ -"""``Tee`` — fan-out side effects across N branches on a single sample. - -Branches run sequentially on the **same** ``Sample`` object and **same** -``metadata`` dict. There are no automatic copies; if isolation is needed, -place an explicit ``CopyInputOp`` / ``CopySampleOp`` at the start of a -branch. Later branches see what earlier branches wrote to ``metadata``. - -If any op in any branch returns ``None``, ``Tee`` propagates ``None`` -(consistent with ``FilterOp`` semantics — the whole sample is dropped). -""" - -from typing import Any, List, Optional - -from confluid import configurable, flow -from confluid.fluid import Fluid - -from sampleflux.sample import Sample - - -@configurable(category="op", group="compose") -class Tee: - """Run N op-list branches sequentially on the same sample / metadata. - - Args: - branches: A list of op-lists; each inner list is a chain of callables - ``Sample -> Optional[Sample]`` run in order. - """ - - def __init__(self, branches: Optional[List[List[Any]]] = None) -> None: - # Lazy / zero-arg: store config only; no branches ⇒ __call__ passes the sample through. - self.branches = [list(b) for b in branches] if branches else [] - - def __call__(self, sample: Sample) -> Optional[Sample]: - current: Optional[Sample] = sample - for branch in self.branches: - for i, op in enumerate(branch): - if current is None: - return None - if isinstance(op, Fluid): - op = flow(op) - branch[i] = op - if op is None: - continue - current = op(current) - return current - - def close(self) -> None: - """Propagate close() to inner ops that own resources.""" - for branch in self.branches: - for op in branch: - close_fn = getattr(op, "close", None) - if callable(close_fn): - close_fn() diff --git a/sampleflux/ops/torch.py b/sampleflux/ops/torch.py index 8495a16..e9e6e99 100644 --- a/sampleflux/ops/torch.py +++ b/sampleflux/ops/torch.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional, Sequence, Union +from typing import Optional, Sequence, Union import numpy as np import torch @@ -6,22 +6,9 @@ from sampleflux.sample import Sample from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType -from sampleflux.windows import ( - WINDOW_SUM_KEY, - WINDOW_SUMSQ_KEY, - SpectrumScaling, - WindowName, - get_window, - window_metadata, - window_sums, -) _TORCH = ArrayType(frameworks={"torch"}) _TORCH_FLOAT = ArrayType(dtype="floating", frameworks={"torch"}) -# Spectrum-scaling ops emit complex (none/amplitude) OR real (power/density). -_TORCH_COMPLEX_OR_FLOAT = UnionType( - (ArrayType(dtype="complex", frameworks={"torch"}), ArrayType(dtype="floating", frameworks={"torch"})) -) @configurable(category="op", group="torch") @@ -222,376 +209,3 @@ def __call__(self, sample: Sample) -> Sample: tensor = (tensor - mean_t) / std_t return sample._replace(input=tensor) - - -def _apply_window(tensor: torch.Tensor, window: np.ndarray, dim: int) -> torch.Tensor: - """Multiply ``tensor`` by the 1-D numpy ``window`` broadcast along ``dim`` (dtype/device-preserving).""" - if tensor.is_complex(): - real_dtype = tensor.real.dtype - elif tensor.is_floating_point(): - real_dtype = tensor.dtype - else: - real_dtype = torch.float32 - w = torch.as_tensor(window, dtype=real_dtype, device=tensor.device) - moved = torch.movedim(tensor, dim, -1) - return torch.movedim(moved * w, -1, dim) - - -def _fold_one_sided(spectrum: torch.Tensor, dim: int) -> torch.Tensor: - """Fold a two-sided spectrum (natural order, DC at index 0) to one-sided (mirror of windows.fold_one_sided).""" - moved = torch.movedim(spectrum, dim, -1) - n = moved.shape[-1] - out = moved[..., : n // 2 + 1].clone() - if n % 2 == 0: - out[..., 1:-1] = out[..., 1:-1] * 2 # exclude DC and Nyquist - else: - out[..., 1:] = out[..., 1:] * 2 - return torch.movedim(out, -1, dim) - - -def _scale_spectrum( - spectrum: torch.Tensor, - scaling: SpectrumScaling, - *, - s1: float, - s2: float, - sample_rate: Optional[float], - one_sided: bool, - dim: int, -) -> torch.Tensor: - """Torch mirror of :func:`sampleflux.windows.scale_spectrum` (assumes ``norm="backward"``).""" - if scaling == "none": - out = spectrum - elif scaling == "amplitude": - out = spectrum / s1 - elif scaling == "power": - out = spectrum.abs().square() / (s1 * s1) - elif scaling == "density": - fs = float(sample_rate) if (sample_rate is not None and sample_rate > 0) else 1.0 - out = spectrum.abs().square() / (fs * s2) - else: - raise ValueError(f"unknown scaling {scaling!r}") - return _fold_one_sided(out, dim) if one_sided else out - - -def _resolve_fft_sample_rate(sample: Sample, explicit: Optional[float]) -> Optional[float]: - """Resolve the density rate: explicit → ``metadata['samplerate']`` → ``None`` (mirrors the numpy op).""" - if explicit is not None: - return explicit - if sample.is_batched: - return None - raw = sample.meta.get("samplerate") - return float(raw) if raw else None - - -# FFT normalization mode — see the numpy ``FourierOp`` for the rationale. Exactly the three strings -# ``torch.fft.fft`` accepts for its ``norm=`` argument; a closed ``Literal`` so GUIs enumerate the -# choice via ``typing.get_args(...)``. -FourierNorm = Literal["backward", "ortho", "forward"] - - -@configurable(category="op", group="torch") -class FourierOp: - """Compute the 1-D discrete Fourier transform of ``sample.input`` (``torch.fft.fft``). - - Accepts real **and** complex tensors; the output is always complex — ``complex64`` for - integer / ``float32`` / ``complex64`` input, ``complex128`` for ``float64`` / ``complex128``. - Half-precision (``float16`` / ``bfloat16``) tensors are promoted to ``float32`` first because - ``torch.fft.fft`` does not support them; every other dtype (including integer and bool) is handled - natively (integers auto-promote to ``complex64``). This is the **1-D** transform - (``torch.fft.fft``), not the 2-D / N-D one: for an N-D tensor it runs along a single ``dim`` - (default the last), so a ``[B, N]`` batch transforms per row. :class:`InverseFourierOp` is the - inverse; set ``shift=True`` to center the zero-frequency bin (the standalone :class:`FftShiftOp` - does the same independently). - - **Windowing & units.** Mirrors the numpy ``FourierOp``: ``window`` applies a - :func:`sampleflux.windows.get_window` taper before the transform (default ``"boxcar"`` = none) and - stashes the window correction; ``scaling`` returns the spectrum in real units — ``"amplitude"`` - (V), ``"power"`` (V²) or ``"density"`` (V²/Hz, via ``sample_rate``). ``scaling="none"`` (default) - leaves the raw complex spectrum. Calibrated ``scaling`` requires ``norm="backward"`` (any other - ``norm`` with ``scaling != "none"`` raises). - - Args: - n: Output length along ``dim`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. - dim: Dimension to transform over. Default ``-1`` (the last dim — the natural choice for a 1-D signal). - norm: Normalization — ``"backward"`` (default, unscaled forward), ``"ortho"`` (1/sqrt(n) both ways), - or ``"forward"`` (1/n on the forward transform). Calibrated ``scaling`` requires ``"backward"``. - shift: When True, ``fftshift`` along ``dim`` after transforming (centers the zero bin). Default False. - window: Taper applied before the FFT — a ``WindowName`` (``"boxcar"`` default = no taper). - window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. - periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. - scaling: Units — ``"none"`` (complex, default), ``"amplitude"`` V, ``"power"`` V², ``"density"`` V²/Hz. - sample_rate: Hz, for ``"density"``. ``None`` reads ``metadata["samplerate"]``, else ``1.0`` (normalized). - one_sided: Fold to a one-sided spectrum (real signals). Default ``False``; exclusive with ``shift``. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH_COMPLEX_OR_FLOAT) - - def __init__( - self, - n: Optional[int] = None, - dim: int = -1, - norm: FourierNorm = "backward", - shift: bool = False, - window: WindowName = "boxcar", - window_param: Optional[float] = None, - periodic: bool = True, - scaling: SpectrumScaling = "none", - sample_rate: Optional[float] = None, - one_sided: bool = False, - ) -> None: - # Lazy / zero-arg: store config only. ``n`` and window params are validated lazily in __call__. - self.n = n - self.dim = dim - self.norm = norm - self.shift = bool(shift) - self.window: WindowName = window - self.window_param = window_param - self.periodic = bool(periodic) - self.scaling: SpectrumScaling = scaling - self.sample_rate = sample_rate - self.one_sided = bool(one_sided) - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"FourierOp expects a torch.Tensor, got {type(tensor).__name__}") - if self.scaling != "none" and self.norm != "backward": - raise ValueError( - f"FourierOp: calibrated scaling={self.scaling!r} requires norm='backward' " - f"(the unscaled transform); got norm={self.norm!r}" - ) - if self.shift and self.one_sided: - raise ValueError("FourierOp: shift and one_sided are mutually exclusive (one_sided is a half-spectrum)") - # torch.fft.fft rejects half precision; promote to float32. Integer/bool/float/complex are - # all accepted natively (integers auto-promote to complex64), so leave them untouched. - if tensor.dtype in (torch.float16, torch.bfloat16): - tensor = tensor.float() - window = None - signal = tensor - if self.window != "boxcar": - window = get_window( - self.window, tensor.shape[self.dim], window_param=self.window_param, periodic=self.periodic - ) - signal = _apply_window(tensor, window, self.dim) - out = torch.fft.fft(signal, n=self.n, dim=self.dim, norm=self.norm) - if self.scaling != "none": - s1, s2 = window_sums(window) if window is not None else (float(tensor.shape[self.dim]),) * 2 - out = _scale_spectrum( - out, - self.scaling, - s1=s1, - s2=s2, - sample_rate=_resolve_fft_sample_rate(sample, self.sample_rate), - one_sided=self.one_sided, - dim=self.dim, - ) - if self.shift: - out = torch.fft.fftshift(out, dim=self.dim) - if window is not None and not sample.is_batched: - new_meta = dict(sample.meta) - new_meta.update(window_metadata(self.window, window)) - return sample._replace(input=out, metadata=new_meta) - return sample._replace(input=out) - - -@configurable(category="op", group="torch") -class InverseFourierOp: - """Compute the 1-D inverse discrete Fourier transform of ``sample.input`` (``torch.fft.ifft``). - - The sibling of :class:`FourierOp`: it maps a spectrum back to the time domain. The output is - always complex (``torch.fft.ifft`` always returns complex; take ``.real`` downstream if the - original signal was real). Half-precision (``float16``/``bfloat16``) tensors are promoted to - ``float32`` first (``torch.fft.ifft`` rejects them); other dtypes are handled natively. - ``InverseFourierOp(norm=…)`` must use the **same** ``norm`` as the forward transform to - round-trip. With ``shift=True`` an ``ifftshift`` is applied to the input **before** inverting, - exactly undoing a prior ``FourierOp(shift=True)`` (the correct pairing even for odd-length dims). - - Args: - n: Output length along ``dim`` — zero-pad/truncate to ``n`` points. ``None`` (default) uses the input length. - dim: Dimension to transform over. Default ``-1`` (the last dim — the natural choice for a 1-D signal). - norm: Normalization — must match the forward transform: ``"backward"`` (default), ``"ortho"``, or ``"forward"``. - shift: When True, ``ifftshift`` along ``dim`` before inverting (undoes a prior ``fftshift``). Default False. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=ArrayType(dtype="complex", frameworks={"torch"})) - - def __init__( - self, n: Optional[int] = None, dim: int = -1, norm: FourierNorm = "backward", shift: bool = False - ) -> None: - # Lazy / zero-arg: store config only. ``n`` (if set) is validated lazily by torch in __call__. - self.n = n - self.dim = dim - self.norm = norm - self.shift = bool(shift) - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"InverseFourierOp expects a torch.Tensor, got {type(tensor).__name__}") - # torch.fft.ifft rejects half precision; promote to float32 (mirrors FourierOp). - if tensor.dtype in (torch.float16, torch.bfloat16): - tensor = tensor.float() - if self.shift: - tensor = torch.fft.ifftshift(tensor, dim=self.dim) - out = torch.fft.ifft(tensor, n=self.n, dim=self.dim, norm=self.norm) - return sample._replace(input=out) - - -@configurable(category="op", group="torch") -class FftShiftOp: - """Shift the zero-frequency component to the center of the spectrum (``torch.fft.fftshift``). - - A pure bin-rearrangement — no FFT is computed, so it is dtype- AND shape-preserving and works - on **any** tensor (real, complex, or integer; half precision included). Chain it after - :class:`FourierOp` to center a spectrum for display (the ``FourierOp(shift=True)`` flag is the - one-node convenience), or use it standalone to center an already-computed spectrum such as a 2-D - spectrogram. :class:`IfftShiftOp` is its exact inverse (they differ only for odd-length dims). - - Args: - dim: Dimension to shift. Default ``-1`` (last dim, matches :class:`FourierOp`); ``None`` shifts every dim. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH) - - def __init__(self, dim: Optional[int] = -1) -> None: - self.dim = dim - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"FftShiftOp expects a torch.Tensor, got {type(tensor).__name__}") - return sample._replace(input=torch.fft.fftshift(tensor, dim=self.dim)) - - -@configurable(category="op", group="torch") -class IfftShiftOp: - """Undo an :class:`FftShiftOp` — move the center frequency back to index 0 (``torch.fft.ifftshift``). - - The exact inverse of :class:`FftShiftOp` (the two coincide for even-length dims but differ for - odd-length ones, which is why both exist). Like its sibling it is a pure, dtype- and - shape-preserving rearrangement that accepts any tensor. Apply it before :class:`InverseFourierOp` - to recover the natural FFT bin order (``InverseFourierOp(shift=True)`` folds it in). - - Args: - dim: Dimension to shift. Default ``-1`` (last dim, matches :class:`InverseFourierOp`); ``None`` = all dims. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH) - - def __init__(self, dim: Optional[int] = -1) -> None: - self.dim = dim - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"IfftShiftOp expects a torch.Tensor, got {type(tensor).__name__}") - return sample._replace(input=torch.fft.ifftshift(tensor, dim=self.dim)) - - -@configurable(category="op", group="torch") -class WindowOp: - """Apply a window taper to ``sample.input`` and record the unit-scaling correction (torch mirror). - - The tensor counterpart of :class:`sampleflux.ops.numpy.WindowOp`: multiplies the signal by a - :func:`sampleflux.windows.get_window` taper (broadcast along ``dim``) and stashes the window - correction (``window`` / ``window_sum`` ``S1`` / ``window_sum_sq`` ``S2`` / ``window_enbw_bins`` / - ``window_coherent_gain``) into ``sample.metadata`` for a later :class:`SpectrumScalingOp`. - dtype/device-preserving — real stays real, complex stays complex. - - Args: - window: Which taper — a ``WindowName`` (default ``"hann"``; ``"boxcar"`` is the rectangular identity). - window_param: Kaiser ``β`` (def 8.6) / Tukey ``α`` (def 0.5) / Gaussian ``σ`` std (required); else ignored. - periodic: ``True`` (default) = DFT-even window (correct for FFT analysis); ``False`` = symmetric. - dim: Dimension the window is applied along. Default ``-1`` (the last dim — the 1-D signal). - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH) - - def __init__( - self, - window: WindowName = "hann", - window_param: Optional[float] = None, - periodic: bool = True, - dim: int = -1, - ) -> None: - # Lazy / zero-arg: store config only; window params validated lazily by get_window. - self.window: WindowName = window - self.window_param = window_param - self.periodic = bool(periodic) - self.dim = dim - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"WindowOp expects a torch.Tensor, got {type(tensor).__name__}") - window = get_window(self.window, tensor.shape[self.dim], window_param=self.window_param, periodic=self.periodic) - out = _apply_window(tensor, window, self.dim) - if sample.is_batched: - return sample._replace(input=out) - new_meta = dict(sample.meta) - new_meta.update(window_metadata(self.window, window)) - return sample._replace(input=out, metadata=new_meta) - - -@configurable(category="op", group="torch") -class SpectrumScalingOp: - """Scale a (complex) FFT spectrum to physical units using the window correction (torch mirror). - - The tensor counterpart of :class:`sampleflux.ops.numpy.SpectrumScalingOp`: amplitude (V) / power - (V²) / density (V²/Hz), dividing out the window ``S1``/``S2`` read from the ``window_*`` metadata - (rectangular ``S1=S2=N`` if absent). Assumes the spectrum came from the unscaled forward transform - (``norm="backward"``). Output is complex for ``"none"``/``"amplitude"``, real for - ``"power"``/``"density"``. - - Args: - scaling: Units — ``"none"``, ``"amplitude"`` V, ``"power"`` V² (default), ``"density"`` V²/Hz. - sample_rate: Hz, for ``"density"``. ``None`` (default) reads ``metadata["samplerate"]``, else ``1.0``. - one_sided: Fold to one-sided (real-signal convention: keep 0…N/2, double interior bins). Default ``False``. - dim: Spectrum dimension. Default ``-1``. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH_COMPLEX_OR_FLOAT) - - def __init__( - self, - scaling: SpectrumScaling = "power", - sample_rate: Optional[float] = None, - one_sided: bool = False, - dim: int = -1, - ) -> None: - # Lazy / zero-arg: store config only. - self.scaling: SpectrumScaling = scaling - self.sample_rate = sample_rate - self.one_sided = bool(one_sided) - self.dim = dim - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"SpectrumScalingOp expects a torch.Tensor, got {type(tensor).__name__}") - n = tensor.shape[self.dim] - if sample.is_batched: - s1 = s2 = float(n) - else: - meta = sample.meta - raw_s1, raw_s2 = meta.get(WINDOW_SUM_KEY), meta.get(WINDOW_SUMSQ_KEY) - s1, s2 = ( - (float(raw_s1), float(raw_s2)) if raw_s1 is not None and raw_s2 is not None else (float(n), float(n)) - ) - out = _scale_spectrum( - tensor, - self.scaling, - s1=s1, - s2=s2, - sample_rate=_resolve_fft_sample_rate(sample, self.sample_rate), - one_sided=self.one_sided, - dim=self.dim, - ) - return sample._replace(input=out) diff --git a/sampleflux/ops/transform_chain.py b/sampleflux/ops/transform_chain.py index 8aa0bb9..a6acb17 100644 --- a/sampleflux/ops/transform_chain.py +++ b/sampleflux/ops/transform_chain.py @@ -1,15 +1,15 @@ """``TransformChain`` — group a sequence of ops into a single named unit. -A compose-group op (alongside ``Enable`` / ``Tee`` / ``Parallel``): +A compose-group op (alongside ``Enable`` / ``Parallel``): wrap an ordered list of ``Sample → Sample`` callables so they appear as -one node in FluxStudio (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` +one node on a visual canvas (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` inputs instead of N wired ``SAMPLEFLUX_SAMPLE`` connections) and one named block in a Confluid YAML. Unlike ``Enable`` there is no boolean gate — the chain always fires. Unlike ``Parallel`` there is no worker pool — ops run sequentially in the calling thread. If any op returns ``None`` the chain stops early and -propagates ``None`` (consistent with ``FilterOp`` / ``Tee`` semantics). +propagates ``None`` (consistent with ``FilterOp`` semantics). """ from typing import List, Optional @@ -27,7 +27,7 @@ class TransformChain: """Apply a fixed sequence of ops to every sample, always. Wrap a list of ops into one named unit so they appear as a single node - in FluxStudio (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` inputs) + on a visual canvas (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` inputs) and one block in Confluid YAML instead of N separate connections. If any op in the chain returns ``None`` the remaining ops are skipped diff --git a/sampleflux/paired.py b/sampleflux/paired.py deleted file mode 100644 index bbc52a7..0000000 --- a/sampleflux/paired.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Re-join raw data samples with a sidecar annotation store (the annotation loop). - -The recurring pattern this solves: you have raw data samples (RFUAV I/Q windows, -images, …) coming out of a ``DataSource``, and *separately* a sidecar store of -annotations covering some of them — typically a LabelStudio export that annotaide -writes as a ``sample_id -> record`` JSON mapping. :class:`AnnotationJoinSource` -re-joins the two by a key function so each matched annotation record is attached -to ``Sample.metadata``, ready for training. - - raw data ──annotate (LabelStudio)──▶ annotation store ──AnnotationJoinSource──▶ annotated samples -""" - -from typing import ( - Any, - Callable, - Dict, - Iterable, - Iterator, - Literal, - Optional, - Protocol, - Sequence, - Tuple, - Union, - cast, - runtime_checkable, -) - -from confluid import configurable -from loggair import get_logger - -from sampleflux.discovery import get_callable_path, resolve_callable -from sampleflux.sample import Sample - -logger = get_logger(__name__) - -# Join policy is a closed set. As a Literal it is enforced two ways with no extra -# code: static checkers reject bad values, and Confluid's @configurable validates -# it through pydantic at construction (both the Python and YAML/load paths), so a -# bad policy fails before __init__ runs. It also renders as an enum dropdown in -# the navigaitor form-spec. -Policy = Literal["left_outer", "inner", "right_driven"] - - -@runtime_checkable -class AnnotationStore(Protocol): - """The read contract :class:`AnnotationJoinSource` needs from its annotation store: - membership + lookup + key enumeration (``key -> record``). - - Structural (a ``Protocol``), so it does NOT couple sampleflux to annotaide: - annotaide's ``JSONFileAnnotationStore`` satisfies it — and so does a plain - ``dict`` — without any import or inheritance. The write side (``save`` / - ``delete``) lives in annotaide, not here. - - It is ``@runtime_checkable`` on purpose: Confluid's ``@configurable`` layer - isinstance-validates it at construction, so a non-conforming ``annotations`` - is rejected before ``__init__`` runs (the type does the enforcement — no - manual shape guard needed, mirroring how the ``policy`` ``Literal`` is - validated). Every real store (``dict``, ``JSONFileAnnotationStore``) provides - all three methods. - """ - - def __contains__(self, key: str) -> bool: ... - - def __getitem__(self, key: str) -> Dict[str, Any]: ... - - def keys(self) -> Iterable[str]: ... - - -@configurable -class AnnotationJoinSource: - """Join a data source with a sidecar annotation store via a key function. - - Produces ``Sample`` values where the matched annotation record is flattened into - ``Sample.metadata``. Three join policies cover the scenarios we actually see: - - - ``left_outer``: iterate ``data``; attach the annotation when the key matches, - otherwise emit the sample unannotated. Preserves the data source's ``__len__`` - and ``__getitem__``. (Every sample, annotated where available.) - - ``inner``: same as left_outer, filtered to annotated samples only. - (The labeled subset.) - - ``right_driven``: iterate ``annotations.keys()``; resolve each data sample - via ``data_resolver(key, data)``. Use when annotations are sparse relative - to the data. - - Coarser-granularity joins are expressed by returning a coarser key from - ``key_fn`` so multiple data samples map to the same annotation record. - Use ``extract_fn`` to project the record down to each sample's scope (e.g. - trim a pack-level time-ranged annotation to a single window). Returning - ``None`` from ``extract_fn`` marks the sample as unannotated. - - Args: - data: The data source — any iterable (or ``DataSource``) yielding raw items that - ``Sample.from_any`` can coerce into samples. - annotations: The annotation store — a read-mapping (``key -> record``) - satisfying :class:`AnnotationStore` (``__contains__`` + ``__getitem__`` + - ``keys()``); validated at construction. A plain ``dict`` or annotaide's - ``JSONFileAnnotationStore`` qualifies. - key_fn: ``"module:function"`` path (or a callable) producing the join key - from a sample. Signature: ``(sample: Sample) -> str``. Stored as a path so - the source round-trips through Confluid YAML. - policy: Join policy — one of ``"left_outer"``, ``"inner"``, ``"right_driven"``. - extract_fn: Optional ``"module:function"`` path (or callable) called as - ``extract_fn(record, sample) -> dict | None`` to project the record - per sample. Returning ``None`` marks the sample unannotated. - prefix: Optional string prefix applied to every annotation field when - flattening into ``Sample.metadata``. - store_full_under: If set, also stash the (extracted) record under - ``Sample.metadata[store_full_under]``. - data_resolver: Required for ``right_driven``. ``"module:function"`` path - (or callable) invoked as ``data_resolver(key, data)`` to fetch the data - sample for a given annotation key. - """ - - def __init__( - self, - data: Any = None, - annotations: Optional[AnnotationStore] = None, - key_fn: Union[str, Callable[[Sample], str]] = "", - policy: Policy = "left_outer", - extract_fn: Optional[Union[str, Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]]] = None, - prefix: str = "", - store_full_under: Optional[str] = None, - # data arg is Any (not Iterable[Any]): resolvers are written against a - # concrete source type (e.g. RFUAVSource) and contravariance would reject - # those signatures against a broader annotation. - data_resolver: Optional[Union[str, Callable[[str, Any], Any]]] = None, - ) -> None: - # Lazy / zero-arg: store config only. `annotations` shape is enforced by the AnnotationStore - # Protocol via pydantic at construction; the policy-conditional "right_driven needs a resolver" - # requirement (which a Protocol can't express) is validated lazily in `_iter_right_driven`. - self.data = data - self.annotations: AnnotationStore = annotations if annotations is not None else {} - self.key_fn = get_callable_path(key_fn) if callable(key_fn) else key_fn - self.policy = policy - self.extract_fn = get_callable_path(extract_fn) if callable(extract_fn) else extract_fn - self.prefix = prefix - self.store_full_under = store_full_under - self.data_resolver = get_callable_path(data_resolver) if callable(data_resolver) else data_resolver - - self._key_fn_cache: Optional[Callable[[Sample], str]] = None - self._extract_fn_cache: Optional[Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]] = None - self._data_resolver_cache: Optional[Callable[[str, Any], Any]] = None - self._inner_length: Optional[int] = None - - @property - def _resolved_key_fn(self) -> Callable[[Sample], str]: - if self._key_fn_cache is None: - self._key_fn_cache = resolve_callable(self.key_fn) - return self._key_fn_cache - - @property - def _resolved_extract_fn( - self, - ) -> Optional[Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]]: - if self.extract_fn is None: - return None - if self._extract_fn_cache is None: - self._extract_fn_cache = resolve_callable(self.extract_fn) - return self._extract_fn_cache - - @property - def _resolved_data_resolver(self) -> Callable[[str, Any], Any]: - if self.data_resolver is None: - raise ValueError("data_resolver is not set") - if self._data_resolver_cache is None: - self._data_resolver_cache = resolve_callable(self.data_resolver) - return self._data_resolver_cache - - def _attach(self, sample: Sample, record: Optional[Dict[str, Any]], key: str) -> Sample: - metadata = dict(sample.meta) - metadata["annotation_key"] = key - metadata["annotated"] = record is not None - - if record is not None: - for k, v in record.items(): - metadata[f"{self.prefix}{k}"] = v - if self.store_full_under is not None: - metadata[self.store_full_under] = record - - return sample._replace(metadata=metadata) - - def _lookup(self, sample: Sample) -> Tuple[str, Optional[Dict[str, Any]]]: - key = self._resolved_key_fn(sample) - if key not in self.annotations: - return key, None - record: Optional[Dict[str, Any]] = self.annotations[key] - extract_fn = self._resolved_extract_fn - if extract_fn is not None and record is not None: - record = extract_fn(record, sample) - return key, record - - def __iter__(self) -> Iterator[Sample]: - if self.policy == "right_driven": - yield from self._iter_right_driven() - return - - for item in self.data: - sample = Sample.from_any(item) - key, record = self._lookup(sample) - if self.policy == "inner" and record is None: - continue - yield self._attach(sample, record, key) - - def _iter_right_driven(self) -> Iterator[Sample]: - if self.data_resolver is None: - raise ValueError("policy='right_driven' requires data_resolver") - resolver = self._resolved_data_resolver - for key in self.annotations.keys(): - raw = resolver(key, self.data) - sample = Sample.from_any(raw) - record: Optional[Dict[str, Any]] = self.annotations[key] - extract_fn = self._resolved_extract_fn - if extract_fn is not None and record is not None: - record = extract_fn(record, sample) - if record is None: - # extract_fn signalled "not applicable"; skip this entry - continue - yield self._attach(sample, record, key) - - def __len__(self) -> int: - from collections.abc import Sized - - if self.policy == "left_outer": - # left_outer preserves the data source's length; it must be sized. - if not isinstance(self.data, Sized): - raise TypeError( - f"policy='left_outer' requires a sized data source for len(); " f"got {type(self.data).__name__}" - ) - return len(self.data) - if self.policy == "right_driven": - return len(list(self.annotations.keys())) - - if self._inner_length is None: - count = 0 - for item in self.data: - sample = Sample.from_any(item) - _, record = self._lookup(sample) - if record is not None: - count += 1 - self._inner_length = count - return self._inner_length - - def __getitem__(self, index: int) -> Sample: - if self.policy != "left_outer": - raise TypeError(f"__getitem__ is only supported for policy='left_outer'; got {self.policy!r}") - # Duck-typed on __getitem__ (not isinstance Sequence): workspace sources - # like RFUAVSource / HuggingFaceSource expose __getitem__ without - # subclassing collections.abc.Sequence. - if not hasattr(self.data, "__getitem__"): - raise TypeError("data must support __getitem__ for AnnotationJoinSource.__getitem__") - sample = Sample.from_any(cast(Sequence[Any], self.data)[index]) - key, record = self._lookup(sample) - return self._attach(sample, record, key) diff --git a/sampleflux/sample.py b/sampleflux/sample.py index c6f3638..1d00447 100644 --- a/sampleflux/sample.py +++ b/sampleflux/sample.py @@ -20,6 +20,39 @@ Metadata = Union[Dict[str, Any], List[Dict[str, Any]]] +# The named field VIEWS of a Sample — the closed vocabulary of what a transform can +# process (the taxonomy `sampleflux.kinds` introspects and a visual editor can surface as +# socket types). A view is a real runtime NamedTuple, so an op annotated with one +# receives an object with named fields; the engine binds the view from the flowing +# carrier and merges the result back (untouched fields preserved). +class Pair(NamedTuple): + """The classic metadata-free AI pair ``(input, target)`` — a named 2-tuple view.""" + + input: Any + target: Any = None + + +class InputMeta(NamedTuple): + """The ``(input, metadata)`` view — a transform that reads/writes the input WITH its metadata. + + ``metadata`` follows the same single-vs-batch duality as ``Sample.metadata``: one dict + per item, a list of dicts after collation. + """ + + input: Any + metadata: Metadata = {} + + +class TargetMeta(NamedTuple): + """The ``(target, metadata)`` view — a transform that reads/writes the target WITH its metadata. + + ``metadata`` follows the same single-vs-batch duality as ``Sample.metadata``. + """ + + target: Any + metadata: Metadata = {} + + # Standardized Sample: (input, target, metadata) # This allows SampleFlux to handle complex pipelines while remaining # compatible with simple PyTorch/HF (input, target) pairs. @@ -35,6 +68,14 @@ def to_pair(self) -> Tuple[Any, Any]: """The metadata-free ``(input, target)`` view (the native-engine pair carrier).""" return (self.input, self.target) + def input_meta(self) -> "InputMeta": + """The ``(input, metadata)`` view — the SAME metadata dict (mutation propagates).""" + return InputMeta(self.input, self.meta) + + def target_meta(self) -> "TargetMeta": + """The ``(target, metadata)`` view — the SAME metadata dict (mutation propagates).""" + return TargetMeta(self.target, self.meta) + @property def is_batched(self) -> bool: """True if this Sample holds a BATCH — ``metadata`` is a ``list`` of per-item dicts (one per @@ -102,9 +143,20 @@ def with_type(self, sample_type: "SampleType") -> "Sample": @classmethod def from_any(cls, obj: Any) -> "Sample": - """Coerce raw data from various sources into a Sample.""" + """Coerce raw data from various sources into a Sample. + + The named field VIEWS are recognised BEFORE the generic tuple rule — an + ``InputMeta``/``TargetMeta``/``Pair`` IS a tuple, and positional coercion would + silently misread ``(input, metadata)`` as ``(input, target)``. + """ if isinstance(obj, cls): return obj + if isinstance(obj, InputMeta): + return cls(obj.input, None, obj.metadata) + if isinstance(obj, TargetMeta): + return cls(None, obj.target, obj.metadata) + if isinstance(obj, Pair): + return cls(obj.input, obj.target, {}) if isinstance(obj, tuple): if len(obj) >= 3: return cls(obj[0], obj[1], obj[2] or {}) diff --git a/sampleflux/sources.py b/sampleflux/sources.py index 1558770..57e37e1 100644 --- a/sampleflux/sources.py +++ b/sampleflux/sources.py @@ -20,7 +20,7 @@ # input/target features" — the full-traceability option, kept OPT-IN (``None`` / ``[]`` still = no # extra metadata) so existing configs are unaffected. Resolved against the loaded dataset's # ``column_names`` at construction. Accepted bare (``"*"``) or as the one-element list (``["*"]``); -# FluxStudio's metadata picker offers it as a selectable "*" entry. +# Visual editors offer it as a selectable "*" entry in a metadata picker. METADATA_ALL_FEATURES = "*" @@ -411,7 +411,7 @@ class ConcatSource: The indexable counterpart to :class:`sampleflux.core.JointFlux` (which is iteration-only): ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning sub-source, so a ``ConcatSource`` can itself be wrapped by :class:`DatasetSplit` / - :class:`RangeSource`. (Distinct from :class:`sampleflux.paired.AnnotationJoinSource`, which + :class:`RangeSource`. (Distinct from :class:`waivefront.paired.AnnotationJoinSource`, which *column-joins* annotations onto samples — this one *concatenates* sequences end to end.) Each sub-source must implement ``__len__`` and ``__getitem__``. diff --git a/sampleflux/storage/directory.py b/sampleflux/storage/directory.py index 9d7e00f..7881af2 100644 --- a/sampleflux/storage/directory.py +++ b/sampleflux/storage/directory.py @@ -8,7 +8,7 @@ from sampleflux.storage.base import DataSink, Storage -# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @confluid.configurable(category="sink") class DirectorySink(Storage, DataSink): """ diff --git a/sampleflux/storage/hdf5.py b/sampleflux/storage/hdf5.py index 02984a5..7335839 100644 --- a/sampleflux/storage/hdf5.py +++ b/sampleflux/storage/hdf5.py @@ -77,7 +77,7 @@ def iter_metadata(self) -> "Iterator[tuple[str, dict]]": yield from scan_hdf5_metadata(self.path) -# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @configurable(category="sink") class HDF5Sink(Storage, DataSink): """High-performance HDF5 data sink focused on Sample triplets.""" diff --git a/sampleflux/storage/query.py b/sampleflux/storage/query.py index f21c5af..5dc35db 100644 --- a/sampleflux/storage/query.py +++ b/sampleflux/storage/query.py @@ -4,10 +4,10 @@ - :class:`SupportsMetadataScan` — a source opts in by implementing ``iter_metadata() -> Iterator[(key, metadata_dict)]`` that reads ONLY the metadata - (HDF5 attrs, Zarr ``.zattrs``, a SigMF ``.sigmf-meta`` JSON) — never the data arrays. + (HDF5 attrs, Zarr ``.zattrs``, a sidecar JSON) — never the data arrays. Free-function scanners for the shipped sources live here (``scan_hdf5_metadata`` / - ``scan_zarr_metadata``); ``SigMFSource.iter_metadata`` implements the protocol - directly. + ``scan_zarr_metadata``); any external storage source can implement the protocol + directly (it is structural — no import of this module required). - :class:`MetadataFilterSource` — a view source (``category="source"``) yielding only the samples whose metadata passes a predicate: the YAML-friendly ``where`` expression (the same restricted-eval namespace as ``FormulaOp`` — metadata keys become variables) diff --git a/sampleflux/storage/sigmf.py b/sampleflux/storage/sigmf.py deleted file mode 100644 index a9d799a..0000000 --- a/sampleflux/storage/sigmf.py +++ /dev/null @@ -1,270 +0,0 @@ -"""SigMF storage — one recording (``.sigmf-data`` + ``.sigmf-meta``) per sample. - -`SigMF `_ is the open Signal Metadata Format for raw recordings: a -binary sample file plus a JSON metadata file with ``global`` / ``captures`` / -``annotations`` sections. ``SigMFSink``/``SigMFSource`` are the sampleflux carrier pair -(siblings of the HDF5/Zarr pairs, additive — no migration of existing datasets): - -- the sink writes ``Sample.input`` as the raw ``.sigmf-data`` payload (``core:datatype`` - derived from the numpy dtype) and the sample metadata into the ``.sigmf-meta`` JSON; -- the source reads a directory of recordings back into Sample triplets. - -sampleflux stays domain-neutral: metadata keys are carried VERBATIM — recognised -``core:``-prefixed keys land in their SigMF section, everything else rides the -namespaced ``sampleflux:`` extension in ``global`` (SigMF explicitly supports -namespaced extensions). The waveform VOCABULARY (mapping ``samplerate`` → -``core:sample_rate``, regions → annotations, the torchsig collisions) lives in -``waivefront.vocab`` and plugs in via the ``meta_encoder``/``meta_decoder`` hooks -(dotted callable paths, lazily resolved like ``WrappedOp.f``). - -JSON is hand-rolled deliberately (the format is a stable, simple spec; no dependency to -churn). ``core:sha512`` is optional (``checksum=True``). -""" - -import hashlib -import json -from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union - -import numpy as np -from confluid import configurable -from loggair import get_logger - -from sampleflux.sample import Sample -from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy - -logger = get_logger("sampleflux.storage.sigmf") - -SIGMF_VERSION = "1.0.0" -_EXTENSION_PREFIX = "sampleflux:" - -# numpy dtype <-> SigMF core:datatype (little-endian; the practical interchange subset). -_DTYPE_TO_SIGMF: Dict[str, str] = { - "complex64": "cf32_le", - "complex128": "cf64_le", - "float32": "rf32_le", - "float64": "rf64_le", - "int16": "ri16_le", - "int32": "ri32_le", - "uint8": "ru8", - "int8": "ri8", -} -_SIGMF_TO_DTYPE: Dict[str, str] = {v: k for k, v in _DTYPE_TO_SIGMF.items()} - -MetaEncoder = Callable[[Dict[str, Any]], Tuple[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]]] -MetaDecoder = Callable[[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]], Dict[str, Any]] - - -def _json_safe(value: Any) -> Optional[Any]: - """``value`` if JSON-serializable (numpy scalars unwrapped), else None.""" - if isinstance(value, np.generic): - value = value.item() - try: - json.dumps(value) - except (TypeError, ValueError): - return None - return value - - -def _passthrough_encode(meta: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]]: - """The domain-neutral default encoder: core:* keys verbatim, the rest namespaced into global.""" - global_section: Dict[str, Any] = {} - for key, value in meta.items(): - safe = _json_safe(value) - if safe is None: - logger.debug(f"SigMFSink: metadata key {key!r} is not JSON-serializable — skipped") - continue - if str(key).startswith("core:"): - global_section[str(key)] = safe - else: - global_section[f"{_EXTENSION_PREFIX}{key}"] = safe - return global_section, [], [] - - -def _passthrough_decode( - global_section: Dict[str, Any], captures: List[Dict[str, Any]], annotations: List[Dict[str, Any]] -) -> Dict[str, Any]: - """Inverse of :func:`_passthrough_encode` — unwrap the namespaced keys, keep core:* verbatim.""" - meta: Dict[str, Any] = {} - for key, value in global_section.items(): - if key.startswith(_EXTENSION_PREFIX): - meta[key[len(_EXTENSION_PREFIX) :]] = value - elif key.startswith("core:") and key not in ("core:datatype", "core:version"): - meta[key] = value - if captures: - meta["core:captures"] = captures - if annotations: - meta["core:annotations"] = annotations - return meta - - -def _resolve_hook(hook: Union[str, Callable[..., Any], None], default: Callable[..., Any]) -> Callable[..., Any]: - """A dotted-path / callable / empty hook resolved lazily (the ``WrappedOp.f`` pattern). - - Accepts BOTH ``module:function`` (the discovery-native form) and the friendlier - dotted ``module.function`` (last dot promoted to the separator). - """ - if hook is None or hook == "": - return default - if callable(hook): - return hook - from sampleflux.discovery import resolve_callable - - path = str(hook) - if ":" not in path and "." in path: - module, _, attr = path.rpartition(".") - path = f"{module}:{attr}" - return resolve_callable(path) - - -# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). -@configurable(category="sink") -class SigMFSink(Storage, DataSink): - """Write each Sample as a SigMF recording pair in a directory. - - ``Sample.input`` becomes the raw ``.sigmf-data`` payload; metadata is encoded into - the ``.sigmf-meta`` JSON via ``meta_encoder`` (default: the domain-neutral - passthrough — ``core:*`` keys verbatim, others under ``sampleflux:``; wire - ``waivefront.vocab.to_sigmf`` for the waveform vocabulary). A JSON-serializable - ``Sample.target`` rides ``sampleflux:target`` (SigMF is an input-centric recording - format; array targets are skipped with a debug note). - - Args: - path: Directory the recordings are written into; required at write time, validated lazily. - prefix: Recording filename prefix; files are ``.sigmf-{data,meta}``. - meta_encoder: Dotted path or callable, metadata -> (global, captures, annotations). Blank = passthrough. - checksum: When True, write the ``core:sha512`` of the data payload into the metadata. - """ - - def __init__( - self, - path: Union[str, Path] = "", - prefix: str = "rec_", - meta_encoder: Union[str, MetaEncoder] = "", - checksum: bool = False, - ) -> None: - # Lazy / zero-arg: store config only; the directory is created lazily in open(). - self.path = Path(path) - self.prefix = str(prefix) - self.meta_encoder = meta_encoder - self.checksum = bool(checksum) - self._counter = 0 - self._opened = False - - def open(self) -> "SigMFSink": - if not self._opened: - if str(self.path) in ("", "."): - raise ValueError("SigMFSink: 'path' (the output directory) is required") - self.path.mkdir(parents=True, exist_ok=True) - self._opened = True - return self - - def close(self) -> None: - self._opened = False - - def write(self, sample: Sample) -> None: - self.open() - data = np.ascontiguousarray(to_numpy(sample.input)) - datatype = _DTYPE_TO_SIGMF.get(str(data.dtype)) - if datatype is None: - raise TypeError( - f"SigMFSink: dtype {data.dtype!s} has no SigMF core:datatype mapping " - f"(supported: {sorted(_DTYPE_TO_SIGMF)})" - ) - stem = self.path / f"{self.prefix}{self._counter:05d}" - data.tofile(stem.with_suffix(".sigmf-data")) - - encoder = _resolve_hook(self.meta_encoder, _passthrough_encode) - global_section, captures, annotations = encoder(dict(sample.meta)) - global_section = { - "core:datatype": datatype, - "core:version": SIGMF_VERSION, - **global_section, - } - if self.checksum: - global_section["core:sha512"] = hashlib.sha512(data.tobytes()).hexdigest() - target = _json_safe(sample.target) - if sample.target is not None: - if target is None: - logger.debug("SigMFSink: non-JSON-serializable target skipped (SigMF is input-centric)") - else: - global_section[f"{_EXTENSION_PREFIX}target"] = target - if not captures: - captures = [{"core:sample_start": 0}] - - meta_doc = {"global": global_section, "captures": captures, "annotations": annotations} - stem.with_suffix(".sigmf-meta").write_text(json.dumps(meta_doc, indent=2, sort_keys=True)) - self._counter += 1 - - def flush(self) -> None: - return None - - -@configurable -class SigMFSource(Storage, DataSource): - """Read a directory of SigMF recordings back into Sample triplets. - - The inverse of :class:`SigMFSink`: each ``.sigmf-meta``/``.sigmf-data`` pair yields - one Sample — the payload as ``input`` (dtype from ``core:datatype``), metadata - decoded via ``meta_decoder`` (default: the passthrough inverse; wire - ``waivefront.vocab.from_sigmf`` for the waveform vocabulary), and a stored - ``sampleflux:target`` restored to ``Sample.target``. - - Args: - path: Directory holding the recordings; required at read time, validated lazily. - meta_decoder: Dotted path or callable, (global, captures, annotations) -> metadata. Blank = passthrough. - """ - - def __init__(self, path: Union[str, Path] = "", meta_decoder: Union[str, MetaDecoder] = "") -> None: - # Lazy / zero-arg: store config only; the directory is validated on first access. - self.path = Path(path) - self.meta_decoder = meta_decoder - - def open(self) -> "SigMFSource": - return self - - def close(self) -> None: - return None - - def _meta_files(self) -> List[Path]: - if str(self.path) in ("", ".") or not self.path.is_dir(): - raise ValueError(f"SigMFSource: 'path' {str(self.path)!r} is not a directory of SigMF recordings") - return sorted(self.path.glob("*.sigmf-meta")) - - def _read(self, meta_path: Path) -> Sample: - doc = json.loads(meta_path.read_text()) - global_section: Dict[str, Any] = doc.get("global", {}) - captures: List[Dict[str, Any]] = doc.get("captures", []) - annotations: List[Dict[str, Any]] = doc.get("annotations", []) - - datatype = str(global_section.get("core:datatype", "")) - dtype = _SIGMF_TO_DTYPE.get(datatype) - if dtype is None: - raise ValueError(f"SigMFSource: {meta_path.name}: unsupported core:datatype {datatype!r}") - data = np.fromfile(meta_path.with_suffix(".sigmf-data"), dtype=np.dtype(dtype)) - - decoder = _resolve_hook(self.meta_decoder, _passthrough_decode) - target_key = f"{_EXTENSION_PREFIX}target" - target = global_section.get(target_key) - decodable = {k: v for k, v in global_section.items() if k != target_key} - metadata = decoder(decodable, captures, annotations) - return Sample(input=data, target=target, metadata=metadata) - - def __iter__(self) -> Iterator[Sample]: - for meta_path in self._meta_files(): - yield self._read(meta_path) - - def __len__(self) -> int: - return len(self._meta_files()) - - def __getitem__(self, index: int) -> Sample: - return self._read(self._meta_files()[index]) - - def iter_metadata(self) -> Iterator[Tuple[str, Dict[str, Any]]]: - """(recording stem, decoded metadata) WITHOUT loading any data payload (SupportsMetadataScan).""" - decoder = _resolve_hook(self.meta_decoder, _passthrough_decode) - target_key = f"{_EXTENSION_PREFIX}target" - for meta_path in self._meta_files(): - doc = json.loads(meta_path.read_text()) - global_section = {k: v for k, v in doc.get("global", {}).items() if k != target_key} - yield meta_path.stem, decoder(global_section, doc.get("captures", []), doc.get("annotations", [])) diff --git a/sampleflux/storage/zarr.py b/sampleflux/storage/zarr.py index c286a52..c1dc8d7 100644 --- a/sampleflux/storage/zarr.py +++ b/sampleflux/storage/zarr.py @@ -10,7 +10,7 @@ from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy -# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @confluid.configurable(category="sink") class ZarrGroupSink(Storage, DataSink): """ @@ -117,7 +117,7 @@ def iter_metadata(self) -> "Iterator[tuple[str, dict]]": yield from scan_zarr_metadata(self.path) -# category="sink": surfaced as a FluxStudio sink node (SAMPLEFLUX_OBJECT:sink → DatasetProcessor.sink). +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @confluid.configurable(category="sink") class ZarrBatchSink(Storage, DataSink): """ diff --git a/sampleflux/typespec.py b/sampleflux/typespec.py index 79b9a25..808a6a7 100644 --- a/sampleflux/typespec.py +++ b/sampleflux/typespec.py @@ -23,7 +23,7 @@ value the producer can emit is acceptable to the consumer. Two flavours share the leaf logic: * ``accepts`` — *strict*; used by the runtime check where the producer is a concrete inferred type. -* ``compatible`` — *permissive*; used at edit-time (FluxStudio canvas) and for discovery filtering: +* ``compatible`` — *permissive*; used at edit-time (a visual canvas) and for discovery filtering: ``Any``/unknown on **either** side ⇒ compatible (honours "if not defined, assume Any"), and an unbounded producer axis against a bounded consumer axis is a soft-pass (the runtime check still catches an actual out-of-range value). @@ -35,7 +35,7 @@ ``datasets.Features`` we already depend on (used for the concrete per-sample stored type). Everything is JSON round-trippable (``to_dict`` / :func:`type_from_dict` / :func:`sampletype_from_dict`) -so specs ride the discovery manifest and can be re-implemented by FluxStudio's JS connection-validator. +so specs ride the discovery manifest and can be re-implemented by a GUI connection-validator. """ from __future__ import annotations @@ -67,7 +67,7 @@ TypeSpec = Union["AnyType", "ArrayType", "PythonType", "UnionType", "MappingType", "ListType"] # Closed enumerations for the small, fixed string sets the type system uses — declared as ``Literal`` -# rather than bare ``str`` so authors get a typo-checked value and UIs / the FluxStudio connection- +# rather than bare ``str`` so authors get a typo-checked value and UIs / a GUI connection- # validator enumerate the choices straight from the annotation (``typing.get_args(...)``); the # workspace "prefer closed ``Literal``s over bare strings" mandate. Both are *deliberately closed* — # extend the Literal when adding real support (e.g. a ``"jax"`` framework), don't widen to ``str``. @@ -131,7 +131,7 @@ } #: A concrete dtype name — a closed ``Literal`` (not bare ``str``) so an authored ``ACCEPTS`` / -#: ``PRODUCES`` dtype is typo-checked and UIs / the FluxStudio connection-validator enumerate the +#: ``PRODUCES`` dtype is typo-checked and UIs / a GUI connection-validator enumerate the #: choices via ``typing.get_args(Dtype)``. These ARE the union of the family members above (pinned #: equal in ``tests/test_typespec.py`` so the two can't drift). Authoring uses canonical lowercase #: names; aliases / casing (``"double"``, ``"FLOAT32"``) and genuinely exotic, platform-dependent diff --git a/sampleflux/windows.py b/sampleflux/windows.py deleted file mode 100644 index 68eaf33..0000000 --- a/sampleflux/windows.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Window functions + spectral unit-scaling — the math home for the Fourier ops. - -A raw FFT is *uncalibrated*: to read a spectrum in real units you must (1) taper the -signal with a window to control spectral leakage and (2) divide out the window's gain. -This module is the single, framework-neutral (pure-numpy — scipy is only an optional -sampleflux dependency) source of both: - -* :func:`get_window` builds the taper (``WindowName`` — Hann, Hamming, Blackman-Harris, - flat-top, Kaiser, …). -* :func:`window_sums` / :func:`coherent_gain` / :func:`enbw_bins` give the correction - factors — coherent gain ``S1 = Σw`` (amplitude) and ``S2 = Σw²`` with the equivalent - noise bandwidth (power-spectral density). -* :func:`scale_spectrum` turns a windowed FFT into the chosen ``SpectrumScaling`` units - (amplitude V, power V², density V²/Hz). - -It is a library module (like :mod:`sampleflux.labels` / :mod:`sampleflux.projection`), **not** -``@configurable`` and not entry-pointed. The numpy ops in :mod:`sampleflux.ops.numpy` -(``WindowOp`` / ``SpectrumScalingOp`` / ``FourierOp``) and their torch mirrors in -:mod:`sampleflux.ops.torch` all reuse it — the torch ops take the numpy window coefficients -and the scalar ``S1``/``S2`` corrections, then do the array arithmetic with torch. - -Calibration assumes the **unscaled forward transform** (``numpy.fft.fft`` / -``torch.fft.fft`` with ``norm="backward"`` — the default). The amplitude/power/density -formulas are only meaningful for that normalization, so the ops reject a non-``backward`` -``norm`` combined with a unit ``scaling`` rather than emit silently-wrong units. -""" - -from typing import Dict, Literal, Optional, Tuple, get_args - -import numpy as np - -# --- closed Literals (workspace "prefer closed Literals over bare strings" mandate) --- -# The supported window tapers. ``boxcar`` is the rectangular window (all ones) — i.e. *no* -# taper, the identity — and is the default for FourierOp so its behaviour is unchanged. -WindowName = Literal[ - "boxcar", - "bartlett", - "hann", - "hamming", - "blackman", - "blackmanharris", - "nuttall", - "flattop", - "kaiser", - "tukey", - "gaussian", -] -WINDOW_NAMES: Tuple[WindowName, ...] = get_args(WindowName) - -# Spectral unit-scaling modes. ``none`` = raw FFT (complex, unchanged); ``amplitude`` = -# amplitude spectrum (V, complex); ``power`` = power spectrum (V², real); ``density`` = -# power spectral density (V²/Hz, real). NB this is the GENERAL scaling set — distinct from -# the narrower matplotlib-style ``waivefront.visualizers.SpectrumScaling`` (density/spectrum), -# which is a different module modelling matplotlib's ``scale_by_freq`` toggle. -SpectrumScaling = Literal["none", "amplitude", "power", "density"] -SPECTRUM_SCALINGS: Tuple[SpectrumScaling, ...] = get_args(SpectrumScaling) - -# --- metadata keys: the window correction stashed by WindowOp / FourierOp (when a real -# window is applied) and read back by SpectrumScalingOp so a spectrum computed in one node -# can be scaled to units in another. ``window`` here is the FFT *taper* name — unrelated to -# waivefront's ``window_start_sample`` (a time-slice index). --- -WINDOW_NAME_KEY = "window" -WINDOW_SIZE_KEY = "window_size" # N (number of taps) -WINDOW_SUM_KEY = "window_sum" # S1 = Σw (coherent-gain numerator) -WINDOW_SUMSQ_KEY = "window_sum_sq" # S2 = Σw² -WINDOW_ENBW_KEY = "window_enbw_bins" # equivalent noise bandwidth, N·S2/S1² (bins) -WINDOW_CG_KEY = "window_coherent_gain" # S1/N - -# Generalized-cosine coefficients (scipy / Harris-1978 convention): w[n] = Σ_k a_k·cos(k·φ) -# with φ ∈ [-π, π] over the taps. The alternating shape is carried by cos(k·φ), so the -# coefficients are all positive and sum to 1 at the centre (coherent gain ≈ a_0). -_COSINE_COEFFS: Dict[str, Tuple[float, ...]] = { - "hann": (0.5, 0.5), - "hamming": (0.54, 0.46), - "blackman": (0.42, 0.5, 0.08), - "blackmanharris": (0.35875, 0.48829, 0.14128, 0.01168), - "nuttall": (0.3635819, 0.4891775, 0.1365995, 0.0106411), - "flattop": (0.21557895, 0.41663158, 0.277263158, 0.083578947, 0.006947368), -} - - -def _general_cosine(n: int, coeffs: Tuple[float, ...], periodic: bool) -> np.ndarray: - """Generalized-cosine window of length ``n`` (the Hann/Hamming/Blackman/… family). - - ``periodic=True`` (the DFT-even form correct for FFT spectral analysis) builds the - symmetric window of length ``n+1`` and drops the last sample; ``periodic=False`` is the - plain symmetric window (zero — or near-zero — at both endpoints). - """ - m = n + 1 if periodic else n - fac = np.linspace(-np.pi, np.pi, m) - w = np.zeros(m, dtype=np.float64) - for k, a in enumerate(coeffs): - w = w + a * np.cos(k * fac) - return w[:-1] if periodic else w - - -def _tukey(n: int, alpha: float, periodic: bool) -> np.ndarray: - """Tukey (tapered-cosine) window — ``alpha`` is the cosine-tapered fraction in [0, 1].""" - if alpha <= 0: - return np.ones(n, dtype=np.float64) - if alpha >= 1: - return _general_cosine(n, (0.5, 0.5), periodic) # full cosine taper == Hann - m = n + 1 if periodic else n - idx = np.arange(0, m) - width = int(np.floor(alpha * (m - 1) / 2.0)) - w = np.ones(m, dtype=np.float64) - n1 = idx[: width + 1] - n3 = idx[m - width - 1 :] - w[: width + 1] = 0.5 * (1 + np.cos(np.pi * (-1 + 2.0 * n1 / alpha / (m - 1)))) - w[m - width - 1 :] = 0.5 * (1 + np.cos(np.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / (m - 1)))) - return w[:-1] if periodic else w - - -def _gaussian(n: int, std: float, periodic: bool) -> np.ndarray: - """Gaussian window — ``std`` is the standard deviation in samples (must be > 0).""" - if std <= 0: - raise ValueError(f"gaussian window std must be > 0; got {std!r}") - m = n + 1 if periodic else n - k = np.arange(0, m) - (m - 1) / 2.0 - w = np.exp(-0.5 * (k / std) ** 2) - return np.asarray(w[:-1] if periodic else w, dtype=np.float64) - - -def get_window( - window: WindowName, n: int, *, window_param: Optional[float] = None, periodic: bool = True -) -> np.ndarray: - """Build a length-``n`` window taper as a ``float64`` ndarray. - - Args: - window: Which taper — one of ``WindowName`` (``boxcar`` is the rectangular identity). - n: Number of taps (must be positive); normally the signal length being transformed. - window_param: Shape parameter for the parametrized windows — Kaiser ``β`` (default 8.6), - Tukey ``α`` taper fraction in [0, 1] (default 0.5), or Gaussian ``σ`` std in samples - (required, no default). Ignored by the fixed windows. - periodic: ``True`` (default) = DFT-even window (the correct form for FFT spectral - analysis); ``False`` = symmetric window (zero at both endpoints). - - Returns: - The window coefficients, ``float64``, shape ``(n,)``. - - Raises: - ValueError: unknown ``window``, non-positive ``n``, or a Gaussian without ``window_param``. - """ - if window not in WINDOW_NAMES: - raise ValueError(f"unknown window {window!r}; valid: {WINDOW_NAMES}") - if n <= 0: - raise ValueError(f"window length n must be positive; got {n!r}") - if n == 1: - return np.ones(1, dtype=np.float64) - if window == "boxcar": - return np.ones(n, dtype=np.float64) - if window in _COSINE_COEFFS: - return _general_cosine(n, _COSINE_COEFFS[window], periodic) - if window == "bartlett": - m = n + 1 if periodic else n - w = np.bartlett(m) - return (w[:-1] if periodic else w).astype(np.float64) - if window == "kaiser": - beta = 8.6 if window_param is None else float(window_param) - m = n + 1 if periodic else n - w = np.kaiser(m, beta) - return (w[:-1] if periodic else w).astype(np.float64) - if window == "tukey": - alpha = 0.5 if window_param is None else float(window_param) - return _tukey(n, alpha, periodic) - # window == "gaussian" - if window_param is None: - raise ValueError("gaussian window requires window_param (std in samples)") - return _gaussian(n, float(window_param), periodic) - - -def window_sums(window: np.ndarray) -> Tuple[float, float]: - """Return ``(S1, S2)`` = ``(Σw, Σw²)`` — the two sums the unit corrections need.""" - w = np.asarray(window, dtype=np.float64) - return float(w.sum()), float(np.square(w).sum()) - - -def coherent_gain(window: np.ndarray) -> float: - """Coherent gain ``S1/N`` — the amplitude attenuation the window applies to a tone.""" - w = np.asarray(window, dtype=np.float64) - return float(w.sum() / w.size) - - -def enbw_bins(window: np.ndarray) -> float: - """Equivalent noise bandwidth ``N·S2/S1²`` in **bins** (e.g. ≈1.5 for Hann).""" - s1, s2 = window_sums(window) - return float(np.asarray(window).size * s2 / (s1 * s1)) - - -def window_metadata(window_name: str, window: np.ndarray) -> Dict[str, object]: - """Build the window-correction metadata dict (the keys ``SpectrumScalingOp`` reads).""" - s1, s2 = window_sums(window) - n = int(np.asarray(window).size) - return { - WINDOW_NAME_KEY: window_name, - WINDOW_SIZE_KEY: n, - WINDOW_SUM_KEY: s1, - WINDOW_SUMSQ_KEY: s2, - WINDOW_ENBW_KEY: float(n * s2 / (s1 * s1)), - WINDOW_CG_KEY: float(s1 / n), - } - - -def fold_one_sided(spectrum: np.ndarray, axis: int) -> np.ndarray: - """Fold a two-sided spectrum (natural FFT order, DC at index 0) to one-sided. - - Keeps bins ``0 … N//2`` and doubles the interior bins (everything except DC and, for - even ``N``, the Nyquist bin) so a real signal's one-sided amplitude/power reads its true - value. Meaningful only for spectra of **real** inputs in natural (un-``fftshift``ed) order. - """ - moved = np.swapaxes(np.asarray(spectrum), axis, -1) - n = moved.shape[-1] - out = moved[..., : n // 2 + 1].copy() - if n % 2 == 0: - out[..., 1:-1] = out[..., 1:-1] * 2 # exclude DC (0) and Nyquist (-1) - else: - out[..., 1:] = out[..., 1:] * 2 # no Nyquist bin for odd N - return np.swapaxes(out, axis, -1) - - -def scale_spectrum( - spectrum: np.ndarray, - scaling: SpectrumScaling, - *, - s1: float, - s2: float, - sample_rate: Optional[float] = None, - one_sided: bool = False, - axis: int = -1, -) -> np.ndarray: - """Scale a windowed FFT spectrum to the chosen units (assumes ``norm="backward"``). - - Args: - spectrum: The complex FFT output (windowed, unscaled forward transform). - scaling: ``none`` (complex, unchanged) · ``amplitude`` (V, complex, ``X/S1``) · - ``power`` (V², real, ``|X|²/S1²``) · ``density`` (V²/Hz, real, ``|X|²/(Fs·S2)``). - s1: Window coherent-gain sum ``Σw`` (use ``N`` for an unwindowed / boxcar spectrum). - s2: Window squared sum ``Σw²`` (use ``N`` for boxcar). - sample_rate: ``Fs`` in Hz for ``density`` (V²/Hz). ``None`` / ≤0 → ``1.0`` (density - per normalized frequency, V² per cycle/sample). Ignored by other modes. - one_sided: Fold to a one-sided spectrum (real-input convention) after scaling. - axis: Transform axis (for ``one_sided`` folding and the bin count). - - Returns: - The scaled spectrum — complex for ``none``/``amplitude``, real for ``power``/``density``. - """ - x = np.asarray(spectrum) - if scaling == "none": - out = x - elif scaling == "amplitude": - out = x / s1 - elif scaling == "power": - out = np.square(np.abs(x)) / (s1 * s1) - elif scaling == "density": - fs = float(sample_rate) if (sample_rate is not None and sample_rate > 0) else 1.0 - out = np.square(np.abs(x)) / (fs * s2) - else: - raise ValueError(f"unknown scaling {scaling!r}; valid: {SPECTRUM_SCALINGS}") - if one_sided: - out = fold_one_sided(out, axis) - return out diff --git a/tests/test_categories.py b/tests/test_categories.py index b588284..7cfb411 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,7 +10,6 @@ from confluid.registry import get_registry from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.ops.capture import CaptureOutputOp from sampleflux.ops.configure import ConfigureOp from sampleflux.ops.copy import CopyInputOp from sampleflux.ops.debug import PrintSampleOp @@ -18,17 +17,7 @@ from sampleflux.ops.formula import FormulaOp from sampleflux.ops.image import ConvertToImageOp, NormalizeToUint8Op from sampleflux.ops.metadata import DropMetadataOp -from sampleflux.ops.numpy import ( - FftShiftOp, - FourierOp, - IfftShiftOp, - InverseFourierOp, - RescaleOp, - SpectrumScalingOp, - StandardizeOp, - ThresholdOp, - WindowOp, -) +from sampleflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp from sampleflux.ops.parallel import Parallel from sampleflux.ops.sink import SampleSinkOp from sampleflux.ops.stash import StashTargetOp, UnstashTargetOp @@ -39,14 +28,7 @@ MasksToDetectionBoxesOp, MetadataToTargetOp, ) -from sampleflux.ops.tee import Tee -from sampleflux.ops.torch import FftShiftOp as TorchFftShiftOp -from sampleflux.ops.torch import FourierOp as TorchFourierOp -from sampleflux.ops.torch import IfftShiftOp as TorchIfftShiftOp -from sampleflux.ops.torch import InverseFourierOp as TorchInverseFourierOp -from sampleflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp from sampleflux.ops.torch import ToTensorOp -from sampleflux.ops.torch import WindowOp as TorchWindowOp from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource from sampleflux.storage.directory import DirectorySink @@ -94,19 +76,6 @@ def test_op_classes_tagged() -> None: assert RescaleOp.__confluid_category__ == "op" assert StandardizeOp.__confluid_category__ == "op" assert ThresholdOp.__confluid_category__ == "op" - assert FourierOp.__confluid_category__ == "op" - assert TorchFourierOp.__confluid_category__ == "op" - assert InverseFourierOp.__confluid_category__ == "op" - assert TorchInverseFourierOp.__confluid_category__ == "op" - assert FftShiftOp.__confluid_category__ == "op" - assert TorchFftShiftOp.__confluid_category__ == "op" - assert IfftShiftOp.__confluid_category__ == "op" - assert TorchIfftShiftOp.__confluid_category__ == "op" - assert WindowOp.__confluid_category__ == "op" - assert TorchWindowOp.__confluid_category__ == "op" - assert SpectrumScalingOp.__confluid_category__ == "op" - assert TorchSpectrumScalingOp.__confluid_category__ == "op" - assert Tee.__confluid_category__ == "op" assert Enable.__confluid_category__ == "op" assert TransformChain.__confluid_category__ == "op" assert SampleSinkOp.__confluid_category__ == "op" @@ -117,7 +86,6 @@ def test_op_classes_tagged() -> None: assert MasksToDetectionBoxesOp.__confluid_category__ == "op" assert ConfigureOp.__confluid_category__ == "op" assert FormulaOp.__confluid_category__ == "op" - assert CaptureOutputOp.__confluid_category__ == "op" def test_storage_sink_classes_tagged() -> None: @@ -141,19 +109,7 @@ def test_op_group_tags() -> None: assert RescaleOp.__confluid_group__ == "numpy" assert StandardizeOp.__confluid_group__ == "numpy" assert ThresholdOp.__confluid_group__ == "numpy" - assert FourierOp.__confluid_group__ == "numpy" - assert InverseFourierOp.__confluid_group__ == "numpy" - assert FftShiftOp.__confluid_group__ == "numpy" - assert IfftShiftOp.__confluid_group__ == "numpy" assert ToTensorOp.__confluid_group__ == "torch" - assert TorchFourierOp.__confluid_group__ == "torch" - assert TorchInverseFourierOp.__confluid_group__ == "torch" - assert TorchFftShiftOp.__confluid_group__ == "torch" - assert TorchIfftShiftOp.__confluid_group__ == "torch" - assert WindowOp.__confluid_group__ == "numpy" - assert SpectrumScalingOp.__confluid_group__ == "numpy" - assert TorchWindowOp.__confluid_group__ == "torch" - assert TorchSpectrumScalingOp.__confluid_group__ == "torch" assert CopyInputOp.__confluid_group__ == "structure" assert DropMetadataOp.__confluid_group__ == "structure" assert PrintSampleOp.__confluid_group__ == "debug" @@ -164,13 +120,11 @@ def test_op_group_tags() -> None: assert DecodeTargetOp.__confluid_group__ == "structure" assert CocoToTorchVisionDetectionOp.__confluid_group__ == "structure" assert MasksToDetectionBoxesOp.__confluid_group__ == "structure" - assert Tee.__confluid_group__ == "compose" assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" assert TransformChain.__confluid_group__ == "compose" assert ConfigureOp.__confluid_group__ == "compose" assert FormulaOp.__confluid_group__ == "compose" - assert CaptureOutputOp.__confluid_group__ == "compose" assert ConvertToImageOp.__confluid_group__ == "image" assert NormalizeToUint8Op.__confluid_group__ == "image" assert SampleSinkOp.__confluid_group__ == "sink" @@ -195,11 +149,6 @@ def test_categories_enumerable_via_registry() -> None: "RescaleOp", "StandardizeOp", "ThresholdOp", - "FourierOp", - "InverseFourierOp", - "FftShiftOp", - "IfftShiftOp", - "Tee", "Enable", "SampleSinkOp", "MetadataToTargetOp", @@ -222,18 +171,12 @@ def test_groups_enumerable_via_registry() -> None: "RescaleOp", "StandardizeOp", "ThresholdOp", - "FourierOp", - "InverseFourierOp", - "FftShiftOp", - "IfftShiftOp", } <= registry.list_classes(group="numpy") # The FFT ops exist in BOTH framework groups (a numpy + a torch variant under the one name, # exactly like RescaleOp/StandardizeOp), so they surface under the torch group too. - assert {"ToTensorOp", "FourierOp", "InverseFourierOp", "FftShiftOp", "IfftShiftOp"} <= registry.list_classes( - group="torch" - ) + assert {"ToTensorOp"} <= registry.list_classes(group="torch") assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") - assert {"Tee", "Parallel", "Enable", "TransformChain"} <= registry.list_classes(group="compose") + assert {"Parallel", "Enable", "TransformChain"} <= registry.list_classes(group="compose") assert {"SampleSinkOp"} <= registry.list_classes(group="sink") assert { "MetadataToTargetOp", @@ -243,4 +186,4 @@ def test_groups_enumerable_via_registry() -> None: "MasksToDetectionBoxesOp", } <= registry.list_classes(group="structure") # group × category intersect, like task × role. - assert "Tee" in registry.list_classes(category="op", group="compose") + assert "TransformChain" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_fourier_ops.py b/tests/test_fourier_ops.py deleted file mode 100644 index 14faafa..0000000 --- a/tests/test_fourier_ops.py +++ /dev/null @@ -1,655 +0,0 @@ -"""Tests for the 1-D Fourier-transform ops: ``sampleflux.ops.numpy.FourierOp`` and -``sampleflux.ops.torch.FourierOp``. - -Both compute the 1-D DFT (``numpy.fft.fft`` / ``torch.fft.fft``) of ``sample.input`` and -ALWAYS yield a complex result — for real and complex inputs alike. The tests pin: the -real/complex/integer dtype-promotion rules, the round-trip against the inverse transform, -the ``n`` / ``axis``-``dim`` / ``norm`` parameters, the framework type guards, the -``ACCEPTS``/``PRODUCES`` contract conformance, and the closed-``Literal`` ``norm`` validation. -""" - -import numpy as np -import pytest -import torch -from pydantic import ValidationError - -from sampleflux.ops import FftShiftOp as FlatFftShiftOp -from sampleflux.ops import FourierOp as FlatFourierOp -from sampleflux.ops import IfftShiftOp as FlatIfftShiftOp -from sampleflux.ops import InverseFourierOp as FlatInverseFourierOp -from sampleflux.ops.numpy import FftShiftOp as NpFftShiftOp -from sampleflux.ops.numpy import FourierNorm -from sampleflux.ops.numpy import FourierOp as NpFourierOp -from sampleflux.ops.numpy import IfftShiftOp as NpIfftShiftOp -from sampleflux.ops.numpy import InverseFourierOp as NpInverseFourierOp -from sampleflux.ops.numpy import SpectrumScalingOp as NpSpectrumScalingOp -from sampleflux.ops.numpy import WindowOp as NpWindowOp -from sampleflux.ops.torch import FftShiftOp as TorchFftShiftOp -from sampleflux.ops.torch import FourierOp as TorchFourierOp -from sampleflux.ops.torch import IfftShiftOp as TorchIfftShiftOp -from sampleflux.ops.torch import InverseFourierOp as TorchInverseFourierOp -from sampleflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp -from sampleflux.ops.torch import WindowOp as TorchWindowOp -from sampleflux.sample import Sample -from sampleflux.typespec import infer_sample_type -from sampleflux.windows import WINDOW_SUM_KEY - - -def test_flat_imports_are_torch_variants() -> None: - """``from sampleflux.ops import …`` resolves the FFT ops to their torch variants — the package's - documented convention that flat data-op imports default to torch (mirrors RescaleOp etc.).""" - assert FlatFourierOp is TorchFourierOp - assert FlatInverseFourierOp is TorchInverseFourierOp - assert FlatFftShiftOp is TorchFftShiftOp - assert FlatIfftShiftOp is TorchIfftShiftOp - - -# --------------------------------------------------------------------------- -# numpy FourierOp -# --------------------------------------------------------------------------- - - -class TestNumpyFourierOp: - def test_real_float32_matches_numpy_and_is_complex64(self) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) - out = NpFourierOp()(Sample(input=x)) - assert isinstance(out.input, np.ndarray) - assert out.input.dtype == np.complex64 - assert np.allclose(out.input, np.fft.fft(x)) - - def test_real_float64_promotes_to_complex128(self) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float64) - out = NpFourierOp()(Sample(input=x)) - assert out.input.dtype == np.complex128 - assert np.allclose(out.input, np.fft.fft(x)) - - def test_integer_input_promotes_to_complex128(self) -> None: - x = np.arange(8, dtype=np.int64) - out = NpFourierOp()(Sample(input=x)) - assert out.input.dtype == np.complex128 - assert np.allclose(out.input, np.fft.fft(x)) - - def test_complex64_input_stays_complex64(self) -> None: - x = np.array([1 + 2j, 3 - 1j, 0 + 0j, -2 + 1j], dtype=np.complex64) - out = NpFourierOp()(Sample(input=x)) - assert out.input.dtype == np.complex64 - assert np.allclose(out.input, np.fft.fft(x)) - - def test_complex128_input_stays_complex128(self) -> None: - x = np.array([1 + 2j, 3 - 1j, 0 + 0j, -2 + 1j], dtype=np.complex128) - out = NpFourierOp()(Sample(input=x)) - assert out.input.dtype == np.complex128 - - def test_constant_signal_has_only_dc_component(self) -> None: - # FFT of a length-4 constant [1,1,1,1] is [4, 0, 0, 0] (all energy in the DC bin). - out = NpFourierOp()(Sample(input=np.ones(4, dtype=np.float64))) - assert np.allclose(out.input, np.array([4, 0, 0, 0])) - - def test_roundtrip_via_ifft_recovers_input(self) -> None: - x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) - out = NpFourierOp()(Sample(input=x)) - recovered = np.fft.ifft(out.input) - assert np.allclose(recovered.real, x, atol=1e-9) - - def test_n_zero_pads(self) -> None: - x = np.arange(4, dtype=np.float64) - out = NpFourierOp(n=8)(Sample(input=x)) - assert out.input.shape == (8,) - assert np.allclose(out.input, np.fft.fft(x, n=8)) - - def test_n_truncates(self) -> None: - x = np.arange(8, dtype=np.float64) - out = NpFourierOp(n=4)(Sample(input=x)) - assert out.input.shape == (4,) - assert np.allclose(out.input, np.fft.fft(x, n=4)) - - def test_axis_transforms_per_row_of_batch(self) -> None: - x = np.random.RandomState(0).randn(3, 8) - out = NpFourierOp(axis=-1)(Sample(input=x)) - assert out.input.shape == (3, 8) - # Each row transformed independently == the per-row 1-D FFT. - for i in range(3): - assert np.allclose(out.input[i], np.fft.fft(x[i])) - - def test_axis_zero(self) -> None: - x = np.random.RandomState(1).randn(8, 3) - out = NpFourierOp(axis=0)(Sample(input=x)) - assert np.allclose(out.input, np.fft.fft(x, axis=0)) - - @pytest.mark.parametrize("norm", ["backward", "ortho", "forward"]) - def test_norm_modes_match_numpy(self, norm: FourierNorm) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float64) - out = NpFourierOp(norm=norm)(Sample(input=x)) - assert np.allclose(out.input, np.fft.fft(x, norm=norm)) - - def test_shift_flag_matches_manual_fftshift(self) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) # odd length — fftshift is non-trivial - shifted = NpFourierOp(shift=True)(Sample(input=x)).input - assert np.allclose(shifted, np.fft.fftshift(NpFourierOp()(Sample(input=x)).input)) - - def test_shift_defaults_off(self) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0]) - plain = NpFourierOp()(Sample(input=x)).input - assert np.allclose(NpFourierOp(shift=False)(Sample(input=x)).input, plain) - assert not np.allclose(NpFourierOp(shift=True)(Sample(input=x)).input, plain) - - def test_preserves_target_and_metadata(self) -> None: - out = NpFourierOp()(Sample(input=np.ones(4), target=5, metadata={"k": "v"})) - assert out.target == 5 - assert out.meta == {"k": "v"} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="FourierOp expects an np.ndarray"): - NpFourierOp()(Sample(input=torch.zeros(4))) - - def test_zero_arg_construction(self) -> None: - op = NpFourierOp() - assert op.n is None and op.axis == -1 and op.norm == "backward" - - def test_invalid_norm_rejected_at_construction(self) -> None: - # ``norm`` is a closed ``Literal`` — confluid's pydantic schema rejects an out-of-set value. - with pytest.raises((ValueError, ValidationError)): - NpFourierOp(norm="bogus") # type: ignore[arg-type] - - def test_produces_contract_conforms_to_real_output(self) -> None: - out = NpFourierOp()(Sample(input=np.ones(4, dtype=np.float32))) - assert NpFourierOp.PRODUCES.accepts(infer_sample_type(out)) - - def test_produces_contract_conforms_to_complex_output(self) -> None: - x = np.array([1 + 2j, 3 - 1j], dtype=np.complex128) - out = NpFourierOp()(Sample(input=x)) - assert NpFourierOp.PRODUCES.accepts(infer_sample_type(out)) - - -# --------------------------------------------------------------------------- -# torch FourierOp -# --------------------------------------------------------------------------- - - -class TestTorchFourierOp: - def test_real_float32_matches_torch_and_is_complex64(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0]) - out = TorchFourierOp()(Sample(input=t)) - assert isinstance(out.input, torch.Tensor) - assert out.input.dtype == torch.complex64 - assert torch.allclose(out.input, torch.fft.fft(t)) - - def test_real_float64_promotes_to_complex128(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float64) - out = TorchFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex128 - assert torch.allclose(out.input, torch.fft.fft(t)) - - def test_integer_input_handled_natively(self) -> None: - # torch.fft.fft auto-promotes integer tensors to complex64 — no manual cast needed. - t = torch.arange(8) - out = TorchFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - assert torch.allclose(out.input, torch.fft.fft(t)) - - def test_bool_input_handled_natively(self) -> None: - t = torch.tensor([True, False, True, True]) - out = TorchFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - - def test_complex64_input_stays_complex64(self) -> None: - t = torch.tensor([1 + 2j, 3 - 1j, 0 + 0j, -2 + 1j], dtype=torch.complex64) - out = TorchFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - assert torch.allclose(out.input, torch.fft.fft(t)) - - def test_float16_promoted_to_float32_without_mutating_input(self) -> None: - # torch.fft.fft rejects half precision; the op promotes to float32 first. The promotion - # is local — the caller's tensor is untouched. - t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float16) - out = TorchFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - assert t.dtype == torch.float16 - assert torch.allclose(out.input, torch.fft.fft(t.float())) - - def test_bfloat16_promoted_to_float32(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.bfloat16) - out = TorchFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - - def test_constant_signal_has_only_dc_component(self) -> None: - out = TorchFourierOp()(Sample(input=torch.ones(4))) - assert torch.allclose(out.input, torch.tensor([4, 0, 0, 0], dtype=torch.complex64)) - - def test_roundtrip_via_ifft_recovers_input(self) -> None: - t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) - out = TorchFourierOp()(Sample(input=t)) - recovered = torch.fft.ifft(out.input) - assert torch.allclose(recovered.real, t, atol=1e-9) - - def test_n_zero_pads(self) -> None: - t = torch.arange(4, dtype=torch.float64) - out = TorchFourierOp(n=8)(Sample(input=t)) - assert out.input.shape == (8,) - assert torch.allclose(out.input, torch.fft.fft(t, n=8)) - - def test_n_truncates(self) -> None: - t = torch.arange(8, dtype=torch.float64) - out = TorchFourierOp(n=4)(Sample(input=t)) - assert out.input.shape == (4,) - - def test_dim_transforms_per_row_of_batch(self) -> None: - t = torch.randn(3, 8) - out = TorchFourierOp(dim=-1)(Sample(input=t)) - assert out.input.shape == (3, 8) - for i in range(3): - assert torch.allclose(out.input[i], torch.fft.fft(t[i])) - - def test_dim_zero(self) -> None: - t = torch.randn(8, 3) - out = TorchFourierOp(dim=0)(Sample(input=t)) - assert torch.allclose(out.input, torch.fft.fft(t, dim=0)) - - @pytest.mark.parametrize("norm", ["backward", "ortho", "forward"]) - def test_norm_modes_match_torch(self, norm: FourierNorm) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float64) - out = TorchFourierOp(norm=norm)(Sample(input=t)) - assert torch.allclose(out.input, torch.fft.fft(t, norm=norm)) - - def test_shift_flag_matches_manual_fftshift(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0]) # odd length — fftshift is non-trivial - shifted = TorchFourierOp(shift=True)(Sample(input=t)).input - assert torch.allclose(shifted, torch.fft.fftshift(TorchFourierOp()(Sample(input=t)).input)) - - def test_shift_defaults_off(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0]) - plain = TorchFourierOp()(Sample(input=t)).input - assert torch.allclose(TorchFourierOp(shift=False)(Sample(input=t)).input, plain) - assert not torch.allclose(TorchFourierOp(shift=True)(Sample(input=t)).input, plain) - - def test_preserves_target_and_metadata(self) -> None: - out = TorchFourierOp()(Sample(input=torch.ones(4), target=7, metadata={"k": "v"})) - assert out.target == 7 - assert out.meta == {"k": "v"} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="FourierOp expects a torch.Tensor"): - TorchFourierOp()(Sample(input=np.zeros(4))) - - def test_zero_arg_construction(self) -> None: - op = TorchFourierOp() - assert op.n is None and op.dim == -1 and op.norm == "backward" - - def test_invalid_norm_rejected_at_construction(self) -> None: - with pytest.raises((ValueError, ValidationError)): - TorchFourierOp(norm="bogus") # type: ignore[arg-type] - - def test_produces_contract_conforms_to_real_output(self) -> None: - out = TorchFourierOp()(Sample(input=torch.ones(4))) - assert TorchFourierOp.PRODUCES.accepts(infer_sample_type(out)) - - def test_produces_contract_conforms_to_complex_output(self) -> None: - t = torch.tensor([1 + 2j, 3 - 1j], dtype=torch.complex128) - out = TorchFourierOp()(Sample(input=t)) - assert TorchFourierOp.PRODUCES.accepts(infer_sample_type(out)) - - -# --------------------------------------------------------------------------- -# numpy InverseFourierOp -# --------------------------------------------------------------------------- - - -class TestNumpyInverseFourierOp: - def test_matches_numpy_ifft_and_is_complex(self) -> None: - x = np.array([10.0, -2.0, 0.0, 4.0]) - out = NpInverseFourierOp()(Sample(input=x)) - assert out.input.dtype == np.complex128 - assert np.allclose(out.input, np.fft.ifft(x)) - - def test_inverts_forward_transform(self) -> None: - x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) # odd length - spectrum = NpFourierOp()(Sample(input=x)) - recovered = NpInverseFourierOp()(spectrum) - assert np.allclose(recovered.input.real, x, atol=1e-9) - - def test_shift_inverts_forward_shift(self) -> None: - # InverseFourierOp(shift=True) exactly undoes FourierOp(shift=True), odd length included. - x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) - centered = NpFourierOp(shift=True)(Sample(input=x)) - recovered = NpInverseFourierOp(shift=True)(centered) - assert np.allclose(recovered.input.real, x, atol=1e-9) - - @pytest.mark.parametrize("norm", ["backward", "ortho", "forward"]) - def test_roundtrip_under_each_norm(self, norm: FourierNorm) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float64) - spectrum = NpFourierOp(norm=norm)(Sample(input=x)) - recovered = NpInverseFourierOp(norm=norm)(spectrum) - assert np.allclose(recovered.input.real, x, atol=1e-9) - - def test_n_truncates(self) -> None: - x = np.arange(8, dtype=np.float64) - out = NpInverseFourierOp(n=4)(Sample(input=x)) - assert out.input.shape == (4,) - assert np.allclose(out.input, np.fft.ifft(x, n=4)) - - def test_preserves_target_and_metadata(self) -> None: - out = NpInverseFourierOp()(Sample(input=np.ones(4), target=5, metadata={"k": "v"})) - assert out.target == 5 - assert out.meta == {"k": "v"} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="InverseFourierOp expects an np.ndarray"): - NpInverseFourierOp()(Sample(input=torch.zeros(4))) - - def test_zero_arg_construction(self) -> None: - op = NpInverseFourierOp() - assert op.n is None and op.axis == -1 and op.norm == "backward" and op.shift is False - - def test_invalid_norm_rejected_at_construction(self) -> None: - with pytest.raises((ValueError, ValidationError)): - NpInverseFourierOp(norm="bogus") # type: ignore[arg-type] - - def test_produces_contract_conforms(self) -> None: - out = NpInverseFourierOp()(Sample(input=np.ones(4))) - assert NpInverseFourierOp.PRODUCES.accepts(infer_sample_type(out)) - - -# --------------------------------------------------------------------------- -# torch InverseFourierOp -# --------------------------------------------------------------------------- - - -class TestTorchInverseFourierOp: - def test_matches_torch_ifft_and_is_complex(self) -> None: - t = torch.tensor([10.0, -2.0, 0.0, 4.0]) - out = TorchInverseFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - assert torch.allclose(out.input, torch.fft.ifft(t)) - - def test_inverts_forward_transform(self) -> None: - t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) - spectrum = TorchFourierOp()(Sample(input=t)) - recovered = TorchInverseFourierOp()(spectrum) - assert torch.allclose(recovered.input.real, t, atol=1e-9) - - def test_shift_inverts_forward_shift(self) -> None: - t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) - centered = TorchFourierOp(shift=True)(Sample(input=t)) - recovered = TorchInverseFourierOp(shift=True)(centered) - assert torch.allclose(recovered.input.real, t, atol=1e-9) - - def test_integer_input_handled_natively(self) -> None: - out = TorchInverseFourierOp()(Sample(input=torch.arange(8))) - assert out.input.dtype == torch.complex64 - - def test_float16_promoted_without_mutating_input(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float16) - out = TorchInverseFourierOp()(Sample(input=t)) - assert out.input.dtype == torch.complex64 - assert t.dtype == torch.float16 - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="InverseFourierOp expects a torch.Tensor"): - TorchInverseFourierOp()(Sample(input=np.zeros(4))) - - def test_zero_arg_construction(self) -> None: - op = TorchInverseFourierOp() - assert op.n is None and op.dim == -1 and op.norm == "backward" and op.shift is False - - def test_produces_contract_conforms(self) -> None: - out = TorchInverseFourierOp()(Sample(input=torch.ones(4))) - assert TorchInverseFourierOp.PRODUCES.accepts(infer_sample_type(out)) - - -# --------------------------------------------------------------------------- -# fftshift / ifftshift ops (numpy + torch) — pure, dtype-preserving rearrangements -# --------------------------------------------------------------------------- - - -class TestNumpyShiftOps: - def test_fftshift_matches_numpy(self) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) - assert np.allclose(NpFftShiftOp()(Sample(input=x)).input, np.fft.fftshift(x)) - - def test_ifftshift_matches_numpy(self) -> None: - x = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) - assert np.allclose(NpIfftShiftOp()(Sample(input=x)).input, np.fft.ifftshift(x)) - - def test_ifftshift_inverts_fftshift_odd_length(self) -> None: - x = np.arange(5) - shifted = NpFftShiftOp()(Sample(input=x)) - restored = NpIfftShiftOp()(shifted) - assert np.array_equal(restored.input, x) - - def test_preserves_dtype_integer_and_complex(self) -> None: - assert NpFftShiftOp()(Sample(input=np.arange(5))).input.dtype == np.int64 - cx = np.array([1 + 1j, 2 - 2j, 3j], dtype=np.complex64) - assert NpFftShiftOp()(Sample(input=cx)).input.dtype == np.complex64 - - def test_axis_shifts_per_row(self) -> None: - x = np.arange(15).reshape(3, 5) - assert np.allclose(NpFftShiftOp(axis=-1)(Sample(input=x)).input, np.fft.fftshift(x, axes=-1)) - - def test_axis_none_shifts_all_axes(self) -> None: - x = np.arange(15).reshape(3, 5) - assert np.allclose(NpFftShiftOp(axis=None)(Sample(input=x)).input, np.fft.fftshift(x)) - - def test_preserves_target_and_metadata(self) -> None: - out = NpFftShiftOp()(Sample(input=np.arange(4), target=9, metadata={"k": "v"})) - assert out.target == 9 - assert out.meta == {"k": "v"} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="FftShiftOp expects an np.ndarray"): - NpFftShiftOp()(Sample(input=torch.zeros(4))) - with pytest.raises(TypeError, match="IfftShiftOp expects an np.ndarray"): - NpIfftShiftOp()(Sample(input=torch.zeros(4))) - - def test_zero_arg_construction(self) -> None: - assert NpFftShiftOp().axis == -1 - assert NpIfftShiftOp().axis == -1 - - def test_produces_contract_conforms(self) -> None: - out = NpFftShiftOp()(Sample(input=np.arange(5))) - assert NpFftShiftOp.PRODUCES.accepts(infer_sample_type(out)) - - -class TestTorchShiftOps: - def test_fftshift_matches_torch(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0]) - assert torch.allclose(TorchFftShiftOp()(Sample(input=t)).input, torch.fft.fftshift(t)) - - def test_ifftshift_matches_torch(self) -> None: - t = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0]) - assert torch.allclose(TorchIfftShiftOp()(Sample(input=t)).input, torch.fft.ifftshift(t)) - - def test_ifftshift_inverts_fftshift_odd_length(self) -> None: - t = torch.arange(5) - restored = TorchIfftShiftOp()(TorchFftShiftOp()(Sample(input=t))) - assert torch.equal(restored.input, t) - - def test_preserves_dtype_half_and_integer(self) -> None: - # Pure rearrangement — no FFT — so half precision (which the FFT ops reject) passes through. - assert TorchFftShiftOp()(Sample(input=torch.arange(5))).input.dtype == torch.int64 - half = torch.tensor([1.0, 2.0, 3.0, 4.0], dtype=torch.float16) - assert TorchFftShiftOp()(Sample(input=half)).input.dtype == torch.float16 - - def test_dim_shifts_per_row(self) -> None: - t = torch.arange(15).reshape(3, 5) - assert torch.equal(TorchFftShiftOp(dim=-1)(Sample(input=t)).input, torch.fft.fftshift(t, dim=-1)) - - def test_dim_none_shifts_all_dims(self) -> None: - t = torch.arange(15).reshape(3, 5) - assert torch.equal(TorchFftShiftOp(dim=None)(Sample(input=t)).input, torch.fft.fftshift(t)) - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="FftShiftOp expects a torch.Tensor"): - TorchFftShiftOp()(Sample(input=np.zeros(4))) - with pytest.raises(TypeError, match="IfftShiftOp expects a torch.Tensor"): - TorchIfftShiftOp()(Sample(input=np.zeros(4))) - - def test_zero_arg_construction(self) -> None: - assert TorchFftShiftOp().dim == -1 - assert TorchIfftShiftOp().dim == -1 - - def test_produces_contract_conforms(self) -> None: - out = TorchFftShiftOp()(Sample(input=torch.arange(5))) - assert TorchFftShiftOp.PRODUCES.accepts(infer_sample_type(out)) - - -# --------------------------------------------------------------------------- -# Full pipeline round-trips combining the ops -# --------------------------------------------------------------------------- - - -class TestRoundTrips: - def test_numpy_fft_shift_unshift_ifft_recovers(self) -> None: - # FourierOp -> FftShiftOp -> IfftShiftOp -> InverseFourierOp == identity (real signal). - x = np.array([1.0, -2.0, 3.5, 0.0, 7.0], dtype=np.float64) - s = NpFourierOp()(Sample(input=x)) - s = NpFftShiftOp()(s) - s = NpIfftShiftOp()(s) - s = NpInverseFourierOp()(s) - assert np.allclose(s.input.real, x, atol=1e-9) - - def test_torch_fft_shift_unshift_ifft_recovers(self) -> None: - t = torch.tensor([1.0, -2.0, 3.5, 0.0, 7.0], dtype=torch.float64) - s = TorchFourierOp()(Sample(input=t)) - s = TorchFftShiftOp()(s) - s = TorchIfftShiftOp()(s) - s = TorchInverseFourierOp()(s) - assert torch.allclose(s.input.real, t, atol=1e-9) - - -# --------------------------------------------------------------------------- -# Windowing + unit scaling (WindowOp / SpectrumScalingOp + FourierOp options) -# --------------------------------------------------------------------------- - - -def _np_tone(n: int = 1024, bin_index: int = 64, amp: float = 1.0) -> np.ndarray: - return np.asarray(amp * np.exp(2j * np.pi * bin_index * np.arange(n) / n), dtype=np.complex64) - - -class TestNumpyWindowAndScaling: - def test_default_fourierop_unchanged_and_metadata_byte_identical(self) -> None: - x = _np_tone() - s = Sample(input=x, target=None, metadata={"k": 1}) - out = NpFourierOp()(s) - assert np.allclose(out.input, np.fft.fft(x)) and out.input.dtype == np.complex64 - assert out.metadata == s.metadata # boxcar/none stamps nothing - - def test_window_option_stashes_correction(self) -> None: - out = NpFourierOp(window="hann")(Sample(input=_np_tone(), target=None, metadata={})) - assert out.meta["window"] == "hann" - assert out.meta["window_sum"] == pytest.approx(512.0) - assert out.meta["window_enbw_bins"] == pytest.approx(1.5) - - def test_windowop_preserves_complex64_dtype(self) -> None: - out = NpWindowOp(window="hann")(Sample(input=_np_tone(), target=None, metadata={})) - assert out.input.dtype == np.complex64 # not upcast to complex128 by the float window - assert out.meta["window_sum"] == pytest.approx(512.0) - - def test_amplitude_and_power_recover_tone(self) -> None: - s = Sample(input=_np_tone(amp=1.0), target=None, metadata={}) - amp = NpFourierOp(window="hann", scaling="amplitude")(s).input - pw = NpFourierOp(window="hann", scaling="power")(s).input - assert np.abs(amp).max() == pytest.approx(1.0, abs=1e-4) - assert pw.max() == pytest.approx(1.0, abs=1e-4) - assert np.iscomplexobj(amp) and not np.iscomplexobj(pw) - - def test_one_node_equals_explicit_chain(self) -> None: - s = Sample(input=_np_tone(), target=None, metadata={"samplerate": 1000.0}) - one = NpFourierOp(window="hann", scaling="density", sample_rate=1000.0)(s).input - chain = NpSpectrumScalingOp(scaling="density", sample_rate=1000.0)( - NpFourierOp()(NpWindowOp(window="hann")(s)) - ).input - assert np.allclose(one, chain) - - def test_spectrumscaling_rectangular_fallback_without_metadata(self) -> None: - s = Sample(input=_np_tone(), target=None, metadata={}) - raw = NpFourierOp()(s) # boxcar default → no window correction stashed - assert WINDOW_SUM_KEY not in raw.metadata - pw = NpSpectrumScalingOp(scaling="power")(raw).input - n = raw.input.shape[-1] - assert np.allclose(pw, np.abs(raw.input) ** 2 / n**2) # rectangular S1=S2=N fallback - - def test_scaling_requires_backward_norm(self) -> None: - s = Sample(input=_np_tone(), target=None, metadata={}) - with pytest.raises(ValueError, match="norm='backward'"): - NpFourierOp(scaling="power", norm="ortho")(s) - - def test_shift_and_one_sided_mutually_exclusive(self) -> None: - s = Sample(input=_np_tone(), target=None, metadata={}) - with pytest.raises(ValueError, match="mutually exclusive"): - NpFourierOp(scaling="power", shift=True, one_sided=True)(s) - - def test_one_sided_real_signal_amplitude(self) -> None: - n = 1024 - t = np.arange(n) - x = (2.0 * np.cos(2 * np.pi * 16 * t / n)).astype(np.float64) - amp = NpFourierOp(scaling="amplitude", one_sided=True)(Sample(input=x, target=None, metadata={})).input - assert amp.shape[0] == n // 2 + 1 - assert np.abs(amp).max() == pytest.approx(2.0, abs=1e-6) - - def test_produces_accepts_complex_and_real(self) -> None: - s = Sample(input=_np_tone(), target=None, metadata={}) - assert NpFourierOp.PRODUCES.accepts(infer_sample_type(NpFourierOp()(s))) # complex - assert NpFourierOp.PRODUCES.accepts(infer_sample_type(NpFourierOp(scaling="power")(s))) # real - scaled = NpSpectrumScalingOp(scaling="power")(NpFourierOp()(s)) - assert NpSpectrumScalingOp.PRODUCES.accepts(infer_sample_type(scaled)) - - def test_sample_rate_from_metadata(self) -> None: - s = Sample(input=_np_tone(), target=None, metadata={"samplerate": 500.0}) - from_meta = NpFourierOp(window="hann", scaling="density")(s).input - explicit = NpFourierOp(window="hann", scaling="density", sample_rate=500.0)(s).input - assert np.allclose(from_meta, explicit) - - def test_spectrumscaling_density_no_rate_runs(self) -> None: - # exercises the normalized-frequency fallback (Fs=1.0) + debug log branch - s = Sample(input=_np_tone(), target=None, metadata={}) - out = NpSpectrumScalingOp(scaling="density")(NpFourierOp()(s)).input - assert out.shape == (1024,) and not np.iscomplexobj(out) - - -class TestTorchWindowAndScaling: - def test_default_unchanged(self) -> None: - x = torch.view_as_complex(torch.randn(1024, 2)) - out = TorchFourierOp()(Sample(input=x, target=None, metadata={"k": 1})) - assert torch.allclose(out.input, torch.fft.fft(x)) - assert out.metadata == {"k": 1} # boxcar/none stamps nothing - - def test_amplitude_recovers_tone(self) -> None: - n = 1024 - x = torch.exp(2j * torch.pi * 64 * torch.arange(n) / n).to(torch.complex64) - amp = TorchFourierOp(window="hann", scaling="amplitude")(Sample(input=x, target=None, metadata={})).input - assert amp.abs().max().item() == pytest.approx(1.0, abs=1e-3) - - def test_window_stashes_and_preserves_dtype(self) -> None: - x = torch.exp(2j * torch.pi * 64 * torch.arange(1024) / 1024).to(torch.complex64) - out = TorchWindowOp(window="hann")(Sample(input=x, target=None, metadata={})) - assert out.input.dtype == torch.complex64 - assert out.meta["window_sum"] == pytest.approx(512.0) - - def test_power_is_real_and_scaling_requires_backward_norm(self) -> None: - x = torch.view_as_complex(torch.randn(64, 2)) - pw = TorchFourierOp(window="hann", scaling="power")(Sample(input=x, target=None, metadata={})).input - assert not pw.is_complex() - with pytest.raises(ValueError, match="norm='backward'"): - TorchFourierOp(scaling="power", norm="ortho")(Sample(input=x, target=None, metadata={})) - - def test_spectrumscaling_standalone_matches_one_node(self) -> None: - n = 256 - x = torch.exp(2j * torch.pi * 20 * torch.arange(n) / n).to(torch.complex64) - s = Sample(input=x, target=None, metadata={}) - one = TorchFourierOp(window="hamming", scaling="power")(s).input - chain = TorchSpectrumScalingOp(scaling="power")(TorchFourierOp()(TorchWindowOp(window="hamming")(s))).input - assert torch.allclose(one, chain, atol=1e-4) - - -def test_numpy_torch_parity_window_and_scaling() -> None: - n = 512 - base = np.exp(2j * np.pi * 40 * np.arange(n) / n).astype(np.complex64) - meta = {"samplerate": 2000.0} - for scaling in ("none", "amplitude", "power", "density"): - npo = NpFourierOp(window="blackmanharris", scaling=scaling, sample_rate=2000.0)( - Sample(input=base.copy(), target=None, metadata=dict(meta)) - ).input - to = TorchFourierOp(window="blackmanharris", scaling=scaling, sample_rate=2000.0)( - Sample(input=torch.from_numpy(base.copy()), target=None, metadata=dict(meta)) - ).input - assert np.allclose(npo, to.numpy(), rtol=1e-3, atol=1e-3), scaling diff --git a/tests/test_kinds.py b/tests/test_kinds.py index 0f124af..30f330f 100644 --- a/tests/test_kinds.py +++ b/tests/test_kinds.py @@ -1,14 +1,14 @@ """Tests for op-kind introspection (`sampleflux.kinds`) and the native multi-type engine.""" -from typing import Any, Iterable, Iterator, Optional, Tuple +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, cast import numpy as np import pytest from confluid import configurable from sampleflux.core import Flux -from sampleflux.kinds import SAMPLE_KINDS, OpContract, classify_carrier, op_contract -from sampleflux.sample import Sample +from sampleflux.kinds import SAMPLE_KINDS, Input, OpContract, Target, classify_carrier, op_contract +from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta # --------------------------------------------------------------------------- # Fixture ops (module-level so they pickle for spawn parity) @@ -83,7 +83,17 @@ def __call__(self, sample: "Sample") -> "Sample": class TestClassify: def test_kinds_taxonomy_is_closed(self) -> None: - assert SAMPLE_KINDS == ("sample", "pair", "value", "any") + assert SAMPLE_KINDS == ( + "sample", + "pair", + "input", + "target", + "metadata", + "input_meta", + "target_meta", + "value", + "any", + ) def test_classify_carrier(self) -> None: assert classify_carrier(Sample(1)) == "sample" @@ -260,3 +270,311 @@ def test_stack_fallback_to_list(self) -> None: out = collate(["a", "b"], key="value") assert out == ["a", "b"] + + +# --------------------------------------------------------------------------- +# The field-scope grid + call styles (the (input, target, metadata) taxonomy) +# --------------------------------------------------------------------------- + + +@configurable +class BareInputOp: + """Processes ONLY the input value (any array/tensor/dict), declared via the Input alias.""" + + def __call__(self, x: Input): # type: ignore[no-untyped-def] + return x * 2 + + +@configurable +class BareTargetOp: + """Processes ONLY the target value.""" + + def __call__(self, t: Target): # type: ignore[no-untyped-def] + return t + 100 + + +@configurable +class UnpackedPairOp: + """transform(input, target) — the classic AI signature, unpacked.""" + + def __call__(self, input, target): # type: ignore[no-untyped-def] + return input * 2, target + 1 + + +@configurable +class UnpackedInputMetaOp: + """transform(input, metadata) — input with its metadata, unpacked.""" + + def __call__(self, input, metadata): # type: ignore[no-untyped-def] + metadata["seen"] = True + return input + 1, metadata + + +@configurable +class UnpackedTargetMetaOp: + """transform(target, metadata) — target side selected by the first param name.""" + + def __call__(self, target, metadata): # type: ignore[no-untyped-def] + return target * 10, {**metadata, "t": True} + + +@configurable +class UnpackedSampleOp: + """transform(input, target, metadata) — the full triple, unpacked.""" + + def __call__(self, input, target, metadata): # type: ignore[no-untyped-def] + return input + 1, target + 1, {**metadata, "s": True} + + +@configurable +class PackedInputMetaOp: + """Packed InputMeta view — the op receives a named (input, metadata) object.""" + + def __call__(self, view: InputMeta) -> InputMeta: + meta = cast(Dict[str, Any], view.metadata) # per-sample ops always see the dict form + return InputMeta(view.input * 3, {**meta, "packed": True}) + + +@configurable +class PackedTargetMetaOp: + """Packed TargetMeta view.""" + + def __call__(self, view: TargetMeta) -> TargetMeta: + return TargetMeta(view.target - 1, view.metadata) + + +@configurable +class PackedNamedPairOp: + """Packed Pair view (the named 2-tuple form).""" + + def __call__(self, p: Pair) -> Pair: + return Pair(p.input + 0.5, p.target) + + +@configurable +class OptionalExtraArgOp: + """One REQUIRED param + optional extras — must stay single-argument (packed/any).""" + + def __call__(self, sample, extra=None): # type: ignore[no-untyped-def] + return sample + + +class TestGridContracts: + def test_bare_field_marks(self) -> None: + assert op_contract(BareInputOp()) == OpContract("input", "any", False, "packed") + assert op_contract(BareTargetOp()).accepts == "target" + + def test_unpacked_pair(self) -> None: + assert op_contract(UnpackedPairOp()) == OpContract("pair", "any", False, "unpacked", ("input", "target")) + + def test_unpacked_meta_variants_by_param_names(self) -> None: + assert op_contract(UnpackedInputMetaOp()) == OpContract( + "input_meta", "any", False, "unpacked", ("input", "metadata") + ) + assert op_contract(UnpackedTargetMetaOp()).accepts == "target_meta" + + def test_unpacked_sample_triple(self) -> None: + assert op_contract(UnpackedSampleOp()) == OpContract( + "sample", "any", False, "unpacked", ("input", "target", "metadata") + ) + + def test_packed_views(self) -> None: + assert op_contract(PackedInputMetaOp()).accepts == "input_meta" + assert op_contract(PackedInputMetaOp()).style == "packed" + assert op_contract(PackedTargetMetaOp()).accepts == "target_meta" + assert op_contract(PackedNamedPairOp()).accepts == "pair" + + def test_optional_extras_keep_single_arg_semantics(self) -> None: + # Required arity 1 -> packed/any: op(sample) exactly as today. + assert op_contract(OptionalExtraArgOp()) == OpContract("any", "any", False, "packed") + + def test_call_style_override(self) -> None: + class Opaque: + SAMPLE_KIND_IN = "pair" + CALL_STYLE = "unpacked" + + def __call__(self, *args): # type: ignore[no-untyped-def] + return args[0], args[1] + + contract = op_contract(Opaque()) + assert contract.accepts == "pair" and contract.style == "unpacked" + + def test_classify_carrier_views_before_tuple_rule(self) -> None: + assert classify_carrier(InputMeta(1, {"a": 1})) == "input_meta" + assert classify_carrier(TargetMeta(1, {})) == "target_meta" + assert classify_carrier(Pair(1, 2)) == "pair" + assert classify_carrier((1, 2)) == "pair" # the plain tuple stays a pair + + +class TestGridEngineBinding: + def _samples(self, n: int = 2) -> list: + return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] + + def test_bare_input_op_preserves_target_and_meta(self) -> None: + out = list(Flux(source=self._samples(), ops=[BareInputOp()])) + assert [s.input for s in out] == [0.0, 2.0] + assert [s.target for s in out] == [0, 1] + assert [s.meta["idx"] for s in out] == [0, 1] + + def test_bare_target_op(self) -> None: + out = list(Flux(source=self._samples(), ops=[BareTargetOp()])) + assert [s.target for s in out] == [100, 101] + assert [s.input for s in out] == [0.0, 1.0] + + def test_unpacked_pair_op_merges_back(self) -> None: + out = list(Flux(source=self._samples(), ops=[UnpackedPairOp()])) + assert [(s.input, s.target) for s in out] == [(0.0, 1), (2.0, 2)] + assert [s.meta["idx"] for s in out] == [0, 1] # metadata preserved + + def test_unpacked_input_meta_op(self) -> None: + out = list(Flux(source=self._samples(), ops=[UnpackedInputMetaOp()])) + assert [s.input for s in out] == [1.0, 2.0] + assert all(s.meta["seen"] is True for s in out) + assert [s.target for s in out] == [0, 1] # target untouched + + def test_unpacked_target_meta_op(self) -> None: + out = list(Flux(source=self._samples(), ops=[UnpackedTargetMetaOp()])) + assert [s.target for s in out] == [0, 10] + assert all(s.meta["t"] is True for s in out) + assert [s.input for s in out] == [0.0, 1.0] + + def test_unpacked_sample_op(self) -> None: + out = list(Flux(source=self._samples(), ops=[UnpackedSampleOp()])) + assert [(s.input, s.target) for s in out] == [(1.0, 1), (2.0, 2)] + assert all(s.meta["s"] is True and "idx" in s.meta for s in out) + + def test_packed_views_merge_back(self) -> None: + out = list(Flux(source=self._samples(), ops=[PackedInputMetaOp(), PackedTargetMetaOp()])) + assert [s.input for s in out] == [0.0, 3.0] + assert [s.target for s in out] == [-1, 0] + assert all(s.meta["packed"] is True for s in out) + + def test_packed_named_pair(self) -> None: + out = list(Flux(source=self._samples(), ops=[PackedNamedPairOp()])) + assert [s.input for s in out] == [0.5, 1.5] + assert [s.meta["idx"] for s in out] == [0, 1] + + def test_grid_chain_mixes_all_styles(self) -> None: + ops = [BareInputOp(), UnpackedPairOp(), PackedInputMetaOp(), SampleOp()] + out = list(Flux(source=self._samples(1), ops=ops)) + # 0.0 -> *2=0.0 -> pair(*2, +1)=(0.0, 1) -> *3=0.0 -> SampleOp(+1 input)=1.0 + assert out[0].input == 1.0 and out[0].target == 1 + assert out[0].meta["packed"] is True and out[0].meta["idx"] == 0 + + def test_none_drops_in_every_scope(self) -> None: + @configurable + class DropPair: + def __call__(self, input, target): # type: ignore[no-untyped-def] + return None + + assert list(Flux(source=self._samples(), ops=[DropPair()])) == [] + + def test_pair_scope_single_return_is_a_loud_error(self) -> None: + @configurable + class BadPair: + def __call__(self, input, target): # type: ignore[no-untyped-def] + return input # ambiguous — must be a 2-tuple / Sample / None + + with pytest.raises(TypeError, match="same arity"): + list(Flux(source=self._samples(1), ops=[BadPair()])) + + def test_native_bare_input_on_value_carrier_stays_value(self) -> None: + out = list(Flux(source=[1.0, 2.0], ops=[BareInputOp()], native=True)) + assert out == [2.0, 4.0] # no promotion — bare values stay bare + + def test_native_unpacked_pair_on_pair_carrier_stays_pair(self) -> None: + out = list(Flux(source=[(1.0, 1), (2.0, 2)], ops=[UnpackedPairOp()], native=True)) + assert out == [(2.0, 2), (4.0, 3)] + + def test_native_view_carrier_promotes_field_correct(self) -> None: + # An InputMeta carrier + a sample-op: from_any must NOT misread metadata as target. + out = list(Flux(source=[InputMeta(1.0, {"m": 1})], ops=[SampleOp()], native=True)) + assert out[0].input == 2.0 and out[0].target is None and out[0].meta == {"m": 1} + + def test_view_collate_defaults(self) -> None: + from sampleflux.collate import collate + + batch = collate([InputMeta(np.ones(2), {"i": 0}), InputMeta(np.zeros(2), {"i": 1})]) + assert isinstance(batch, InputMeta) and batch.input.shape == (2, 2) + metas = cast(List[Dict[str, Any]], batch.metadata) # batched form: list of per-item dicts + assert metas[1] == {"i": 1} + + +# --------------------------------------------------------------------------- +# Combination bindings — mixed views as separate arguments +# --------------------------------------------------------------------------- + + +@configurable +class DualViewOp: + """transform(InputMeta, TargetMeta) — both fields, each WITH its metadata.""" + + def __call__(self, im: InputMeta, tm: TargetMeta): # type: ignore[no-untyped-def] + meta = cast(Dict[str, Any], im.metadata) + return InputMeta(im.input * 2, {**meta, "im": True}), TargetMeta( + tm.target + 1, {**meta, "im": True, "tm": True} + ) + + +@configurable +class MixedMarkViewOp: + """transform(Input, TargetMeta) — a bare input value + the target with metadata.""" + + def __call__(self, x: Input, tm: TargetMeta): # type: ignore[no-untyped-def] + return x + 0.5, tm + + +@configurable +class MetadataOnlyOp: + """transform(metadata) — the metadata-only cell of the grid, declared by dict annotation.""" + + def __call__(self, m: dict) -> dict: + return {**m, "canonical": True} + + +class TestCombinationBindings: + def _samples(self, n: int = 2) -> list: + return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] + + def test_dual_view_contract(self) -> None: + contract = op_contract(DualViewOp()) + assert contract.bindings == ("input_meta", "target_meta") + assert contract.accepts == "sample" # covers input+target+metadata — the grid summary + assert contract.style == "unpacked" + + def test_dual_view_execution_merges_all_fields(self) -> None: + out = list(Flux(source=self._samples(), ops=[DualViewOp()])) + assert [s.input for s in out] == [0.0, 2.0] + assert [s.target for s in out] == [1, 2] + # last metadata-bearing element wins (it layered im's write too) + assert all(s.meta["im"] is True and s.meta["tm"] is True and "idx" in s.meta for s in out) + + def test_mixed_mark_and_view(self) -> None: + contract = op_contract(MixedMarkViewOp()) + assert contract.bindings == ("input", "target_meta") + out = list(Flux(source=self._samples(1), ops=[MixedMarkViewOp()])) + assert out[0].input == 0.5 and out[0].target == 0 and out[0].meta == {"idx": 0} + + def test_metadata_only_scope(self) -> None: + contract = op_contract(MetadataOnlyOp()) + assert contract.accepts == "metadata" and contract.style == "packed" + out = list(Flux(source=self._samples(1), ops=[MetadataOnlyOp()])) + assert out[0].meta == {"idx": 0, "canonical": True} + assert out[0].input == 0.0 and out[0].target == 0 # untouched + + def test_wrong_arity_return_is_a_loud_error(self) -> None: + @configurable + class Bad: + def __call__(self, im: InputMeta, tm: TargetMeta): # type: ignore[no-untyped-def] + return im # must be a 2-tuple / Sample / None + + with pytest.raises(TypeError, match="same arity"): + list(Flux(source=self._samples(1), ops=[Bad()])) + + def test_binding_names_override_positions(self) -> None: + @configurable + class TargetFirst: + def __call__(self, target, input): # type: ignore[no-untyped-def] + return target, input + + assert op_contract(TargetFirst()).bindings == ("target", "input") diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index f71b73f..57c9e36 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -13,27 +13,10 @@ from confluid import parse_param_docs # type: ignore[import-not-found] from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.ops.numpy import ( - ConnectedComponentsOp, - FftShiftOp, - FourierOp, - IfftShiftOp, - InverseFourierOp, - SpectrumScalingOp, - StandardizeOp, - ThresholdOp, - WindowOp, -) +from sampleflux.ops.numpy import ConnectedComponentsOp, StandardizeOp, ThresholdOp from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp -from sampleflux.ops.tee import Tee -from sampleflux.ops.torch import FftShiftOp as TorchFftShiftOp -from sampleflux.ops.torch import FourierOp as TorchFourierOp -from sampleflux.ops.torch import IfftShiftOp as TorchIfftShiftOp -from sampleflux.ops.torch import InverseFourierOp as TorchInverseFourierOp -from sampleflux.ops.torch import SpectrumScalingOp as TorchSpectrumScalingOp from sampleflux.ops.torch import StandardizeOp as TorchStandardizeOp from sampleflux.ops.torch import ToTensorOp -from sampleflux.ops.torch import WindowOp as TorchWindowOp from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import HuggingFaceSource @@ -43,24 +26,11 @@ JointFlux, FilterOp, WrappedOp, - Tee, StandardizeOp, ThresholdOp, ConnectedComponentsOp, - FourierOp, - InverseFourierOp, - FftShiftOp, - IfftShiftOp, ToTensorOp, TorchStandardizeOp, - TorchFourierOp, - TorchInverseFourierOp, - TorchFftShiftOp, - TorchIfftShiftOp, - WindowOp, - SpectrumScalingOp, - TorchWindowOp, - TorchSpectrumScalingOp, MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp, diff --git a/tests/test_ops.py b/tests/test_ops.py index 8a202e4..38860a6 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -8,7 +8,6 @@ from PIL import Image from sampleflux.ops import ( - CaptureOutputOp, ConfigureOp, CopyInputOp, CopyMetadataOp, @@ -21,7 +20,6 @@ StashInputOp, StashTargetOp, SwapInputTargetOp, - Tee, ToTensorOp, UnsqueezeOp, UnstashInputOp, @@ -450,63 +448,6 @@ def test_validation_rejects_unknown_string(self) -> None: # --------------------------------------------------------------------------- -# Tee -# --------------------------------------------------------------------------- - - -class TestTee: - def test_two_branches_share_metadata(self) -> None: - sample = Sample(input=np.array([1.0]), target=None, metadata={}) - - def writer_a(s: Sample) -> Sample: - s.meta["a"] = 1 - return s - - def writer_b(s: Sample) -> Sample: - assert s.meta["a"] == 1 # branch A's write is visible - s.meta["b"] = 2 - return s - - out = Tee(branches=[[writer_a], [writer_b]])(sample) - assert out is not None - assert out.meta == {"a": 1, "b": 2} - - def test_branches_run_sequentially(self) -> None: - from typing import Callable - - order: list[str] = [] - - def make(tag: str) -> Callable[[Sample], Sample]: - def op(s: Sample) -> Sample: - order.append(tag) - return s - - return op - - Tee(branches=[[make("A1"), make("A2")], [make("B1"), make("B2")]])(Sample(input=None)) - assert order == ["A1", "A2", "B1", "B2"] - - def test_none_propagates(self) -> None: - def filter_out(s: Sample) -> None: - return None - - def should_not_run(s: Sample) -> Sample: - raise AssertionError("downstream branch must not run after None") - - out = Tee(branches=[[filter_out], [should_not_run]])(Sample(input=1)) - assert out is None - - def test_input_mutations_flow_into_next_branch(self) -> None: - def to_zero(s: Sample) -> Sample: - return s._replace(input=0) - - def must_see_zero(s: Sample) -> Sample: - assert s.input == 0 - return s._replace(input=99) - - out = Tee(branches=[[to_zero], [must_see_zero]])(Sample(input=42)) - assert out is not None - assert out.input == 99 # --------------------------------------------------------------------------- @@ -765,7 +706,7 @@ def test_missing_target_or_param_raise_lazily(self) -> None: ConfigureOp(target=ThresholdOp())(sample) def test_compute_chain_filtering_drops_sample(self) -> None: - """A compute op returning None propagates the drop (FilterOp semantics, like Tee).""" + """A compute op returning None propagates the drop (FilterOp semantics).""" sample = Sample(input=np.array([1.0]), target=None, metadata={}) op = ConfigureOp(ops=[lambda s: None], target=ThresholdOp(), param="low_level") assert op(sample) is None @@ -786,122 +727,6 @@ def test_fluid_markers_flow_lazily(self) -> None: np.testing.assert_array_equal(out.input, [False, True]) -class TestCaptureOutputOp: - class _DrawOp: - """Stub op: transforms the sample (+1) and exposes its drawn value as an @output-like property.""" - - def __init__(self, value: float = 0.0) -> None: - self._value = value - self._last: object = None - self.calls = 0 - - def __call__(self, sample: Sample) -> Sample: - self.calls += 1 - self._last = self._value - return sample._replace(input=sample.input + 1) - - @property - def drawn(self) -> object: - return self._last - - def test_records_output_into_metadata_and_keeps_transform(self) -> None: - op = self._DrawOp(value=42.0) - sample = Sample(input=np.array([1.0]), target=None, metadata={}) - out = CaptureOutputOp(op=op, output="drawn", key="captured")(sample) - assert out is not None - assert out.meta["captured"] == 42.0 # the @output value rides metadata - np.testing.assert_array_equal(out.input, [2.0]) # the wrapped op's transform is kept - assert op.calls == 1 - - def test_default_key_is_output_name(self) -> None: - out = CaptureOutputOp(op=self._DrawOp(value=7.0), output="drawn")(Sample(input=0, target=None, metadata={})) - assert out is not None and out.meta["drawn"] == 7.0 - - def test_multi_capture_applies_op_once(self) -> None: - class _Multi: - def __init__(self) -> None: - self.calls = 0 - - def __call__(self, s: Sample) -> Sample: - self.calls += 1 - return s - - @property - def a(self) -> int: - return 1 - - @property - def b(self) -> int: - return 2 - - op = _Multi() - out = CaptureOutputOp(op=op, captures={"a": "ka", "b": "kb"})(Sample(input=0, target=None, metadata={})) - assert out is not None and out.meta["ka"] == 1 and out.meta["kb"] == 2 - assert op.calls == 1 # one application, several captures - - def test_captures_actual_drawn_value_not_recompute(self) -> None: - """A stochastic @output must be captured from the SAME application — never a fresh re-draw.""" - seq = iter([11.0, 22.0, 33.0]) - - class _Stochastic: - def __init__(self) -> None: - self._last: object = None - - def __call__(self, s: Sample) -> Sample: - self._last = next(seq) - return s - - @property - def drawn(self) -> object: - return self._last - - out = CaptureOutputOp(op=_Stochastic(), output="drawn", key="v")(Sample(input=0, target=None, metadata={})) - assert out is not None and out.meta["v"] == 11.0 # the first (and only) draw - - def test_requires_op(self) -> None: - with pytest.raises(ValueError, match="'op'"): - CaptureOutputOp(output="x")(Sample(input=0, target=None, metadata={})) - - def test_requires_something_to_capture(self) -> None: - with pytest.raises(ValueError, match="nothing to capture"): - CaptureOutputOp(op=self._DrawOp())(Sample(input=0, target=None, metadata={})) - - def test_missing_attribute_raises(self) -> None: - with pytest.raises(AttributeError, match="no @output attribute"): - CaptureOutputOp(op=self._DrawOp(), output="nope")(Sample(input=0, target=None, metadata={})) - - def test_filtering_op_propagates_none(self) -> None: - assert ( - CaptureOutputOp(op=lambda s: None, output="x", key="k")(Sample(input=0, target=None, metadata={})) is None - ) - - def test_reads_output_through_target_wrapper(self) -> None: - """The @output is read THROUGH a ``.target`` wrapper (e.g. a ConfigureOp), so a node that is - both a capture-consumer (its param configured) AND a capture-producer composes.""" - from typing import Callable - - class _Wrapper: # mimics ConfigureOp: applies .target and exposes it as .target - def __init__(self, target: Callable[[Sample], Sample]) -> None: - self.target = target - - def __call__(self, s: Sample) -> Sample: - return self.target(s) - - out = CaptureOutputOp(op=_Wrapper(self._DrawOp(value=99.0)), output="drawn", key="v")( - Sample(input=np.array([1.0]), target=None, metadata={}) - ) - assert out is not None and out.meta["v"] == 99.0 - np.testing.assert_array_equal(out.input, [2.0]) - - def test_fluid_marker_op_flows_lazily(self) -> None: - from confluid.fluid import Class - - op = CaptureOutputOp(op=Class(self._DrawOp, value=5.0), output="drawn", key="v") - out = op(Sample(input=np.array([1.0]), target=None, metadata={})) - assert out is not None and out.meta["v"] == 5.0 - np.testing.assert_array_equal(out.input, [2.0]) - - # --------------------------------------------------------------------------- # numpy.resolve_expression # --------------------------------------------------------------------------- diff --git a/tests/test_paired.py b/tests/test_paired.py deleted file mode 100644 index 267bc0d..0000000 --- a/tests/test_paired.py +++ /dev/null @@ -1,497 +0,0 @@ -"""Tests for sampleflux.paired.AnnotationJoinSource.""" - -from typing import Any, Dict, Iterator, Optional - -import confluid # type: ignore[import-not-found] -import pytest - -from sampleflux.discovery import get_callable_path -from sampleflux.paired import AnnotationJoinSource -from sampleflux.sample import Sample - -# --------------------------------------------------------------------------- -# Test fixtures -# --------------------------------------------------------------------------- - - -@confluid.configurable -class PairedIndexedSource: - """Indexable data source producing Samples keyed by integer id.""" - - def __init__(self, size: int = 4) -> None: - self.size = size - - def __len__(self) -> int: - return self.size - - def __getitem__(self, index: int) -> Sample: - return Sample(input=index * 10, target=None, metadata={"id": f"s{index}"}) - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.size): - yield self[i] - - -@confluid.configurable -class DictStore: - """Minimal mapping-shaped annotations for tests.""" - - def __init__(self, records: Optional[Dict[str, Dict[str, Any]]] = None) -> None: - self.records: Dict[str, Dict[str, Any]] = records or {} - - def __contains__(self, key: str) -> bool: - return key in self.records - - def __getitem__(self, key: str) -> Dict[str, Any]: - return self.records[key] - - def keys(self) -> Any: - return self.records.keys() - - -# Module-level callables so resolve_callable can find them -def sample_id_key(sample: Sample) -> str: - return str(sample.meta["id"]) - - -def identity_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - return record - - -def none_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - return None - - -def odd_only_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - """Return the record only for odd-id samples, None otherwise.""" - idx = int(str(sample.meta["id"])[1:]) - if idx % 2 == 1: - return record - return None - - -def resolve_by_id(key: str, data: PairedIndexedSource) -> Sample: - """Reverse-lookup: key 's' -> data[N].""" - idx = int(key[1:]) - return data[idx] - - -# --------------------------------------------------------------------------- -# left_outer -# --------------------------------------------------------------------------- - - -def test_left_outer_emits_all_data() -> None: - data = PairedIndexedSource(size=4) - store = DictStore({"s0": {"label": "a"}, "s2": {"label": "c"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) - - samples = list(paired) - - assert len(samples) == 4 - assert [s.input for s in samples] == [0, 10, 20, 30] - assert [s.meta["annotated"] for s in samples] == [True, False, True, False] - - -def test_left_outer_flattens_record_into_metadata() -> None: - data = PairedIndexedSource(size=2) - store = DictStore({"s0": {"label": "dog", "confidence": 0.9}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) - - samples = list(paired) - - assert samples[0].meta["label"] == "dog" - assert samples[0].meta["confidence"] == 0.9 - assert samples[0].meta["annotation_key"] == "s0" - assert "label" not in samples[1].meta - assert samples[1].meta["annotation_key"] == "s1" - - -def test_left_outer_preserves_original_metadata() -> None: - data = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) - - sample = list(paired)[0] - assert sample.meta["id"] == "s0" # data metadata survived - assert sample.meta["label"] == "x" - - -def test_left_outer_prefix() -> None: - data = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, prefix="ann_") - - sample = list(paired)[0] - assert sample.meta["ann_label"] == "x" - assert "label" not in sample.meta - - -def test_left_outer_store_full_under() -> None: - data = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x", "score": 0.5}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - store_full_under="raw_annotation", - ) - - sample = list(paired)[0] - assert sample.meta["raw_annotation"] == {"label": "x", "score": 0.5} - # Still flattened too - assert sample.meta["label"] == "x" - - -def test_left_outer_len_delegates_to_data() -> None: - data = PairedIndexedSource(size=7) - store = DictStore({"s0": {"label": "a"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) - - assert len(paired) == 7 - - -def test_left_outer_getitem_matched_and_unmatched() -> None: - data = PairedIndexedSource(size=3) - store = DictStore({"s1": {"label": "y"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key) - - matched = paired[1] - assert matched.meta["annotated"] is True - assert matched.meta["label"] == "y" - - unmatched = paired[0] - assert unmatched.meta["annotated"] is False - assert "label" not in unmatched.meta - - -# --------------------------------------------------------------------------- -# inner -# --------------------------------------------------------------------------- - - -def test_inner_emits_only_matched() -> None: - data = PairedIndexedSource(size=4) - store = DictStore({"s0": {"label": "a"}, "s3": {"label": "d"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, policy="inner") - - samples = list(paired) - - assert len(samples) == 2 - assert {s.meta["annotation_key"] for s in samples} == {"s0", "s3"} - assert all(s.meta["annotated"] for s in samples) - - -def test_inner_len_is_cached_scan() -> None: - data = PairedIndexedSource(size=10) - store = DictStore({f"s{i}": {"label": "x"} for i in (0, 2, 4, 6)}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, policy="inner") - - assert len(paired) == 4 - # Second call hits cache; should still be correct. - assert len(paired) == 4 - - -def test_inner_rejects_getitem() -> None: - data = PairedIndexedSource(size=2) - store = DictStore({"s0": {"label": "a"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=sample_id_key, policy="inner") - - with pytest.raises(TypeError, match="left_outer"): - _ = paired[0] - - -# --------------------------------------------------------------------------- -# extract_fn -# --------------------------------------------------------------------------- - - -def test_extract_fn_transforms_record() -> None: - data = PairedIndexedSource(size=2) - store = DictStore({"s0": {"label": "a"}, "s1": {"label": "b"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - extract_fn=identity_extract, - ) - - samples = list(paired) - - assert samples[0].meta["label"] == "a" - assert samples[1].meta["label"] == "b" - - -def test_extract_fn_returning_none_marks_unannotated() -> None: - data = PairedIndexedSource(size=3) - store = DictStore({"s0": {"label": "x"}, "s1": {"label": "y"}, "s2": {"label": "z"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - extract_fn=odd_only_extract, - ) - - samples = list(paired) - - # Every sample emitted under left_outer; only odd ones are annotated - assert [s.meta["annotated"] for s in samples] == [False, True, False] - - -def test_extract_fn_with_inner_policy_filters() -> None: - data = PairedIndexedSource(size=4) - store = DictStore({f"s{i}": {"label": "x"} for i in range(4)}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - extract_fn=odd_only_extract, - policy="inner", - ) - - samples = list(paired) - - assert {s.meta["annotation_key"] for s in samples} == {"s1", "s3"} - - -def test_extract_fn_none_suppresses_flattening() -> None: - data = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - extract_fn=none_extract, - ) - - sample = list(paired)[0] - assert sample.meta["annotated"] is False - assert "label" not in sample.meta - - -# --------------------------------------------------------------------------- -# Coarser-key / broadcast -# --------------------------------------------------------------------------- - - -def pack_key(sample: Sample) -> str: - """Every sample shares the key 'pack' - simulates pack-level annotation.""" - return "pack" - - -def test_coarser_key_broadcasts_to_all_matching_samples() -> None: - data = PairedIndexedSource(size=3) - store = DictStore({"pack": {"drone": "dji_mavic"}}) - paired = AnnotationJoinSource(data=data, annotations=store, key_fn=pack_key) - - samples = list(paired) - - assert all(s.meta["annotated"] for s in samples) - assert all(s.meta["drone"] == "dji_mavic" for s in samples) - - -# --------------------------------------------------------------------------- -# right_driven -# --------------------------------------------------------------------------- - - -def test_right_driven_iterates_annotation_keys() -> None: - data = PairedIndexedSource(size=10) - store = DictStore({"s0": {"label": "a"}, "s3": {"label": "d"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - policy="right_driven", - data_resolver=resolve_by_id, - ) - - samples = list(paired) - - assert len(samples) == 2 - assert [s.meta["annotation_key"] for s in samples] == ["s0", "s3"] - assert [s.input for s in samples] == [0, 30] - - -def test_right_driven_len_is_annotations_len() -> None: - data = PairedIndexedSource(size=100) - store = DictStore({"s1": {"label": "a"}, "s5": {"label": "b"}, "s9": {"label": "c"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - policy="right_driven", - data_resolver=resolve_by_id, - ) - - assert len(paired) == 3 - - -def test_right_driven_skips_when_extract_fn_returns_none() -> None: - data = PairedIndexedSource(size=4) - store = DictStore({f"s{i}": {"label": "x"} for i in range(4)}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - policy="right_driven", - data_resolver=resolve_by_id, - extract_fn=odd_only_extract, - ) - - samples = list(paired) - - assert {s.meta["annotation_key"] for s in samples} == {"s1", "s3"} - - -# --------------------------------------------------------------------------- -# Validation -# --------------------------------------------------------------------------- - - -def test_invalid_policy_raises() -> None: - # policy is a Literal: Confluid's @configurable validates it via pydantic at - # construction (pydantic's ValidationError is a ValueError subclass). - with pytest.raises(ValueError, match="policy"): - AnnotationJoinSource( - data=PairedIndexedSource(), - annotations=DictStore(), - key_fn=sample_id_key, - policy="outer_join", # type: ignore[arg-type] - ) - - -def test_right_driven_requires_data_resolver() -> None: - # Lazy: construction succeeds; the policy-conditional requirement is validated on iteration. - src = AnnotationJoinSource( - data=PairedIndexedSource(), - annotations=DictStore(), - key_fn=sample_id_key, - policy="right_driven", - ) - with pytest.raises(ValueError, match="data_resolver"): - list(src) - - -def test_right_driven_requires_annotations_keys_method() -> None: - class NoKeys: - def __contains__(self, k: str) -> bool: # pragma: no cover - defensive - return False - - def __getitem__(self, k: str) -> Any: # pragma: no cover - defensive - raise KeyError(k) - - # A store missing keys() doesn't satisfy the AnnotationStore Protocol; - # Confluid validates the param via pydantic at construction (ValidationError - # is a ValueError subclass). - with pytest.raises(ValueError, match="AnnotationStore"): - AnnotationJoinSource( - data=PairedIndexedSource(), - annotations=NoKeys(), # type: ignore[arg-type] # intentionally missing keys() - key_fn=sample_id_key, - policy="right_driven", - data_resolver=resolve_by_id, - ) - - -def test_left_outer_requires_mapping_interface() -> None: - class NoContains: - pass - - # Not mapping-shaped → fails the AnnotationStore Protocol at construction. - with pytest.raises(ValueError, match="AnnotationStore"): - AnnotationJoinSource( - data=PairedIndexedSource(), - annotations=NoContains(), # type: ignore[arg-type] # intentionally not mapping-shaped - key_fn=sample_id_key, - ) - - -# --------------------------------------------------------------------------- -# String-path resolution -# --------------------------------------------------------------------------- - - -def test_key_fn_accepts_string_path() -> None: - data = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=get_callable_path(sample_id_key), - ) - - sample = list(paired)[0] - assert sample.meta["label"] == "x" - - -def test_extract_fn_accepts_string_path() -> None: - data = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - extract_fn=get_callable_path(identity_extract), - ) - - sample = list(paired)[0] - assert sample.meta["label"] == "x" - - -def test_callable_is_stored_as_string() -> None: - paired = AnnotationJoinSource( - data=PairedIndexedSource(), - annotations=DictStore(), - key_fn=sample_id_key, - ) - - assert isinstance(paired.key_fn, str) - assert ":sample_id_key" in paired.key_fn - - -# --------------------------------------------------------------------------- -# Chained / composed -# --------------------------------------------------------------------------- - - -def test_chained_paired_sources_compose() -> None: - """Pack-level + window-level annotations merged via two AnnotationJoinSources.""" - data = PairedIndexedSource(size=3) - pack_store = DictStore({"pack": {"drone": "mavic"}}) - window_store = DictStore({"s1": {"event": "takeoff"}}) - - pack_paired = AnnotationJoinSource(data=data, annotations=pack_store, key_fn=pack_key) - full_paired = AnnotationJoinSource(data=pack_paired, annotations=window_store, key_fn=sample_id_key) - - samples = list(full_paired) - - # Every sample has drone (from pack), s1 also has event - assert all(s.meta["drone"] == "mavic" for s in samples) - assert samples[1].meta["event"] == "takeoff" - assert "event" not in samples[0].meta - - -# --------------------------------------------------------------------------- -# Confluid serialization round-trip -# --------------------------------------------------------------------------- - - -def test_confluid_roundtrip_preserves_behavior() -> None: - data = PairedIndexedSource(size=3) - store = DictStore({"s0": {"label": "a"}, "s2": {"label": "c"}}) - paired = AnnotationJoinSource( - data=data, - annotations=store, - key_fn=sample_id_key, - policy="left_outer", - prefix="ann_", - ) - - yaml_state = confluid.dump(paired) - restored = confluid.load(yaml_state) - - original_result = [(s.input, s.meta.get("ann_label"), s.meta["annotated"]) for s in paired] - restored_result = [(s.input, s.meta.get("ann_label"), s.meta["annotated"]) for s in restored] - - assert original_result == restored_result diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..cca78ea --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,78 @@ +"""Tests for the metadata query layer (`sampleflux.storage.query`).""" + +from pathlib import Path + +import numpy as np +import pytest + +from sampleflux.sample import Sample +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.query import MetadataFilterSource, SupportsMetadataScan, scan_hdf5_metadata +from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource + + +class TestMetadataQuery: + def _write_hdf5(self, path: Path) -> Path: + sink = HDF5Sink(path=path) + for i in range(4): + sink.write( + Sample( + input=np.ones(3) * i, + metadata={"snr_db": float(i * 5), "drone": "DJI" if i % 2 else "Parrot", "mask": np.ones((2, 2))}, + ) + ) + sink.flush() + sink.close() + return path + + def test_hdf5_scan_reads_no_arrays(self, tmp_path: Path) -> None: + path = self._write_hdf5(tmp_path / "d.h5") + scanned = list(scan_hdf5_metadata(path)) + assert len(scanned) == 4 + _key, meta = scanned[2] + assert meta["snr_db"] == 10.0 + assert meta["mask"].startswith(" None: + path = self._write_hdf5(tmp_path / "d.h5") + assert isinstance(HDF5Source(path=path), SupportsMetadataScan) + assert isinstance(ZarrGroupSource(path=str(tmp_path / "z")), SupportsMetadataScan) + + def test_filter_source_where_expression_on_hdf5(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + view = MetadataFilterSource(source=source, where="snr_db >= 10") + assert len(view) == 2 + assert [s.meta["snr_db"] for s in view] == [10.0, 15.0] + assert view[0].meta["snr_db"] == 10.0 # random access into matches + + def test_filter_source_string_and_predicate_compose(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + view = MetadataFilterSource(source=source, where="drone == 'DJI'", predicate=lambda m: m["snr_db"] > 5) + assert [s.meta["snr_db"] for s in view] == [15.0] + + def test_missing_key_is_non_matching_not_fatal(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + assert len(MetadataFilterSource(source=source, where="no_such_key > 1")) == 0 + + def test_malformed_expression_fails_loudly(self, tmp_path: Path) -> None: + source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) + with pytest.raises(ValueError, match="failed"): + len(MetadataFilterSource(source=source, where="snr_db +* 2")) + + def test_empty_filter_is_rejected(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="empty filter"): + len(MetadataFilterSource(source=[Sample(1)])) + + def test_fallback_full_iteration_for_plain_sources(self) -> None: + plain = [Sample(input=i, metadata={"v": i}) for i in range(5)] + view = MetadataFilterSource(source=plain, where="v % 2 == 0") + assert [s.input for s in view] == [0, 2, 4] + + def test_zarr_scan_and_filter(self, tmp_path: Path) -> None: + sink = ZarrGroupSink(path=str(tmp_path / "z")) + for i in range(3): + sink.write(Sample(input=np.ones(2) * i, metadata={"v": i})) + sink.flush() + source = ZarrGroupSource(path=str(tmp_path / "z")) + view = MetadataFilterSource(source=source, where="v == 1") + assert len(view) == 1 and view[0].meta["v"] == 1 diff --git a/tests/test_sigmf.py b/tests/test_sigmf.py deleted file mode 100644 index c0be33c..0000000 --- a/tests/test_sigmf.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Tests for the SigMF storage pair (`sampleflux.storage.sigmf`) and the metadata query layer.""" - -import json -from pathlib import Path - -import numpy as np -import pytest - -from sampleflux.sample import Sample -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.query import MetadataFilterSource, SupportsMetadataScan, scan_hdf5_metadata -from sampleflux.storage.sigmf import SigMFSink, SigMFSource -from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource - - -def _iq(n: int = 16, seed: float = 1.0) -> np.ndarray: - return (np.arange(n) * seed + 1j * np.arange(n)).astype(np.complex64) - - -class TestSigMFRoundTrip: - def test_complex64_iq_round_trips(self, tmp_path: Path) -> None: - sink = SigMFSink(path=tmp_path / "recs") - sink.write(Sample(input=_iq(), target=None, metadata={"samplerate": 1e6, "drone": "DJI"})) - sink.write(Sample(input=_iq(seed=2.0), target=3, metadata={"snr_db": 12.5})) - sink.flush() - - source = SigMFSource(path=tmp_path / "recs") - samples = list(source) - assert len(samples) == len(source) == 2 - np.testing.assert_array_equal(samples[0].input, _iq()) - assert samples[0].input.dtype == np.complex64 - assert samples[0].meta["samplerate"] == 1e6 and samples[0].meta["drone"] == "DJI" - assert samples[1].target == 3 # JSON-able target restored - assert source[1].meta["snr_db"] == 12.5 # random access - - def test_float_and_int_dtypes(self, tmp_path: Path) -> None: - for arr in (np.ones(4, dtype=np.float32), np.arange(4, dtype=np.int16)): - sink = SigMFSink(path=tmp_path / str(arr.dtype)) - sink.write(Sample(input=arr, metadata={})) - out = list(SigMFSource(path=tmp_path / str(arr.dtype)))[0] - np.testing.assert_array_equal(out.input, arr) - assert out.input.dtype == arr.dtype - - def test_unsupported_dtype_raises(self, tmp_path: Path) -> None: - sink = SigMFSink(path=tmp_path / "bad") - with pytest.raises(TypeError, match="core:datatype"): - sink.write(Sample(input=np.ones(2, dtype=np.float16), metadata={})) - - def test_meta_file_shape_and_checksum(self, tmp_path: Path) -> None: - sink = SigMFSink(path=tmp_path / "recs", checksum=True) - sink.write(Sample(input=_iq(), metadata={"core:description": "capture"})) - doc = json.loads(next((tmp_path / "recs").glob("*.sigmf-meta")).read_text()) - assert doc["global"]["core:datatype"] == "cf32_le" - assert doc["global"]["core:version"] == "1.0.0" - assert doc["global"]["core:description"] == "capture" # core: keys ride verbatim - assert len(doc["global"]["core:sha512"]) == 128 - assert doc["captures"] == [{"core:sample_start": 0}] - - def test_non_serializable_metadata_skipped_not_fatal(self, tmp_path: Path) -> None: - sink = SigMFSink(path=tmp_path / "recs") - sink.write(Sample(input=_iq(), metadata={"ok": 1, "bad": np.ones(3)})) - out = list(SigMFSource(path=tmp_path / "recs"))[0] - assert out.meta["ok"] == 1 and "bad" not in out.meta - - def test_sink_requires_path(self) -> None: - with pytest.raises(ValueError, match="'path'"): - SigMFSink().write(Sample(input=_iq())) - - def test_source_requires_directory(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="not a directory"): - len(SigMFSource(path=tmp_path / "missing")) - - def test_waivefront_vocab_hooks_round_trip(self, tmp_path: Path) -> None: - meta = { - "samplerate": 2e6, - "center_freq": 2.4e9, - "snr": "clean", - "annotated_regions": [[100.0, 200.0, 0.5, 1.0]], - "annotated_labels": ["wifi"], - "drone": "DJI", - } - sink = SigMFSink(path=tmp_path / "wf", meta_encoder="waivefront.vocab.to_sigmf") - sink.write(Sample(input=_iq(), metadata=meta)) - doc = json.loads(next((tmp_path / "wf").glob("*.sigmf-meta")).read_text()) - assert doc["global"]["core:sample_rate"] == 2e6 - assert doc["captures"][0]["core:frequency"] == 2.4e9 - assert doc["global"]["waivefront:snr_raw"] == "clean" # unparseable snr kept verbatim - annotation = doc["annotations"][0] - assert annotation["core:freq_lower_edge"] == 100.0 - assert annotation["core:sample_start"] == int(0.5 * 2e6) - assert annotation["core:label"] == "wifi" and annotation["waivefront:role"] == "annotated" - - out = list(SigMFSource(path=tmp_path / "wf", meta_decoder="waivefront.vocab.from_sigmf"))[0] - assert out.meta["samplerate"] == 2e6 and out.meta["center_freq"] == 2.4e9 - assert out.meta["snr"] == "clean" and out.meta["drone"] == "DJI" - region = out.meta["annotated_regions"][0] - assert region[0] == 100.0 and region[2] == pytest.approx(0.5) and region[3] == pytest.approx(1.0) - assert out.meta["annotated_labels"] == ["wifi"] - - -class TestMetadataQuery: - def _write_hdf5(self, path: Path) -> Path: - sink = HDF5Sink(path=path) - for i in range(4): - sink.write( - Sample( - input=np.ones(3) * i, - metadata={"snr_db": float(i * 5), "drone": "DJI" if i % 2 else "Parrot", "mask": np.ones((2, 2))}, - ) - ) - sink.flush() - sink.close() - return path - - def test_hdf5_scan_reads_no_arrays(self, tmp_path: Path) -> None: - path = self._write_hdf5(tmp_path / "d.h5") - scanned = list(scan_hdf5_metadata(path)) - assert len(scanned) == 4 - _key, meta = scanned[2] - assert meta["snr_db"] == 10.0 - assert meta["mask"].startswith(" None: - path = self._write_hdf5(tmp_path / "d.h5") - assert isinstance(HDF5Source(path=path), SupportsMetadataScan) - assert isinstance(SigMFSource(path=tmp_path), SupportsMetadataScan) - assert isinstance(ZarrGroupSource(path=str(tmp_path / "z")), SupportsMetadataScan) - - def test_filter_source_where_expression_on_hdf5(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - view = MetadataFilterSource(source=source, where="snr_db >= 10") - assert len(view) == 2 - assert [s.meta["snr_db"] for s in view] == [10.0, 15.0] - assert view[0].meta["snr_db"] == 10.0 # random access into matches - - def test_filter_source_string_and_predicate_compose(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - view = MetadataFilterSource(source=source, where="drone == 'DJI'", predicate=lambda m: m["snr_db"] > 5) - assert [s.meta["snr_db"] for s in view] == [15.0] - - def test_missing_key_is_non_matching_not_fatal(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - assert len(MetadataFilterSource(source=source, where="no_such_key > 1")) == 0 - - def test_malformed_expression_fails_loudly(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - with pytest.raises(ValueError, match="failed"): - len(MetadataFilterSource(source=source, where="snr_db +* 2")) - - def test_empty_filter_is_rejected(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="empty filter"): - len(MetadataFilterSource(source=[Sample(1)])) - - def test_fallback_full_iteration_for_plain_sources(self) -> None: - plain = [Sample(input=i, metadata={"v": i}) for i in range(5)] - view = MetadataFilterSource(source=plain, where="v % 2 == 0") - assert [s.input for s in view] == [0, 2, 4] - - def test_zarr_scan_and_filter(self, tmp_path: Path) -> None: - sink = ZarrGroupSink(path=str(tmp_path / "z")) - for i in range(3): - sink.write(Sample(input=np.ones(2) * i, metadata={"v": i})) - sink.flush() - source = ZarrGroupSource(path=str(tmp_path / "z")) - view = MetadataFilterSource(source=source, where="v == 1") - assert len(view) == 1 and view[0].meta["v"] == 1 - - def test_sigmf_scan_and_filter(self, tmp_path: Path) -> None: - sink = SigMFSink(path=tmp_path / "recs") - for i in range(3): - sink.write(Sample(input=_iq(seed=float(i + 1)), metadata={"snr_db": float(i * 10)})) - source = SigMFSource(path=tmp_path / "recs") - view = MetadataFilterSource(source=source, where="snr_db >= 10") - assert len(view) == 2 - np.testing.assert_array_equal(view[0].input, _iq(seed=2.0)) diff --git a/tests/test_windows.py b/tests/test_windows.py deleted file mode 100644 index 42aa2e2..0000000 --- a/tests/test_windows.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Tests for :mod:`sampleflux.windows` — the window functions + spectral unit-scaling math. - -Pins (1) the closed Literals match their runtime tuples, (2) the pure-numpy windows match -scipy (when available) and have the right correction constants (Hann coherent gain 0.5 / ENBW -1.5 bins, flat-top CG 0.2156), and (3) ``scale_spectrum`` returns calibrated units — a -unit-amplitude tone reads amplitude 1.0 / power 1.0, and the power-to-density ratio is the -window's equivalent noise bandwidth in Hz. -""" - -from typing import cast, get_args - -import numpy as np -import pytest - -from sampleflux import windows as W -from sampleflux.windows import WindowName - - -def test_literals_match_runtime_tuples() -> None: - assert W.WINDOW_NAMES == get_args(W.WindowName) - assert W.SPECTRUM_SCALINGS == get_args(W.SpectrumScaling) - assert "boxcar" in W.WINDOW_NAMES and "hann" in W.WINDOW_NAMES - assert W.SPECTRUM_SCALINGS == ("none", "amplitude", "power", "density") - - -@pytest.mark.parametrize("name", W.WINDOW_NAMES) -def test_get_window_builds_each(name: WindowName) -> None: - wp = {"kaiser": 8.6, "tukey": 0.5, "gaussian": 7.0}.get(name) - w = W.get_window(name, 64, window_param=wp) - assert w.shape == (64,) and w.dtype == np.float64 - assert np.all(np.isfinite(w)) - # boxcar is the rectangular identity; every other taper has a sub-unity mean (it attenuates) - if name == "boxcar": - assert np.allclose(w, 1.0) - else: - assert w.mean() < 1.0 - - -def test_get_window_periodic_differs_from_symmetric() -> None: - p = W.get_window("hann", 64, periodic=True) - s = W.get_window("hann", 64, periodic=False) - assert not np.allclose(p, s) - # symmetric Hann is zero at both endpoints; periodic is zero only at index 0 - assert s[0] == pytest.approx(0.0) and s[-1] == pytest.approx(0.0) - assert p[0] == pytest.approx(0.0) - - -def test_get_window_matches_scipy() -> None: - sw = pytest.importorskip("scipy.signal") - specs = { - "boxcar": "boxcar", - "bartlett": "bartlett", - "hann": "hann", - "hamming": "hamming", - "blackman": "blackman", - "blackmanharris": "blackmanharris", - "nuttall": "nuttall", - "flattop": "flattop", - "kaiser": ("kaiser", 8.6), - "tukey": ("tukey", 0.5), - "gaussian": ("gaussian", 7.0), - } - for name, spec in specs.items(): - wp = {"kaiser": 8.6, "tukey": 0.5, "gaussian": 7.0}.get(name) - for periodic in (True, False): - mine = W.get_window(cast(WindowName, name), 128, window_param=wp, periodic=periodic) - ref = sw.get_window(spec, 128, fftbins=periodic) - assert np.allclose(mine, ref, atol=1e-12), f"{name} periodic={periodic}" - - -def test_get_window_errors() -> None: - with pytest.raises(ValueError, match="unknown window"): - W.get_window("nope", 16) # type: ignore[arg-type] - with pytest.raises(ValueError, match="must be positive"): - W.get_window("hann", 0) - with pytest.raises(ValueError, match="gaussian window requires"): - W.get_window("gaussian", 16) # no window_param - - -def test_correction_constants() -> None: - # asymptotic (large-N) coherent gain + ENBW for the textbook windows - cases = {"boxcar": (1.0, 1.0), "hann": (0.5, 1.5), "hamming": (0.54, 1.363), "flattop": (0.2156, 3.77)} - for name, (cg, enbw) in cases.items(): - w = W.get_window(cast(WindowName, name), 8192) - assert W.coherent_gain(w) == pytest.approx(cg, abs=2e-3) - assert W.enbw_bins(w) == pytest.approx(enbw, abs=2e-2) - - -def test_window_sums_and_metadata() -> None: - w = W.get_window("hann", 1024) - s1, s2 = W.window_sums(w) - assert s1 == pytest.approx(512.0) and s2 == pytest.approx(384.0) - meta = W.window_metadata("hann", w) - assert meta[W.WINDOW_NAME_KEY] == "hann" - assert meta[W.WINDOW_SIZE_KEY] == 1024 - assert meta[W.WINDOW_SUM_KEY] == pytest.approx(512.0) - assert meta[W.WINDOW_SUMSQ_KEY] == pytest.approx(384.0) - assert meta[W.WINDOW_ENBW_KEY] == pytest.approx(1.5) - assert meta[W.WINDOW_CG_KEY] == pytest.approx(0.5) - - -def _tone(n: int, bin_index: int, amp: float = 1.0) -> np.ndarray: - return np.asarray(amp * np.exp(2j * np.pi * bin_index * np.arange(n) / n), dtype=np.complex64) - - -def test_scale_spectrum_none_is_passthrough() -> None: - x = _tone(256, 10) - X = np.fft.fft(x) - out = W.scale_spectrum(X, "none", s1=256.0, s2=256.0) - assert np.array_equal(out, X) - - -def test_scale_spectrum_amplitude_and_power_recover_tone() -> None: - n = 1024 - X = np.fft.fft(_tone(n, 64, amp=1.0)) - amp = W.scale_spectrum(X, "amplitude", s1=float(n), s2=float(n)) - pw = W.scale_spectrum(X, "power", s1=float(n), s2=float(n)) - assert np.abs(amp).max() == pytest.approx(1.0, abs=1e-4) # amplitude V - assert pw.max() == pytest.approx(1.0, abs=1e-4) # power V² = amplitude² - assert np.iscomplexobj(amp) and not np.iscomplexobj(pw) # power is real - - -def test_scale_spectrum_density_is_power_over_enbw_hz() -> None: - n, fs = 1024, 1000.0 - rng = np.random.default_rng(0) - x = (rng.standard_normal(n) + 1j * rng.standard_normal(n)).astype(np.complex64) - w = W.get_window("hann", n) - Xw = np.fft.fft(x * w) - s1, s2 = W.window_sums(w) - pw = W.scale_spectrum(Xw, "power", s1=s1, s2=s2) - den = W.scale_spectrum(Xw, "density", s1=s1, s2=s2, sample_rate=fs) - enbw_hz = fs * s2 / (s1 * s1) # = Fs · ENBW_bins / N - assert np.allclose(pw, den * enbw_hz) # power = density × ENBW_Hz, bin-for-bin - - -def test_scale_spectrum_density_normalized_without_rate() -> None: - X = np.fft.fft(_tone(256, 8)) - den = W.scale_spectrum(X, "density", s1=256.0, s2=256.0) # Fs defaults to 1.0 - den_fs1 = W.scale_spectrum(X, "density", s1=256.0, s2=256.0, sample_rate=1.0) - assert np.allclose(den, den_fs1) - - -def test_fold_one_sided_even_and_odd() -> None: - # real cosine, amplitude 2 at bin 4 → one-sided amplitude reads 2 at that bin - for n in (64, 65): - t = np.arange(n) - x = 2.0 * np.cos(2 * np.pi * 4 * t / n) - amp = W.scale_spectrum(np.fft.fft(x), "amplitude", s1=float(n), s2=float(n), one_sided=True) - assert amp.shape[0] == n // 2 + 1 - assert np.abs(amp).max() == pytest.approx(2.0, abs=1e-6) - - -def test_fold_one_sided_preserves_dc_and_nyquist() -> None: - n = 64 - x = np.ones(n) * 3.0 # pure DC - one = W.fold_one_sided(np.fft.fft(x), axis=-1) - assert one[0] == pytest.approx(3.0 * n) # DC not doubled From fb4ace65250f854b18b7d68fe60a4d7688d1ddc1 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 19 Jul 2026 17:49:47 +0200 Subject: [PATCH 024/102] =?UTF-8?q?feat:=20augmentation=20ops=20=E2=80=94?= =?UTF-8?q?=20albumentations/torchvision=20adapters=20+=20generated=20per-?= =?UTF-8?q?transform=20op=20families?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AlbumentationsOp / TorchvisionTransformOp adapters (joint input+mask/boxes draws, target Literal knob) - auto-generated Alb*/Tv* op families via ops/_augment_bridge (one @configurable op per library transform) - palette groups augment/albumentations + augment/torchvision; [vision] extra for torchvision - docs/augmentation.md + two runnable examples; RandomApply/TransformChain/Enable test updates --- AGENTS.md | 2 + README.md | 2 + docs/augmentation.md | 129 ++++++++ examples/augmentation_ops.py | 140 +++++++++ examples/augmentation_training.py | 129 ++++++++ pyproject.toml | 18 ++ sampleflux/ops/_augment_bridge.py | 202 +++++++++++++ sampleflux/ops/albumentations.py | 199 +++++++++++++ sampleflux/ops/albumentations_transforms.py | 65 ++++ sampleflux/ops/configure.py | 8 +- sampleflux/ops/context.py | 11 +- sampleflux/ops/enable.py | 13 +- sampleflux/ops/parallel.py | 8 +- sampleflux/ops/random_apply.py | 11 +- sampleflux/ops/torchvision.py | 201 +++++++++++++ sampleflux/ops/torchvision_transforms.py | 69 +++++ sampleflux/ops/transform_chain.py | 6 +- tests/test_augment_ops.py | 312 ++++++++++++++++++++ tests/test_categories.py | 15 + tests/test_enable.py | 3 + tests/test_node_docs.py | 4 + tests/test_random_apply.py | 23 +- 22 files changed, 1549 insertions(+), 21 deletions(-) create mode 100644 docs/augmentation.md create mode 100644 examples/augmentation_ops.py create mode 100644 examples/augmentation_training.py create mode 100644 sampleflux/ops/_augment_bridge.py create mode 100644 sampleflux/ops/albumentations.py create mode 100644 sampleflux/ops/albumentations_transforms.py create mode 100644 sampleflux/ops/torchvision.py create mode 100644 sampleflux/ops/torchvision_transforms.py create mode 100644 tests/test_augment_ops.py diff --git a/AGENTS.md b/AGENTS.md index fb17613..780df9e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,8 @@ - **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `target_from`/`metadata_from` (fan-in slots, Mix field semantics), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **The Transform Taxonomy Is a GRID — field scope × call style, with COMBINABLE per-param bindings (`sampleflux.kinds`, 2026-07-18; EXTENDS the introspection mandate below):** A transform declares WHICH slice of the `Sample(input, target, metadata)` triple it processes and HOW it wants to be called, from its `__call__` signature alone. `SampleKind` is now the closed Literal `sample`/`pair`/`input`/`target`/`input_meta`/`target_meta`/`value`/`any`; `CallStyle` is `packed`/`unpacked`; `OpContract` carries both. **Named views are real NamedTuples in `sampleflux.sample`** — `Pair(input, target)`, `InputMeta(input, metadata)`, `TargetMeta(target, metadata)` (metadata typed the same dict-or-list `Metadata` duality as Sample) — recognised by `Sample.from_any` AND `classify_carrier` BEFORE the generic 2-tuple rule (a view IS a tuple; positional coercion would misread `(input, metadata)` as `(input, target)` — pinned). Bare-field intent uses the PEP-593 marks `INPUT`/`TARGET` (aliases `Input = Annotated[Any, INPUT]`, `Target = …`; mark your own type via `Annotated[np.ndarray, INPUT]`). **Detection (`op_contract`) — per-parameter BINDINGS:** arity counts REQUIRED positional params only (optional extras keep single-arg semantics — the back-compat guard). Arity 2–3 → unpacked with `OpContract.bindings`: each param resolves ANNOTATION-first (`InputMeta`/`TargetMeta`/`Input`/`Target` marks, `dict` → `metadata`), then NAME (`input`/`target`/`metadata`/`meta`), then the POSITIONAL default `(input, target, metadata)` — so `f(input, target)`, `f(input, metadata)`, `f(target, metadata)`, `f(input, target, metadata)` reproduce the classic rules AND every combination works: `f(im: InputMeta, tm: TargetMeta)` (input AND target each WITH metadata), `f(x: Input, tm: TargetMeta)`, name-reordered `f(target, input)`, … `accepts` stays the covered-fields grid SUMMARY (`_bindings_summary`: i+t+m→sample, i+m→input_meta, …). Arity 1 → packed, scope from the annotation (`dict` → the new `metadata`-only scope; the `MetaDict` alias/`METADATA` mark exist for explicitness); untyped single stays `any` (NEVER name-sniff a single param — existing ops are untouched). `CALL_STYLE` joins the class-attr escape hatches. **Binding + merge-back:** unpacked ops route through `core._apply_bindings` — each argument bound per its binding (`_BIND_GET`), the result MUST be a same-arity tuple / a full `Sample` / `None` (a wrong arity OR a single named VIEW from a multi-binding op is a LOUD TypeError — a view IS a 2-tuple and would silently misread as two elements, guarded + pinned); each returned element merges per its binding (a view/2-tuple element on a `*_meta` binding replaces value+metadata, a bare element replaces only the value; metadata-bearing elements merge left-to-right, LAST write wins). Packed scopes in `core._apply_view`: `None` drops; a returned `Sample` takes over; `input`/`target` → bare value replaces the field; `metadata` → the dict in, a dict out (else loud error); `pair` → a `Pair` in, a 2-tuple out replaces input+target (metadata KEPT); the meta views receive the ACTUAL metadata dict (in-place mutation propagates). Packed `pair` binds a `Pair` (it IS a tuple, so plain-tuple-annotated ops index it identically while Pair-annotated ops get named fields). **Native fast lanes** (`_apply_op_native`): pair-op on a pair carrier and input-op on a bare value stay metadata-free native; every other non-any combination PROMOTES via the view-correct `from_any` (sticky). Default collates for `input_meta`/`target_meta` mirror the pair form (stacked value + list-of-dicts metadata). These NAMES are the vocabulary FluxStudio will surface as socket types (a later stage — TASKS.md). Pins: `tests/test_kinds.py::TestGridContracts`/`TestGridEngineBinding`. +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19):** Every op that wraps/applies OTHER ops — `TransformChain`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `sampleflux.core._apply_op(sample, op)`, NEVER `op(sample)` directly. `_apply_op` is the engine's single contract-aware chokepoint: it introspects the inner op's transform-taxonomy contract (`op_contract`) and binds the declared view (pair / input / target / `*_meta`, packed or unpacked), so a field-scoped op (e.g. a pair-scoped augmentation adapter) nests inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(sample)` call crashes on any non-sample-scoped op with a misleading "missing positional argument" `TypeError`. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Sample]`. Pins: `tests/test_augment_ops.py` (pair op inside `RandomApply`/`TransformChain`/`Enable`). +- **Augmentation = Adapter Ops + GENERATED Per-Transform Families, Never Reimplement (`sampleflux.ops.albumentations` / `.torchvision` / `.albumentations_transforms` / `.torchvision_transforms` / `._augment_bridge`, 2026-07-19):** Library augmentation enters the pipeline through two SAMPLE-scoped adapter ops — `AlbumentationsOp` (albumentations, numpy HWC in/out, core dep) and `TorchvisionTransformOp` (torchvision `transforms.v2`, CHW tensors out, `[vision]` extra, ALL torchvision imports lazy in the ADAPTER module so it imports without the library — pinned by `test_module_imports_without_torchvision`) — plus the AUTO-GENERATED per-transform op families: `sampleflux.ops._augment_bridge.generate_transform_ops` (the waivefront-helios auto-bridge pattern) walks each library's public transform classes at import time and emits one adapter SUBCLASS per transform (`Alb` ~115 ops, group `augment/albumentations`; `Tv` ~55 ops, group `augment/torchvision`) with a synthesized `__signature__`/`__annotations__`/spliced `Args:` docstring (transform params + `target`/`seed` appended LAST), the base `__call__` RE-STATED in the class dict (canvas op-classification reads `vars(cls)` — inherited-only methods are invisible), and a `raw_transform` property an adapter's `transforms` list UNWRAPS (so canvas transform nodes dock into a Compose-style adapter node). The `Alb`/`Tv` prefixes are MANDATORY (confluid's registry is flat + name-keyed; the libraries share bare names like `ColorJitter`/`Normalize`/`Resize` — the helios `Helios*` precedent); composition/container transforms are NOT generated (chaining is native); per-class generation failures skip with a DEBUG note, never break import; `torchvision_transforms` imports safely without torchvision (zero ops). Adapters: `@configurable(category="op", group="augment", random=True)`, `__call__(self, sample: Sample)` — SAMPLE-scoped deliberately, because the visual-canvas op classifier only recognises `__call__(sample: Sample)` and invokes `op(sample)` directly (pair-scoped ops are engine-legal but canvas-invisible until the kinds-grid socket stage lands); ONE library draw still moves input AND target jointly per the closed `TargetMode = Literal["none","mask","boxes"]` knob (`"boxes"` consumes the torchvision detection dict from `CocoToTorchVisionDetectionOp`/`MasksToDetectionBoxesOp`; albumentations `bbox_params` are AUTO-ADDED when the op composes — only a prebuilt `A.Compose` must carry its own, validated loudly). Config surface: `transform` (ONE transform / prebuilt Compose) XOR `transforms` (list, composed lazily; entries may be live objects, Confluid markers — flowed lazily — or generated ops); `seed` on the albumentations side maps onto `A.Compose(seed=...)` (rejected with a prebuilt Compose); NO probability knobs (gating is `RandomApply`). **YAML is Confluid-NATIVE ONLY** — nested `!class:albumentations.HorizontalFlip` / `!class:torchvision.transforms.v2.X` nodes or registered short names (`!class:AlbHorizontalFlip`), dump→load round-trips both (the engine captures foreign-class ctor kwargs); the earlier `A.to_dict()` dict-spec surface was REMOVED (user-rejected — never resurrect a library-specific serialization format as config). Don't add a third adapter without real demand, and never bake a specific augmentation as a bespoke hand-written sampleflux op — wrap the library or use the generated family. Docs: `docs/augmentation.md`; examples: `examples/augmentation_ops.py` / `examples/augmentation_training.py`; pins: `tests/test_augment_ops.py`, `tests/test_categories.py`. - **Op Kind Is INTROSPECTED, Never Declared in the Engine (`sampleflux.kinds`, 2026-07-17):** The native multi-type engine (`Flux(native=True)`, OPT-IN — the `native=False` default coerces to `Sample` exactly as before, so all consumers are untouched) carries `Sample` triplets, metadata-free **pairs** (2-tuples), and bare **values** through one pipeline, adapting each op via `op_contract(op)` → `OpContract(accepts, produces, expands)` cached per type: `__call__`'s first-param annotation (`Sample`→`sample`, `tuple[...]`→`pair`, missing/`Any`→`any`) and return annotation (`Iterator[...]`/`Iterable[...]`/`List[...]` → `expands=True` — a `Tuple` return is a PAIR, never an expansion). ANY introspection failure (lazy imports, unresolvable forward refs) degrades to `any` so an untyped/exotic op behaves exactly as today; the class attrs `SAMPLE_KIND_IN`/`SAMPLE_KIND_OUT`/`EXPANDS` are the explicit escape hatch and ALWAYS win. Adaptation rules (`core._apply_op_native`): a pair-op on a Sample gets `sample.to_pair()` and its returned pair merges back via `_replace` (METADATA PRESERVED); a sample-op on a pair/value gets a PROMOTED `Sample.from_any` view — promotion is one-way and STICKY (op-written metadata is never dropped); an any-op gets the carrier verbatim. `SampleKind` is a closed Literal (`sample`/`pair`/`value`/`any`; runtime tuple `SAMPLE_KINDS = get_args(...)` — one source of truth); `classify_carrier` is the runtime classifier (ONLY a 2-tuple is a pair). **Collation is the pluggable registry `sampleflux.collate`** (`register_collate(key)` / `get_collate` / `collate(items, key=None)` — key defaults to the detected kind): sampleflux registers `"sample"` (list-form batched metadata — the `is_batched` convention) / `"pair"` / `"value"` defaults; consumers register task aliases ADDITIVELY and their divergent conventions (deltaid/raidar `{"per_sample": …}`) are deliberately NOT unified (TASKS.md follow-up). Pins: `tests/test_kinds.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op whose return annotation is `Iterator[...]`/`Iterable[...]`/`List[...]` (or that carries `EXPANDS = True`) is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op's contract expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `HDF5WindowSource`/`RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. `_refresh_type` is applied per CHILD (`core._expand`). Pins: `tests/test_expanding_ops.py`. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. diff --git a/README.md b/README.md index 8cebd39..0622020 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ for sample in flux: | [docs/projection.md](docs/projection.md) | Field projection (`SupportsProjection`), lazy target walks, `num_classes`, the fittable `LabelMap` | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImageOp`, `NormalizeToUint8Op`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | +| [docs/augmentation.md](docs/augmentation.md) | Augmentation via albumentations / torchvision `transforms.v2` — joint input+target (mask/boxes) adapters, the generated `Alb*`/`Tv*` per-transform ops, seeding, Confluid-native YAML | ## 🧭 Scope: a modality-neutral engine @@ -64,6 +65,7 @@ SampleFlux is designed to sit between your data catalog and your training loop, - **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into `Sample` triplets with full metadata traceability (see [docs/sources.md](docs/sources.md)). - **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node, every run reproducible. - **PyTorch**: `Flux` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#multi-type-carriers--the-collate-registry-samplefluxcollate). +- **Augmentation libraries**: `AlbumentationsOp` / `TorchvisionTransformOp` wrap [albumentations](https://albumentations.ai) and torchvision `transforms.v2` as ops that augment input AND target (mask / detection boxes) jointly — plus an auto-generated op per individual library transform (`AlbHorizontalFlip`, `TvColorJitter`, …), each a graph node and a Confluid `!class:` one-liner (see [docs/augmentation.md](docs/augmentation.md); torchvision via `pip install "sampleflux[vision]"`). ## 🔧 Installation diff --git a/docs/augmentation.md b/docs/augmentation.md new file mode 100644 index 0000000..7d0addc --- /dev/null +++ b/docs/augmentation.md @@ -0,0 +1,129 @@ +# Augmentation — well-known libraries as SampleFlux ops + +SampleFlux does not reimplement augmentations. Two adapter ops wrap the established +libraries — and a **generated op family** turns every individual library transform into +its own first-class op: + +| Surface | What it is | Example | +|---|---|---| +| `sampleflux.ops.albumentations.AlbumentationsOp` | Adapter running one/many [albumentations](https://albumentations.ai) transforms | `AlbumentationsOp(transforms=[...], target="mask", seed=0)` | +| `sampleflux.ops.torchvision.TorchvisionTransformOp` | Adapter running one/many torchvision `transforms.v2` transforms | `TorchvisionTransformOp(transforms=[...], target="mask")` | +| `sampleflux.ops.albumentations_transforms` | **Auto-generated**: one `Alb` op per albumentations transform (~115) | `AlbHorizontalFlip(p=0.5, target="mask")` | +| `sampleflux.ops.torchvision_transforms` | **Auto-generated**: one `Tv` op per v2 transform (~55) | `TvRandomHorizontalFlip(p=0.5, target="mask")` | + +All are ordinary sample-scoped ops (`__call__(sample)`): they chain in a `Flux` ops list, +inside `TransformChain` / `RandomApply` / `Enable`, in Confluid YAML, and as individual +nodes on a visual canvas (palette groups `augment`, `augment/albumentations`, +`augment/torchvision`). One library draw applies jointly to `sample.input` and — per the +`target` mode — its mask / boxes; metadata passes through untouched. + +Torchvision requires the `vision` extra: `pip install "sampleflux[vision]"` +(albumentations is a core dependency; without torchvision the `Tv*` family is simply +empty and everything else works). + +## Target modes + +The `target` knob is a closed `Literal["none", "mask", "boxes"]` on every op above: + +- `"none"` (default) — input-only augmentation (color jitter, noise, blur); the sample's + target passes through untouched. +- `"mask"` — `sample.target` is a segmentation mask (2-D array or PIL `L` image); image + and mask receive the SAME spatial transform. +- `"boxes"` — `sample.target` is the torchvision detection dict + `{"boxes": [N,4] xyxy-pixel, "labels": [N]}` — exactly what `CocoToTorchVisionDetectionOp` + and `MasksToDetectionBoxesOp` emit — and boxes move with the image. The required + albumentations `bbox_params` are added automatically when the op builds the Compose; + only a prebuilt `A.Compose` must carry its own. + +```python +import albumentations as A +from sampleflux import Flux +from sampleflux.ops.albumentations import AlbumentationsOp +from sampleflux.ops.albumentations_transforms import AlbRandomBrightnessContrast + +flux = Flux(source=samples, ops=[ + AlbumentationsOp( # several transforms, one op + transforms=[A.HorizontalFlip(p=0.5), A.Affine(translate_percent=0.1, p=1.0)], + target="mask", seed=0, + ), + AlbRandomBrightnessContrast(p=0.5), # or one generated op per transform +]) +``` + +## YAML — Confluid-native, both directions + +Transforms are ordinary nested `!class:` nodes (dotted paths or registered short names) — +no library-specific serialization formats. `confluid.dump` round-trips both forms. + +```yaml +# Adapter with a transforms list (dotted library paths): +- !class:sampleflux.ops.albumentations.AlbumentationsOp + target: mask + seed: 0 + transforms: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.Affine + translate_percent: 0.1 + +# Generated per-transform ops (registered short names): +- !class:AlbHorizontalFlip + p: 0.5 + target: mask +- !class:TvRandomHorizontalFlip + p: 0.5 + target: mask +``` + +## The generated op families + +`sampleflux.ops._augment_bridge` walks each library's public transform classes at import +time and generates one op per transform (the waivefront-helios auto-bridge pattern): a +subclass of the adapter whose constructor mirrors the transform's own parameters (plus +`target` / `seed`), with a synthesized signature and `Args:` docstring so form-specs, +MCP schemas, and canvas widgets see the real parameters. + +- The `Alb` / `Tv` name prefixes are MANDATORY: the confluid registry is flat and + name-keyed, and the two libraries share many bare names (`ColorJitter`, `Normalize`, + `Resize`, …). +- Zero-arg construction always works; a transform's required parameter (e.g. + `AlbRandomCrop.height`) surfaces lazily as the library's own missing-argument error on + first call. +- Composition/container transforms (`Compose`, `OneOf`, v2 `RandomApply`, …) are NOT + generated — chaining ops is native SampleFlux (`ops:` lists, `TransformChain`, + `RandomApply`). +- A generated op wired into an adapter's `transforms` list unwraps to its inner library + transform (`raw_transform`), so canvas graphs can feed transform nodes into one + Compose-style adapter node too. + +## Layout contract (the main footgun) + +The two libraries disagree about layout, and the ops keep each library's native +convention instead of hiding it: + +- **albumentations** (`AlbumentationsOp`, `Alb*`) consumes numpy **HWC** (PIL converts on + entry) and emits numpy HWC — put it BEFORE `ToTensorOp` in the chain. +- **torchvision** (`TorchvisionTransformOp`, `Tv*`) emits **CHW torch tensors** (numpy + HWC converts on entry, PIL passes through as PIL) — no `ToTensorOp` needed after it. + +Don't chain one library's output straight into the other without accounting for this. + +## Randomness & seeding + +All augmentation ops carry `random=True` (the confluid stochastic mark). Stochasticity +lives where each library puts it: + +- albumentations: the `seed` knob (maps onto `A.Compose(seed=N)`); a prebuilt + `A.Compose` carries its own seed instead. +- torchvision v2: the global torch RNG — `torch.manual_seed(N)`. +- per-sample gating: wrap in `RandomApply(op=..., probability=..., random_state=N)` + (each albumentations transform also carries its own `p`). + +## Examples + +- [`examples/augmentation_ops.py`](../examples/augmentation_ops.py) — the tour: all + three target modes, cross-library parity, boxes mirroring, target-side encoding, the + generated op families, gated composition, and the Confluid-native YAML round-trip. +- [`examples/augmentation_training.py`](../examples/augmentation_training.py) — end to + end: synthetic images+masks → joint geometric + gated photometric augmentation → + `DataLoader` (registry collate) → a tiny CNN trained for 3 epochs with improving loss. diff --git a/examples/augmentation_ops.py b/examples/augmentation_ops.py new file mode 100644 index 0000000..2cbd80b --- /dev/null +++ b/examples/augmentation_ops.py @@ -0,0 +1,140 @@ +"""Augmentation adapters: well-known libraries as SampleFlux ops, for input AND target. + +Demonstrates the library-augmentation surface: +1. input-only augmentation — ``AlbumentationsOp`` flips the image, target untouched; +2. joint input+target — ``target="mask"`` flips image AND segmentation mask consistently + (one library draw, metadata preserved); +3. the same joint flip through torchvision ``transforms.v2`` — cross-library parity; +4. detection boxes — ``MasksToDetectionBoxesOp`` derives xyxy boxes from the mask, then + both adapters transform image AND boxes jointly (``target="boxes"``, bbox_params + added automatically); +5. target-side augmentation/encoding — ``MetadataToTargetOp`` + ``EncodeTargetOp`` turn a + raw metadata label into the supervised class id; +6. the GENERATED per-transform ops — every library transform is its own op + (``AlbHorizontalFlip``, ``TvRandomHorizontalFlip``, …) chaining like any other op; +7. stochastic composition — ``TransformChain(RandomApply(AlbRandomBrightnessContrast))`` + gated per sample; +8. Confluid-NATIVE YAML — nested ``!class:albumentations.HorizontalFlip`` nodes and the + registered short names (``!class:AlbHorizontalFlip``), with dump→load parity. + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import albumentations as A +import confluid # type: ignore[import-not-found] +import numpy as np +import torch +from torchvision.transforms import v2 + +from sampleflux import Flux, Sample +from sampleflux.ops.albumentations import AlbumentationsOp +from sampleflux.ops.albumentations_transforms import AlbHorizontalFlip, AlbRandomBrightnessContrast +from sampleflux.ops.random_apply import RandomApply +from sampleflux.ops.target import EncodeTargetOp, MasksToDetectionBoxesOp, MetadataToTargetOp +from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.torchvision import TorchvisionTransformOp +from sampleflux.ops.torchvision_transforms import TvRandomHorizontalFlip +from sampleflux.ops.transform_chain import TransformChain + + +def make_sample() -> Sample: + """A deterministic 48x48 RGB gradient with a bright square and its binary mask.""" + height = width = 48 + image = np.linspace(0, 200, height * width * 3, dtype=np.float64).reshape(height, width, 3) + image = image.astype(np.uint8) + mask = np.zeros((height, width), dtype=np.uint8) + image[8:20, 4:16] = 255 # bright square, deliberately OFF-center so a flip moves it + mask[8:20, 4:16] = 1 + return Sample(image, mask, {"label": "square", "idx": 0}) + + +def main() -> None: + sample = make_sample() + image, mask = sample.input, sample.target + + # 1. Input-only augmentation: the target and metadata pass through untouched. + out = list(Flux(source=[sample], ops=[AlbumentationsOp(A.HorizontalFlip(p=1.0))]))[0] + assert np.array_equal(out.input, image[:, ::-1]) + assert np.array_equal(out.target, mask) + assert out.meta == sample.meta + print("1. albumentations input-only: image flipped, mask + metadata untouched") + + # 2. Joint input+target: ONE random draw moves image AND mask together; metadata + # survives verbatim. + out_alb = list(Flux(source=[sample], ops=[AlbumentationsOp(A.HorizontalFlip(p=1.0), target="mask")]))[0] + assert np.array_equal(out_alb.input, image[:, ::-1]) + assert np.array_equal(out_alb.target, mask[:, ::-1]) + assert out_alb.meta == sample.meta + print("2. albumentations target='mask': image AND mask flipped consistently") + + # 3. Same augmentation via torchvision transforms.v2 — identical pixels, different + # layout contract (torchvision emits CHW tensors; albumentations stays HWC numpy). + tv_op = TorchvisionTransformOp(v2.RandomHorizontalFlip(p=1.0), target="mask") + out_tv = list(Flux(source=[sample], ops=[tv_op]))[0] + assert np.array_equal(out_tv.input.permute(1, 2, 0).numpy(), out_alb.input) + assert np.array_equal(out_tv.target.numpy(), out_alb.target) + print("3. torchvision target='mask': cross-library parity (CHW tensor out)") + + # 4. Detection boxes: derive {"boxes" xyxy, "labels"} from the mask, then flip image + # AND boxes jointly. The adapter adds the required bbox_params automatically. + det = MasksToDetectionBoxesOp()(sample) + (x0, y0, x1, y1) = det.target["boxes"][0].tolist() + mirrored = [det.input.shape[1] - x1, y0, det.input.shape[1] - x0, y1] + out_tvb = list(Flux(source=[det], ops=[TorchvisionTransformOp(v2.RandomHorizontalFlip(p=1.0), target="boxes")]))[0] + assert out_tvb.target["boxes"][0].tolist() == mirrored + out_albb = list(Flux(source=[det], ops=[AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="boxes")]))[0] + assert np.allclose(out_albb.target["boxes"][0].tolist(), mirrored, atol=1e-4) + print(f"4. target='boxes': {[x0, y0, x1, y1]} -> {mirrored} (both libraries agree)") + + # 5. Target-side augmentation/encoding: raw label from metadata -> supervised class id. + encode = [MetadataToTargetOp(key="label"), EncodeTargetOp(mapping={"square": 0, "disc": 1})] + out_enc = list(Flux(source=[sample], ops=encode))[0] + assert out_enc.target == 0 + print("5. MetadataToTargetOp + EncodeTargetOp: metadata['label'] -> class id 0") + + # 6. Generated per-transform ops: every library transform is its OWN op — no wrapper + # boilerplate, the transform's params are the op's params, and it chains anywhere. + out_gen = list(Flux(source=[sample], ops=[AlbHorizontalFlip(p=1.0, target="mask")]))[0] + assert np.array_equal(out_gen.target, mask[:, ::-1]) + out_gen_tv = list(Flux(source=[sample], ops=[TvRandomHorizontalFlip(p=1.0, target="mask")]))[0] + assert np.array_equal(out_gen_tv.target.numpy(), mask[:, ::-1]) + print("6. generated ops: AlbHorizontalFlip / TvRandomHorizontalFlip chain like any op") + + # 7. Stochastic composition: gate a generated photometric op per sample, tensorize. + chain = TransformChain( + ops=[ + RandomApply(op=AlbHorizontalFlip(p=1.0, target="mask"), probability=0.5, random_state=0), + AlbRandomBrightnessContrast(p=1.0, seed=0), + ToTensorOp(), + ] + ) + source = [Sample(image.copy(), mask.copy(), {"idx": i}) for i in range(8)] + outputs = list(Flux(source=source, ops=[chain])) + flipped = sum(1 for s in outputs if np.array_equal(s.target, mask[:, ::-1])) + assert all(isinstance(s.input, torch.Tensor) and s.input.shape == (3, 48, 48) for s in outputs) + assert 0 < flipped < len(outputs) # the gate fired for some samples, not all + print(f"7. TransformChain(RandomApply(flip), brightness, ToTensorOp): {flipped}/{len(outputs)} flipped") + + # 8. Confluid-NATIVE YAML: transforms are nested !class: nodes (or registered short + # names) — dump and load round-trip with identical behavior. + yaml_text = ( + "!class:sampleflux.ops.albumentations.AlbumentationsOp\n" + "target: mask\n" + "seed: 0\n" + "transforms:\n" + " - !class:albumentations.HorizontalFlip\n" + " p: 1.0\n" + ) + op = confluid.load(yaml_text) + short = confluid.load("!class:AlbHorizontalFlip\np: 1.0\ntarget: mask\n") + out_yaml = op(sample) + out_short = short(sample) + assert np.array_equal(out_yaml.target, out_short.target) + reloaded = confluid.load(confluid.dump(op)) + assert np.array_equal(reloaded(sample).input, out_yaml.input) + print("8. Confluid-native YAML (nested !class: + short names), dump->load parity:") + print(" " + "\n ".join(confluid.dump(op).strip().splitlines())) + + +if __name__ == "__main__": + main() diff --git a/examples/augmentation_training.py b/examples/augmentation_training.py new file mode 100644 index 0000000..8bc9420 --- /dev/null +++ b/examples/augmentation_training.py @@ -0,0 +1,129 @@ +"""Train a tiny segmentation CNN on an augmented SampleFlux pipeline (end to end). + +Demonstrates the full "augment to train" story: +1. a synthetic, network-free dataset — 64 RGB images with a bright square or disc and + its binary segmentation mask (input AND target); +2. joint geometric augmentation — ``AlbumentationsOp(target="mask")`` flips/translates + image AND mask with one library draw per sample; +3. gated photometric augmentation — ``RandomApply`` fires brightness/contrast on the + input only, for half the samples; +4. tensorization — ``ToTensorOp`` for the image, a raw-callable ``.map(select="target")`` + for the mask (``WrappedOp`` under the hood); +5. ``Flux`` is a ``torch.utils.data.Dataset`` — it plugs straight into a ``DataLoader`` + with the registry collate (``get_collate("sample")``: stacked tensors + list-form + batched metadata), and augmentations re-draw every epoch via random access; +6. a 3-epoch training loop of a tiny CNN (BCE on the mask) — losses must be finite and + improve, proving gradients flow through the augmented pipeline. + +Standalone, zero-arg, exit 0, seconds on CPU (CI runs every ``examples/*.py``). +""" + +import math +from typing import List + +import albumentations as A +import numpy as np +import torch +from torch import nn +from torch.utils.data import DataLoader + +from sampleflux import Flux, Sample +from sampleflux.collate import get_collate +from sampleflux.ops.albumentations import AlbumentationsOp +from sampleflux.ops.albumentations_transforms import AlbRandomBrightnessContrast +from sampleflux.ops.random_apply import RandomApply +from sampleflux.ops.torch import ToTensorOp + +SIZE = 32 # image edge in pixels + + +def make_dataset(count: int = 64, seed: int = 0) -> List[Sample]: + """Synthetic segmentation set: noisy background + one bright square or disc + its mask.""" + rng = np.random.default_rng(seed) + samples = [] + for idx in range(count): + image = rng.integers(0, 60, size=(SIZE, SIZE, 3), dtype=np.uint8) + mask = np.zeros((SIZE, SIZE), dtype=np.uint8) + cy, cx = rng.integers(8, SIZE - 8, size=2) + r = int(rng.integers(3, 7)) + shape = "square" if idx % 2 == 0 else "disc" + if shape == "square": + region = np.zeros((SIZE, SIZE), dtype=bool) + region[cy - r : cy + r, cx - r : cx + r] = True + else: + yy, xx = np.ogrid[:SIZE, :SIZE] + region = (yy - cy) ** 2 + (xx - cx) ** 2 <= r**2 + image[region] = rng.integers(180, 255, size=3, dtype=np.uint8) + mask[region] = 1 + samples.append(Sample(image, mask, {"idx": idx, "shape": shape})) + return samples + + +def mask_to_float(mask: np.ndarray) -> torch.Tensor: + """Binary HxW mask -> float32 (1, H, W) tensor, the shape BCEWithLogitsLoss expects.""" + return torch.from_numpy(np.ascontiguousarray(mask)).float().unsqueeze(0) + + +def build_pipeline(samples: List[Sample]) -> Flux: + """Source -> joint geometric aug -> gated photometric aug -> tensors, all seeded.""" + geometric = AlbumentationsOp( # image AND mask move together (one draw per sample) + transforms=[A.HorizontalFlip(p=0.5), A.Affine(translate_percent=0.1, p=1.0)], + target="mask", + seed=0, + ) + photometric = AlbRandomBrightnessContrast(p=1.0, seed=1) # generated per-transform op + flux = Flux( + source=samples, + ops=[ + geometric, + RandomApply(op=photometric, probability=0.5, random_state=0), + ToTensorOp(), # image -> float CHW in [0, 1] + ], + ) + return flux.map(mask_to_float, select="target") # raw-callable target map (WrappedOp) + + +def main() -> None: + torch.manual_seed(0) + samples = make_dataset() + flux = build_pipeline(samples) + + # Flux implements the torch Dataset protocol; the registry collate stacks + # input/target and keeps per-sample metadata as a list (Sample.is_batched). + loader = DataLoader(flux, batch_size=8, shuffle=True, collate_fn=get_collate("sample")) + + batch = next(iter(loader)) + assert batch.input.shape == (8, 3, SIZE, SIZE) and batch.input.dtype == torch.float32 + assert batch.target.shape == (8, 1, SIZE, SIZE) and batch.target.dtype == torch.float32 + assert batch.is_batched and len(batch.batch_meta) == 8 + print(f"batch: input {tuple(batch.input.shape)}, target {tuple(batch.target.shape)}") + print(f" metadata (first 3): {batch.batch_meta[:3]}") + + model = nn.Sequential( + nn.Conv2d(3, 8, kernel_size=3, padding=1), + nn.ReLU(), + nn.Conv2d(8, 1, kernel_size=3, padding=1), + ) + optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) + criterion = nn.BCEWithLogitsLoss() + + epoch_means = [] + for epoch in range(3): + losses = [] + for batch in loader: # augmentations re-draw here: each epoch sees new variants + optimizer.zero_grad() + loss = criterion(model(batch.input), batch.target) + loss.backward() + optimizer.step() + losses.append(float(loss.detach())) + mean = sum(losses) / len(losses) + epoch_means.append(mean) + print(f"epoch {epoch}: mean loss {mean:.4f}") + + assert all(math.isfinite(v) for v in epoch_means) + assert epoch_means[-1] < epoch_means[0], f"loss did not improve: {epoch_means}" + print(f"loss improved {epoch_means[0]:.4f} -> {epoch_means[-1]:.4f} on augmented data") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index d5dea04..ed17d62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ dev = [ "mypy>=1.0.0,<2.0.0", "pytest>=7.0.0,<9.0.0", "pytest-cov>=4.0.0,<7.0.0", + # CI installs `.[dev]` (unit-tests AND verify-examples), so torchvision here keeps the + # torchvision augmentation adapter + examples exercised in CI without a workflow edit. + "torchvision", ] notebook = [ "matplotlib", @@ -38,6 +41,9 @@ notebook = [ ] vision = [ "scipy", + # TorchvisionTransformOp (sampleflux.ops.torchvision) lazy-imports torchvision; the + # extra makes `pip install sampleflux[vision]` the documented way to enable it. + "torchvision", ] [build-system] @@ -74,6 +80,16 @@ sampleflux-ops-copy = "sampleflux.ops.copy" sampleflux-ops-swap = "sampleflux.ops.swap" sampleflux-ops-target = "sampleflux.ops.target" sampleflux-ops-image = "sampleflux.ops.image" +# Augmentation adapters over well-known libraries (pair-scoped input+target ops). +# Both modules import WITHOUT their library (lazy imports) so discovery stays safe on +# hosts missing torchvision. Entry-point changes need an editable reinstall +# (`aisland setup`, never --reinstall) before FluxStudio/navigaitor discovery sees them. +sampleflux-ops-albumentations = "sampleflux.ops.albumentations" +sampleflux-ops-torchvision = "sampleflux.ops.torchvision" +# The auto-generated per-transform op families (Alb / Tv) — one node per +# library transform, generated at import time by sampleflux.ops._augment_bridge. +sampleflux-ops-albumentations-transforms = "sampleflux.ops.albumentations_transforms" +sampleflux-ops-torchvision-transforms = "sampleflux.ops.torchvision_transforms" # DropMetadataOp (strip metadata keys — e.g. the __taidal_stash_* snapshots) + PrintSampleOp # (log/print a per-sample summary). Entry-point changes need an editable reinstall before # FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). @@ -109,9 +125,11 @@ python_files = ["test_*.py"] [[tool.mypy.overrides]] module = [ + "albumentations.*", "datasets.*", "h5py.*", "PIL.*", + "torchvision.*", "zarr.*", ] ignore_missing_imports = true diff --git a/sampleflux/ops/_augment_bridge.py b/sampleflux/ops/_augment_bridge.py new file mode 100644 index 0000000..bbcf447 --- /dev/null +++ b/sampleflux/ops/_augment_bridge.py @@ -0,0 +1,202 @@ +"""Shared generator for the per-transform augmentation op families (``Alb*`` / ``Tv*``). + +Mirrors the waivefront-helios transform auto-bridge: walk a library's public transform +classes and generate ONE ``@configurable`` SampleFlux op per transform — a subclass of the +library's adapter op (:class:`~sampleflux.ops.albumentations.AlbumentationsOp` / +:class:`~sampleflux.ops.torchvision.TorchvisionTransformOp`) whose constructor mirrors the +transform's own parameters (plus the adapter's ``target`` / ``seed`` knobs). Each +generated op: + +* is a normal sample-scoped op (``__call__(sample)``) — it chains in a ``Flux`` ops list, + a ``TransformChain`` / ``RandomApply``, a Confluid YAML (``!class:AlbHorizontalFlip``), + or a visual canvas exactly like any hand-written op; +* exposes ``raw_transform`` — the configured library transform instance — so an adapter + op's ``transforms`` list unwraps a wired generated op back to the library object; +* carries a synthesized ``__signature__`` / ``__annotations__`` / ``Args:`` docstring so + static introspection (``to_pydantic`` → form-specs / MCP schemas, ``parse_param_docs`` + → widget tooltips) sees the transform's real parameters. + +The mandatory name prefix (``Alb`` / ``Tv``) keeps confluid's flat, name-keyed registry +collision-free — albumentations and torchvision share many bare names (``ColorJitter``, +``Normalize``, ``Resize``, …). +""" + +import inspect +from typing import Any, Iterable, List, Optional, Tuple, Type + +from confluid import configurable +from loggair import get_logger + +from sampleflux.ops.albumentations import TargetMode + +logger = get_logger(__name__) + +#: Adapter-owned constructor names — a library transform whose ctor collides is skipped. +_RESERVED = frozenset({"target", "seed", "transform", "transforms"}) + +_TARGET_DOC = " target: Joint-augmentation mode — ``none`` (input-only, default), ``mask``, or ``boxes``." +_SEED_DOC = " seed: Compose seed for deterministic draws. ``None`` = non-deterministic (default)." + + +def _param_specs(transform_cls: type) -> List[inspect.Parameter]: + """The transform constructor's named parameters (``self`` and variadics dropped).""" + sig = inspect.signature(transform_cls.__init__) + return [ + p + for n, p in sig.parameters.items() + if n != "self" and p.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) + ] + + +def _synth_doc(name: str, transform_cls: type, seed_param: bool) -> str: + """The generated op's docstring: the library docstring with the adapter params spliced in. + + ``parse_param_docs`` reads the (first) Google-style ``Args:`` block, so the adapter's + ``target`` / ``seed`` lines are inserted right after the library's own ``Args:`` + heading — or a fresh block is appended when the library docstring has none. + """ + extra = [_TARGET_DOC] + ([_SEED_DOC] if seed_param else []) + lib_doc = inspect.getdoc(transform_cls) or f"{name} transform (see the library documentation)." + lines = lib_doc.splitlines() + if any(ln.strip() == "Args:" for ln in lines): + out: List[str] = [] + for ln in lines: + out.append(ln) + if ln.strip() == "Args:": + out.extend(extra) + return "\n".join(out) + return lib_doc + "\n\nArgs:\n" + "\n".join(extra) + + +def _make_op( + name: str, + transform_cls: type, + *, + base: type, + prefix: str, + group: str, + module_name: str, + seed_param: bool, +) -> type: + """One generated op class wrapping ``transform_cls`` (see the module docstring).""" + specs = _param_specs(transform_cls) + pnames = [p.name for p in specs] + clash = _RESERVED & set(pnames) + if clash: + raise ValueError(f"constructor params clash with adapter params: {sorted(clash)}") + defaults = {p.name: (None if p.default is inspect.Parameter.empty else p.default) for p in specs} + required = {p.name for p in specs if p.default is inspect.Parameter.empty} + op_name = f"{prefix}{name}" + allowed = set(pnames) | {"target"} | ({"seed"} if seed_param else set()) + + def __init__(self: Any, **kwargs: Any) -> None: + # Lazy / zero-arg: store config only (required transform params default to None and + # surface lazily via the library's own missing-argument error on first call). + unknown = set(kwargs) - allowed + if unknown: + raise TypeError(f"{op_name}: unexpected parameters {sorted(unknown)}") + if seed_param: + base.__init__(self, target=kwargs.get("target", "none"), seed=kwargs.get("seed")) + else: + base.__init__(self, target=kwargs.get("target", "none")) + for pname in pnames: + setattr(self, pname, kwargs.get(pname, defaults[pname])) + self._params_key: Optional[tuple] = None + + # Synthesized signature/annotations: static introspection (to_pydantic / FluxStudio + # widgets / parse_param_docs) sees the transform's real parameters, keyword-only, with + # the adapter's target/seed appended LAST. Required params are defaulted to None so + # zero-arg construction always works (the workspace lazy-init mandate). + sig_params = [inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD)] + for p in specs: + sig_params.append(p.replace(kind=inspect.Parameter.KEYWORD_ONLY, default=defaults[p.name])) + sig_params.append( + inspect.Parameter("target", inspect.Parameter.KEYWORD_ONLY, default="none", annotation=TargetMode) + ) + if seed_param: + sig_params.append( + inspect.Parameter("seed", inspect.Parameter.KEYWORD_ONLY, default=None, annotation=Optional[int]) + ) + __init__.__signature__ = inspect.Signature(sig_params) # type: ignore[attr-defined] + annotations = {p.name: p.annotation for p in specs if p.annotation is not inspect.Parameter.empty} + annotations["target"] = TargetMode + if seed_param: + annotations["seed"] = Optional[int] + __init__.__annotations__ = annotations + + def raw_transform(self: Any) -> Any: + kwargs = {} + for pname in pnames: + value = getattr(self, pname, None) + if value is None and pname in required: + continue # omitted → the library raises its own clear missing-argument error + kwargs[pname] = value + return transform_cls(**kwargs) + + base_pipeline = base.pipeline.fget + + def pipeline(self: Any) -> Any: + # Rebuild the wrapped transform when any mirrored param changed (post-construction + # configuration), then reuse the adapter's compose/cache machinery verbatim. + key: tuple = tuple(repr(getattr(self, pname, None)) for pname in pnames) + key += (self.target, getattr(self, "seed", None)) + if key != getattr(self, "_params_key", None): + self._params_key = key + self.transform = self.raw_transform + return base_pipeline(self) + + namespace = { + "__init__": __init__, + # The base __call__ re-stated in the class dict: canvas op-classification checks + # vars(cls) for __call__, and inherited-only methods are invisible to it. + "__call__": base.__call__, + "__doc__": _synth_doc(name, transform_cls, seed_param), + "__module__": module_name, + "raw_transform": property( + raw_transform, doc="The configured library transform instance (built fresh per access)." + ), + "pipeline": property(pipeline, doc="The live library pipeline for the current parameter values."), + "LIBRARY_CLS": transform_cls, + } + cls = type(op_name, (base,), namespace) + return configurable(category="op", group=group, random=True)(cls) + + +def generate_transform_ops( + *, + classes: Iterable[Tuple[str, type]], + base: Type[Any], + prefix: str, + group: str, + module_globals: dict, + seed_param: bool, +) -> List[str]: + """Generate one op per ``(name, transform_cls)`` into ``module_globals``; returns the sorted names. + + Per-class failures (uninspectable constructor, adapter-param clash) skip that + transform with a DEBUG note and never break the module import — the helios-bridge + warn-and-continue contract. + """ + module_name = module_globals.get("__name__", base.__module__) + names: List[str] = [] + for name, transform_cls in classes: + try: + op_cls = _make_op( + name, + transform_cls, + base=base, + prefix=prefix, + group=group, + module_name=module_name, + seed_param=seed_param, + ) + except Exception as exc: + logger.debug(f"augment bridge: skipping {name}: {exc}") + continue + module_globals[op_cls.__name__] = op_cls + names.append(op_cls.__name__) + logger.debug(f"augment bridge: generated {len(names)} {prefix}* ops in group {group!r}") + return sorted(names) + + +__all__ = ["generate_transform_ops"] diff --git a/sampleflux/ops/albumentations.py b/sampleflux/ops/albumentations.py new file mode 100644 index 0000000..f05dd38 --- /dev/null +++ b/sampleflux/ops/albumentations.py @@ -0,0 +1,199 @@ +"""``AlbumentationsOp`` — run `albumentations `_ transforms as a SampleFlux op. + +One random draw is applied jointly to ``sample.input`` and (per the ``target`` mode) its +segmentation mask / detection boxes, so a geometric augmentation moves image AND target +consistently; metadata passes through untouched. + +Transforms are authored **Confluid-natively** — nested ``!class:`` nodes, never +albumentations' own ``to_dict`` format:: + + - !class:sampleflux.ops.albumentations.AlbumentationsOp + target: mask + seed: 0 + transforms: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.Affine + translate_percent: 0.1 + +For per-transform graph nodes (one op per albumentations transform, e.g. +``AlbHorizontalFlip``) see :mod:`sampleflux.ops.albumentations_transforms`. + +Layout contract: albumentations operates on **numpy HWC** images (PIL inputs are converted +via ``np.asarray``) and the output stays numpy HWC — tensorize downstream with +:class:`~sampleflux.ops.torch.ToTensorOp`. Contrast with +:class:`~sampleflux.ops.torchvision.TorchvisionTransformOp`, which emits CHW torch tensors. +""" + +from typing import Any, Dict, List, Literal, Optional, Tuple + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from sampleflux.sample import Sample + +logger = get_logger(__name__) + +#: Which part of the ``Sample`` rides through the library jointly with the input. Closed +#: set so a typo fails at the call site and UIs / form-specs enumerate the choices. +TargetMode = Literal["none", "mask", "boxes"] + + +def _as_array(value: Any) -> np.ndarray: + """``value`` as a numpy array (PIL images and array-likes alike).""" + return np.asarray(value) + + +def _resolve_transform(entry: Any) -> Any: + """A raw library transform from a wired entry. + + Confluid ``!class:`` / ``!lazy:`` markers are flowed lazily (the RandomApply + paradigm), and a generated per-transform op (``AlbHorizontalFlip`` …) wired on a + visual canvas unwraps to its inner library transform via ``raw_transform``. + """ + from confluid import flow + from confluid.fluid import Fluid + + if isinstance(entry, Fluid): + entry = flow(entry) + return getattr(entry, "raw_transform", entry) + + +@configurable(category="op", group="augment", random=True) +class AlbumentationsOp: + """Apply albumentations transforms to ``sample.input`` (and optionally the target). + + Pass EITHER ``transform`` (one transform, or a prebuilt ``A.Compose``) OR + ``transforms`` (a list composed into an ``A.Compose`` lazily) — never both. Entries + may be live albumentations objects, Confluid ``!class:`` markers, or generated + per-transform ops (:mod:`sampleflux.ops.albumentations_transforms`), which unwrap to + their inner library transform. + + Target modes (the ``target`` knob): + + * ``"none"`` — input-only augmentation (color jitter, noise, blur); the sample's + target passes through untouched. + * ``"mask"`` — ``sample.target`` is a segmentation mask (2-D array or PIL ``L`` + image); image and mask receive the SAME spatial transform. + * ``"boxes"`` — ``sample.target`` is the torchvision detection dict + ``{"boxes": [N,4] xyxy-pixel, "labels": [N]}`` (what + :class:`~sampleflux.ops.target.CocoToTorchVisionDetectionOp` / + :class:`~sampleflux.ops.target.MasksToDetectionBoxesOp` emit). When the op builds + the Compose itself the required ``bbox_params`` are added automatically + (``pascal_voc`` = absolute-pixel xyxy); a prebuilt Compose must carry its own. + + Stochasticity lives in the library: ``seed`` maps onto ``A.Compose(seed=...)``; gate + per sample via :class:`~sampleflux.ops.random_apply.RandomApply` (each albumentations + transform also carries its own ``p``). + + YAML: + + .. code-block:: yaml + + - !class:sampleflux.ops.albumentations.AlbumentationsOp + target: mask + seed: 0 + transforms: + - !class:albumentations.HorizontalFlip + p: 0.5 + + Args: + transform: ONE albumentations transform or a prebuilt ``A.Compose``. Validated lazily on first call. + transforms: List of albumentations transforms composed lazily into an ``A.Compose``. + target: Joint-augmentation mode — ``none`` (input-only, default), ``mask``, or ``boxes``. + seed: ``A.Compose`` seed for deterministic draws. ``None`` = non-deterministic (default). + """ + + def __init__( + self, + transform: Optional[object] = None, + transforms: Optional[List[Any]] = None, + target: TargetMode = "none", + seed: Optional[int] = None, + ) -> None: + # Lazy / zero-arg: store config only; transforms are resolved/validated on first call. + self.transform = transform + self.transforms: List[Any] = list(transforms) if transforms else [] + self.target = target + self.seed = seed + self._pipeline: Optional[object] = None + self._pipeline_key: Optional[tuple] = None + + def _entries(self) -> List[Any]: + """The configured raw transforms (markers flowed, generated ops unwrapped).""" + if self.transform is not None and self.transforms: + raise ValueError("AlbumentationsOp: pass either 'transform' or 'transforms', not both.") + entries = [self.transform] if self.transform is not None else list(self.transforms) + if not entries: + raise ValueError( + "AlbumentationsOp requires 'transform' (one transform / A.Compose) or " + "'transforms' (a list) to be set before calling." + ) + return [_resolve_transform(entry) for entry in entries] + + @property + def pipeline(self) -> Any: + """The live ``A.Compose`` — built lazily, cached until the configuration changes.""" + key = (id(self.transform), tuple(id(t) for t in self.transforms), self.target, self.seed) + if self._pipeline is None or self._pipeline_key != key: + # Albumentations is a hard dependency but slow to import — keep it lazy so + # module import (entry-point discovery) stays light (the target.py precedent). + import albumentations as A + from albumentations.core.composition import BaseCompose + + entries = self._entries() + if len(entries) == 1 and isinstance(entries[0], BaseCompose): + if self.seed is not None: + raise ValueError( + "AlbumentationsOp: 'seed' only applies when the op builds the Compose itself; " + "put the seed on your prebuilt A.Compose(..., seed=...) instead." + ) + self._pipeline = entries[0] + else: + bbox_params = ( + A.BboxParams(format="pascal_voc", label_fields=["labels"]) if self.target == "boxes" else None + ) + self._pipeline = A.Compose(entries, seed=self.seed, bbox_params=bbox_params) + self._pipeline_key = key + return self._pipeline + + def __call__(self, sample: Sample) -> Sample: + image = _as_array(sample.input) + if self.target == "mask": + out = self.pipeline(image=image, mask=_as_array(sample.target)) + return sample._replace(input=out["image"], target=out["mask"]) + if self.target == "boxes": + new_input, new_target = self._apply_boxes(image, sample.target) + return sample._replace(input=new_input, target=new_target) + out = self.pipeline(image=image) + return sample._replace(input=out["image"]) + + def _apply_boxes(self, image: np.ndarray, target: Any) -> Tuple[Any, Dict[str, Any]]: + """Route the torchvision detection dict through albumentations' bbox machinery.""" + import torch + + pipeline = self.pipeline + if not isinstance(target, dict) or "boxes" not in target or "labels" not in target: + raise TypeError( + f"AlbumentationsOp(target='boxes'): sample.target must be the torchvision detection " + f"dict {{'boxes': [N,4] xyxy, 'labels': [N]}}; got {type(target).__name__}. Wire " + "CocoToTorchVisionDetectionOp / MasksToDetectionBoxesOp upstream." + ) + if "bboxes" not in getattr(pipeline, "processors", {}): + raise ValueError( + "AlbumentationsOp(target='boxes'): the prebuilt Compose was built without bbox_params. " + "Construct it as A.Compose([...], bbox_params=A.BboxParams(format='pascal_voc', " + "label_fields=['labels'])) — or pass 'transforms' and let the op add them." + ) + boxes = np.asarray(target["boxes"], dtype=np.float32).reshape(-1, 4) + labels = [int(v) for v in np.asarray(target["labels"]).reshape(-1)] + out = pipeline(image=image, bboxes=boxes.tolist(), labels=labels) + out_boxes = np.asarray(out["bboxes"], dtype=np.float32).reshape(-1, 4) + new_target = dict(target) + new_target["boxes"] = torch.as_tensor(out_boxes, dtype=torch.float32) + new_target["labels"] = torch.as_tensor(list(out["labels"]), dtype=torch.int64).reshape(-1) + return out["image"], new_target + + +__all__ = ["AlbumentationsOp", "TargetMode"] diff --git a/sampleflux/ops/albumentations_transforms.py b/sampleflux/ops/albumentations_transforms.py new file mode 100644 index 0000000..498915a --- /dev/null +++ b/sampleflux/ops/albumentations_transforms.py @@ -0,0 +1,65 @@ +"""Auto-generated ops: every public albumentations transform as its own SampleFlux op. + +Generated at import time by :mod:`sampleflux.ops._augment_bridge` from albumentations' +public namespace — one ``Alb`` op per concrete transform (``AlbHorizontalFlip``, +``AlbAffine``, ``AlbRandomBrightnessContrast``, …), each a subclass of +:class:`~sampleflux.ops.albumentations.AlbumentationsOp` mirroring the transform's own +constructor parameters plus the adapter's ``target`` / ``seed`` knobs. + +YAML (the registered short name resolves via the confluid registry): + +.. code-block:: yaml + + - !class:AlbHorizontalFlip + p: 0.5 + target: mask + +The ``Alb`` prefix is MANDATORY — albumentations and torchvision share many bare class +names (``ColorJitter``, ``Normalize``, ``Resize``, …) and confluid's registry is flat and +name-keyed, so unprefixed names would silently clobber each other. + +Composition transforms (``Compose`` / ``OneOf`` / ``SomeOf`` …) are deliberately NOT +generated — chaining ops is native SampleFlux (``ops:`` lists, ``TransformChain``, +``RandomApply``), and an ``A.Compose`` still wires verbatim into +``AlbumentationsOp(transform=...)``. +""" + +from typing import List, Tuple + +from loggair import get_logger + +from sampleflux.ops._augment_bridge import generate_transform_ops +from sampleflux.ops.albumentations import AlbumentationsOp + +logger = get_logger(__name__) + + +def _library_classes() -> List[Tuple[str, type]]: + """The concrete, generatable albumentations transform classes, sorted by name.""" + import albumentations as A + from albumentations.core.composition import BaseCompose + from albumentations.core.transforms_interface import BasicTransform + + out: List[Tuple[str, type]] = [] + for name in sorted(vars(A)): + obj = getattr(A, name) + if not (isinstance(obj, type) and issubclass(obj, BasicTransform)): + continue + if issubclass(obj, BaseCompose) or obj.__name__ != name or name.startswith("_"): + continue + if obj.__module__.startswith("albumentations.core"): + continue # the abstract bases (BasicTransform / DualTransform / ImageOnlyTransform) + if name == "Lambda": + continue # callable-valued params — not representable as config + out.append((name, obj)) + return out + + +__all__ = generate_transform_ops( + classes=_library_classes(), + base=AlbumentationsOp, + prefix="Alb", + group="augment/albumentations", + module_globals=globals(), + seed_param=True, +) diff --git a/sampleflux/ops/configure.py b/sampleflux/ops/configure.py index ed32370..d5cec8f 100644 --- a/sampleflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -72,6 +72,10 @@ def __call__(self, sample: Sample) -> Optional[Sample]: raise ValueError("ConfigureOp: 'param' (the target attribute to set) is required") if isinstance(self.target, Fluid): self.target = flow(self.target) + # _apply_op = the engine's contract-aware chokepoint, so field-scoped ops + # (e.g. a pair-scoped op from the kinds grid) work in the compute chain and as target. + from sampleflux.core import _apply_op + current: Sample = sample for i, op in enumerate(self.ops): if isinstance(op, Fluid): @@ -79,7 +83,7 @@ def __call__(self, sample: Sample) -> Optional[Sample]: self.ops[i] = op if op is None: continue - result = op(current) + result = _apply_op(current, op) if result is None: return None # the compute chain filtered the sample (FilterOp semantics) current = result @@ -87,7 +91,7 @@ def __call__(self, sample: Sample) -> Optional[Sample]: sample.meta[self.key or self.param] = value target = cast(Any, self.target) setattr(target, self.param, value) - return cast(Optional[Sample], target(sample)) + return _apply_op(sample, target) def close(self) -> None: """Propagate close() to inner ops that own resources.""" diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py index 2d104ef..36835c7 100644 --- a/sampleflux/ops/context.py +++ b/sampleflux/ops/context.py @@ -179,7 +179,11 @@ def __call__(self, sample: Sample) -> Optional[Sample]: value = _cell_field(value, "input") op = cast(Any, self.op) setattr(op, self.param, value) - return cast(Optional[Sample], op(sample)) + # _apply_op = the engine's contract-aware chokepoint, so a field-scoped wrapped op + # (e.g. a pair-scoped op from the kinds grid) applies exactly as in a bare ops list. + from sampleflux.core import _apply_op + + return _apply_op(sample, op) def close(self) -> None: """Propagate close() to the wrapped op if it owns resources.""" @@ -236,7 +240,10 @@ def __call__(self, sample: Sample) -> Optional[Sample]: self.op = _flow_if_fluid(self.op) ctx = require("Capture") op = cast(Any, self.op) - result = op(sample) + # _apply_op = the engine's contract-aware chokepoint (field-scoped ops capture too). + from sampleflux.core import _apply_op + + result = _apply_op(sample, op) if result is None: return None # the wrapped op filtered the sample (FilterOp semantics) for attr, cell in items.items(): diff --git a/sampleflux/ops/enable.py b/sampleflux/ops/enable.py index 291fa5b..6e1d967 100644 --- a/sampleflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -126,7 +126,7 @@ def flag_name(self) -> str: name, _ = self._toggle() return name - def __call__(self, sample: Sample) -> Sample: + def __call__(self, sample: Sample) -> Optional[Sample]: if not self.ops: raise ValueError("Enable requires a non-empty 'ops' list.") if not self.enabled: @@ -134,14 +134,21 @@ def __call__(self, sample: Sample) -> Sample: from confluid import flow from confluid.fluid import Fluid + # _apply_op = the engine's contract-aware chokepoint, so field-scoped ops + # (e.g. a pair-scoped op from the kinds grid) run under the toggle unchanged. + from sampleflux.core import _apply_op + + current: Optional[Sample] = sample for i, op in enumerate(self.ops): + if current is None: + return None if isinstance(op, Fluid): op = flow(op) self.ops[i] = op if op is None: continue - sample = op(sample) - return sample + current = _apply_op(current, op) + return current def close(self) -> None: """Propagate close to inner ops that own resources (e.g. SampleSinkOp).""" diff --git a/sampleflux/ops/parallel.py b/sampleflux/ops/parallel.py index 21898ff..a81427a 100644 --- a/sampleflux/ops/parallel.py +++ b/sampleflux/ops/parallel.py @@ -51,13 +51,17 @@ def _materialize_ops(self) -> None: self.ops[i] = flow(op) def __call__(self, sample: Sample) -> Optional[Sample]: - # Inline fallback for non-streaming callers (e.g. Flux.__getitem__). + # Inline fallback for non-streaming callers (e.g. Flux.__getitem__). Routed through + # _apply_op — the same contract-aware chokepoint the streamed route's _worker_task + # uses — so field-scoped ops (e.g. a pair-scoped op from the kinds grid) behave identically. + from sampleflux.core import _apply_op + self._materialize_ops() current: Optional[Sample] = sample for op in self.ops: if current is None: return None - current = op(current) + current = _apply_op(current, op) return current def stream(self, samples: Iterable[Optional[Sample]]) -> Iterator[Optional[Sample]]: diff --git a/sampleflux/ops/random_apply.py b/sampleflux/ops/random_apply.py index 891ab7e..207d4f6 100644 --- a/sampleflux/ops/random_apply.py +++ b/sampleflux/ops/random_apply.py @@ -9,7 +9,7 @@ """ import random -from typing import Optional, cast +from typing import Optional from confluid import configurable from loggair import get_logger @@ -59,7 +59,7 @@ def __init__( self.random_state = random_state self._gate_rng: Optional[random.Random] = None - def __call__(self, sample: Sample) -> Sample: + def __call__(self, sample: Sample) -> Optional[Sample]: if self.op is None: raise ValueError("RandomApply requires 'op' to be set before calling.") if self._gate_rng is None: @@ -69,9 +69,14 @@ def __call__(self, sample: Sample) -> Sample: from confluid import flow from confluid.fluid import Fluid + # _apply_op is the engine's single contract-aware chokepoint — routing through it + # (instead of op(sample)) lets a field-scoped op (e.g. a pair-scoped op from the + # kinds grid) nest inside the gate exactly as it would sit in a bare ops list. + from sampleflux.core import _apply_op + op = flow(self.op) if isinstance(self.op, Fluid) else self.op self.op = op # cache the flowed op so we only flow once - return cast(Sample, op(sample)) # type: ignore[operator] + return _apply_op(sample, op) __all__ = ["RandomApply"] diff --git a/sampleflux/ops/torchvision.py b/sampleflux/ops/torchvision.py new file mode 100644 index 0000000..526a560 --- /dev/null +++ b/sampleflux/ops/torchvision.py @@ -0,0 +1,201 @@ +"""``TorchvisionTransformOp`` — run torchvision ``transforms.v2`` transforms as a SampleFlux op. + +v2 transforms draw their random parameters ONCE per call and apply them to every +``tv_tensors`` carrier passed in, so a geometric augmentation moves ``sample.input`` AND +(per the ``target`` mode) its segmentation mask / detection boxes consistently; metadata +passes through untouched. + +Transforms are authored **Confluid-natively** as nested ``!class:`` nodes:: + + - !class:sampleflux.ops.torchvision.TorchvisionTransformOp + target: mask + transforms: + - !class:torchvision.transforms.v2.RandomHorizontalFlip + p: 0.5 + +For per-transform graph nodes (one op per v2 transform, e.g. ``TvRandomHorizontalFlip``) +see :mod:`sampleflux.ops.torchvision_transforms`. + +Layout contract: torchvision operates on **CHW torch tensors** (PIL images pass through +natively; numpy HWC arrays are converted on entry) and the output is CHW tensors — no +:class:`~sampleflux.ops.torch.ToTensorOp` needed downstream. Contrast with +:class:`~sampleflux.ops.albumentations.AlbumentationsOp`, which stays numpy HWC. + +This module imports WITHOUT torchvision installed (it is entry-pointed for discovery); +torchvision is lazy-imported on first call and a missing install raises a clear error +pointing at the ``sampleflux[vision]`` extra. +""" + +from typing import Any, List, Optional, Tuple + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from sampleflux.ops.albumentations import TargetMode, _resolve_transform +from sampleflux.sample import Sample + +logger = get_logger(__name__) + + +def _import_v2() -> Any: + """The ``torchvision.transforms.v2`` module, or a clear error naming the extra.""" + try: + from torchvision.transforms import v2 + except ImportError as exc: # pragma: no cover - exercised only without torchvision + raise ImportError( + "TorchvisionTransformOp requires torchvision (transforms.v2 / tv_tensors). " + 'Install it via `pip install "sampleflux[vision]"`.' + ) from exc + return v2 + + +@configurable(category="op", group="augment", random=True) +class TorchvisionTransformOp: + """Apply torchvision ``transforms.v2`` transforms to ``sample.input`` (and optionally the target). + + Pass EITHER ``transform`` (one v2 transform, or a prebuilt ``v2.Compose``) OR + ``transforms`` (a list composed into a ``v2.Compose`` lazily) — never both. Entries + may be live v2 objects, Confluid ``!class:`` markers, or generated per-transform ops + (:mod:`sampleflux.ops.torchvision_transforms`), which unwrap to their inner library + transform. + + Target modes (the ``target`` knob): + + * ``"none"`` — input-only augmentation; the sample's target passes through untouched. + * ``"mask"`` — ``sample.target`` is a segmentation mask (2-D array / tensor or PIL + ``L`` image), wrapped as ``tv_tensors.Mask`` so image and mask receive the SAME + spatial transform. + * ``"boxes"`` — ``sample.target`` is the torchvision detection dict + ``{"boxes": [N,4] xyxy-pixel, "labels": [N]}`` (what + :class:`~sampleflux.ops.target.CocoToTorchVisionDetectionOp` / + :class:`~sampleflux.ops.target.MasksToDetectionBoxesOp` emit); boxes are wrapped as + ``tv_tensors.BoundingBoxes(format="XYXY", canvas_size=(H, W))`` and come back as + plain float32 / int64 tensors. + + Stochasticity lives in the library: v2 transforms draw from torch's global RNG + (``torch.manual_seed(N)`` pins it); gate per sample via + :class:`~sampleflux.ops.random_apply.RandomApply`. + + YAML: + + .. code-block:: yaml + + - !class:sampleflux.ops.torchvision.TorchvisionTransformOp + target: mask + transforms: + - !class:torchvision.transforms.v2.RandomHorizontalFlip + p: 0.5 + + Args: + transform: ONE ``transforms.v2`` transform or a prebuilt ``v2.Compose``. Validated lazily on first call. + transforms: List of v2 transforms composed lazily into a ``v2.Compose``. + target: Joint-augmentation mode — ``none`` (input-only, default), ``mask``, or ``boxes``. + """ + + def __init__( + self, + transform: Optional[object] = None, + transforms: Optional[List[Any]] = None, + target: TargetMode = "none", + ) -> None: + # Lazy / zero-arg: store config only; transforms are resolved/validated on first call. + self.transform = transform + self.transforms: List[Any] = list(transforms) if transforms else [] + self.target = target + self._pipeline: Optional[object] = None + self._pipeline_key: Optional[tuple] = None + + @property + def pipeline(self) -> Any: + """The live v2 transform — built lazily, cached until the configuration changes.""" + if self.transform is not None and self.transforms: + raise ValueError("TorchvisionTransformOp: pass either 'transform' or 'transforms', not both.") + key = (id(self.transform), tuple(id(t) for t in self.transforms)) + if self._pipeline is None or self._pipeline_key != key: + if self.transform is not None: + self._pipeline = _resolve_transform(self.transform) + elif self.transforms: + v2 = _import_v2() + self._pipeline = v2.Compose([_resolve_transform(entry) for entry in self.transforms]) + else: + raise ValueError( + "TorchvisionTransformOp requires 'transform' (one v2 transform / v2.Compose) or " + "'transforms' (a list) to be set before calling." + ) + self._pipeline_key = key + return self._pipeline + + def __call__(self, sample: Sample) -> Sample: + import torch + + _import_v2() # raise the actionable extra hint before any torchvision use + from torchvision import tv_tensors + + pipeline = self.pipeline + image = self._wrap_image(sample.input, tv_tensors, torch) + + if self.target == "mask": + mask = self._wrap_mask(sample.target, tv_tensors, torch) + out_image, out_mask = pipeline(image, mask) + return sample._replace(input=self._unwrap(out_image, torch), target=self._unwrap(out_mask, torch)) + if self.target == "boxes": + target = sample.target + if not isinstance(target, dict) or "boxes" not in target or "labels" not in target: + raise TypeError( + f"TorchvisionTransformOp(target='boxes'): sample.target must be the torchvision " + f"detection dict {{'boxes': [N,4] xyxy, 'labels': [N]}}; got {type(target).__name__}. " + "Wire CocoToTorchVisionDetectionOp / MasksToDetectionBoxesOp upstream." + ) + boxes = tv_tensors.BoundingBoxes( + torch.as_tensor(np.asarray(target["boxes"], dtype=np.float32).reshape(-1, 4)), + format="XYXY", + canvas_size=self._canvas_size(image), + ) + labels = torch.as_tensor(np.asarray(target["labels"]).reshape(-1), dtype=torch.int64) + out_image, out_target = pipeline(image, {**target, "boxes": boxes, "labels": labels}) + out_target["boxes"] = self._unwrap(out_target["boxes"], torch).to(torch.float32) + out_target["labels"] = self._unwrap(out_target["labels"], torch) + return sample._replace(input=self._unwrap(out_image, torch), target=out_target) + return sample._replace(input=self._unwrap(pipeline(image), torch)) + + @staticmethod + def _wrap_image(value: Any, tv_tensors: Any, torch: Any) -> Any: + """``value`` as a v2 carrier: PIL passes through, tensors/arrays become CHW ``tv_tensors.Image``.""" + if hasattr(value, "convert"): # PIL image — v2 transforms handle it natively + return value + if isinstance(value, torch.Tensor): + tensor = value + else: + array = np.asarray(value) + tensor = torch.as_tensor(np.ascontiguousarray(array)) + if tensor.ndim == 3: # numpy convention is HWC; torchvision wants CHW + tensor = tensor.permute(2, 0, 1) + if tensor.ndim == 2: + tensor = tensor.unsqueeze(0) + return tv_tensors.Image(tensor) + + @staticmethod + def _wrap_mask(value: Any, tv_tensors: Any, torch: Any) -> Any: + """``value`` as a ``tv_tensors.Mask`` (PIL ``L`` images and 2-D arrays alike).""" + if hasattr(value, "convert"): + value = np.asarray(value) + tensor = value if isinstance(value, torch.Tensor) else torch.as_tensor(np.ascontiguousarray(value)) + return tv_tensors.Mask(tensor) + + @staticmethod + def _canvas_size(image: Any) -> Tuple[int, int]: + """``(H, W)`` of the wrapped input — the reference frame for bounding boxes.""" + if hasattr(image, "convert"): # PIL + return int(image.height), int(image.width) + return int(image.shape[-2]), int(image.shape[-1]) + + @staticmethod + def _unwrap(value: Any, torch: Any) -> Any: + """Strip the ``tv_tensors`` subclass so plain tensors flow downstream.""" + if isinstance(value, torch.Tensor): + return value.as_subclass(torch.Tensor) + return value + + +__all__ = ["TorchvisionTransformOp"] diff --git a/sampleflux/ops/torchvision_transforms.py b/sampleflux/ops/torchvision_transforms.py new file mode 100644 index 0000000..3326e51 --- /dev/null +++ b/sampleflux/ops/torchvision_transforms.py @@ -0,0 +1,69 @@ +"""Auto-generated ops: every torchvision ``transforms.v2`` transform as its own SampleFlux op. + +Generated at import time by :mod:`sampleflux.ops._augment_bridge` from the +``torchvision.transforms.v2`` namespace — one ``Tv`` op per concrete transform +(``TvRandomHorizontalFlip``, ``TvColorJitter``, …), each a subclass of +:class:`~sampleflux.ops.torchvision.TorchvisionTransformOp` mirroring the transform's own +constructor parameters plus the adapter's ``target`` knob. + +YAML (the registered short name resolves via the confluid registry): + +.. code-block:: yaml + + - !class:TvRandomHorizontalFlip + p: 0.5 + target: mask + +The ``Tv`` prefix is MANDATORY — torchvision and albumentations share many bare class +names (``ColorJitter``, ``Normalize``, ``Resize``, …) and confluid's registry is flat and +name-keyed, so unprefixed names would silently clobber each other. + +The module imports WITHOUT torchvision installed (entry-point discovery stays safe): it +then generates zero ops and logs a debug note pointing at the ``sampleflux[vision]`` +extra. Container transforms (``Compose`` / ``RandomApply`` / ``RandomChoice`` / +``RandomOrder``) are deliberately NOT generated — chaining ops is native SampleFlux. +""" + +from typing import List, Tuple + +from loggair import get_logger + +from sampleflux.ops._augment_bridge import generate_transform_ops +from sampleflux.ops.torchvision import TorchvisionTransformOp + +logger = get_logger(__name__) + +#: v2 names not generated: containers (native chaining) + the deprecated v1 shim ToTensor. +_EXCLUDED = frozenset({"Compose", "RandomApply", "RandomChoice", "RandomOrder", "ToTensor", "Transform"}) + + +def _library_classes() -> List[Tuple[str, type]]: + """The concrete, generatable v2 transform classes, sorted by name (empty without torchvision).""" + try: + from torchvision.transforms import v2 + except ImportError: + logger.debug( + "torchvision not installed — no Tv* transform ops generated " + '(install via `pip install "sampleflux[vision]"`).' + ) + return [] + + out: List[Tuple[str, type]] = [] + for name in sorted(vars(v2)): + obj = getattr(v2, name) + if not (isinstance(obj, type) and issubclass(obj, v2.Transform)): + continue + if name in _EXCLUDED or obj.__name__ != name or name.startswith("_"): + continue + out.append((name, obj)) + return out + + +__all__ = generate_transform_ops( + classes=_library_classes(), + base=TorchvisionTransformOp, + prefix="Tv", + group="augment/torchvision", + module_globals=globals(), + seed_param=False, +) diff --git a/sampleflux/ops/transform_chain.py b/sampleflux/ops/transform_chain.py index a6acb17..e6dbc09 100644 --- a/sampleflux/ops/transform_chain.py +++ b/sampleflux/ops/transform_chain.py @@ -61,6 +61,10 @@ def __call__(self, sample: Sample) -> Optional[Sample]: from confluid import flow from confluid.fluid import Fluid + # _apply_op = the engine's contract-aware chokepoint, so field-scoped ops + # (e.g. a pair-scoped op from the kinds grid) chain exactly as in a bare ops list. + from sampleflux.core import _apply_op + current: Optional[Sample] = sample for i, op in enumerate(self.ops): if current is None: @@ -70,7 +74,7 @@ def __call__(self, sample: Sample) -> Optional[Sample]: self.ops[i] = op if op is None: continue - current = op(current) + current = _apply_op(current, op) return current def close(self) -> None: diff --git a/tests/test_augment_ops.py b/tests/test_augment_ops.py new file mode 100644 index 0000000..4199654 --- /dev/null +++ b/tests/test_augment_ops.py @@ -0,0 +1,312 @@ +"""Tests for the augmentation adapters + the generated per-transform op families. + +``AlbumentationsOp`` / ``TorchvisionTransformOp`` are classic sample-scoped ops +(``__call__(sample)`` — the form every engine, composing op, AND visual-canvas node +classifier handles) applying ONE library draw jointly to input and target per the +``target`` mode. Transforms are configured Confluid-natively (nested ``!class:`` nodes or +the generated ``Alb*`` / ``Tv*`` per-transform ops from +:mod:`sampleflux.ops.albumentations_transforms` / +:mod:`sampleflux.ops.torchvision_transforms`). +""" + +import subprocess +import sys +from pathlib import Path + +import albumentations as A +import confluid # type: ignore[import-not-found] +import numpy as np +import pytest +import torch +from PIL import Image + +import sampleflux.ops.albumentations_transforms as albt +from sampleflux.core import Flux +from sampleflux.kinds import op_contract +from sampleflux.ops.albumentations import AlbumentationsOp +from sampleflux.ops.target import MasksToDetectionBoxesOp +from sampleflux.ops.torchvision import TorchvisionTransformOp +from sampleflux.sample import Sample + + +def _image() -> np.ndarray: + return np.arange(4 * 6 * 3, dtype=np.uint8).reshape(4, 6, 3) + + +def _mask() -> np.ndarray: + mask = np.zeros((4, 6), dtype=np.uint8) + mask[1:3, 0:2] = 1 + return mask + + +def _sample(**meta: object) -> Sample: + return Sample(_image(), _mask(), dict(meta)) + + +def _detection_sample() -> Sample: + # MasksToDetectionBoxesOp derives the tight xyxy box from the mask — the exact + # detection-dict contract both adapters consume in target="boxes" mode. + return MasksToDetectionBoxesOp()(Sample(_image(), _mask(), {})) + + +class TestAlbumentationsOp: + def test_zero_arg_construction_and_lazy_validation(self) -> None: + op = AlbumentationsOp() + with pytest.raises(ValueError, match="transform"): + op(_sample()) + + def test_transform_and_transforms_mutually_exclusive(self) -> None: + op = AlbumentationsOp(transform=A.HorizontalFlip(p=1.0), transforms=[A.HorizontalFlip(p=1.0)]) + with pytest.raises(ValueError, match="not both"): + op(_sample()) + + def test_contract_is_sample_scoped(self) -> None: + # Sample scope is what the engine fast-path, every composing op, AND the visual + # canvas classifier handle — guard the signature. + contract = op_contract(AlbumentationsOp()) + assert contract.accepts == "sample" + + def test_single_transform_input_only(self) -> None: + out = list(Flux(source=[_sample(idx=7)], ops=[AlbumentationsOp(A.HorizontalFlip(p=1.0))]))[0] + assert np.array_equal(out.input, _image()[:, ::-1]) + assert np.array_equal(out.target, _mask()) # target NOT flipped in "none" mode + assert out.meta == {"idx": 7} + + def test_transforms_list_mask_mode(self) -> None: + op = AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="mask", seed=0) + out = list(Flux(source=[_sample(idx=7)], ops=[op]))[0] + assert np.array_equal(out.input, _image()[:, ::-1]) + assert np.array_equal(out.target, _mask()[:, ::-1]) + assert out.meta == {"idx": 7} # metadata untouched + + def test_prebuilt_compose_accepted_seed_rejected(self) -> None: + compose = A.Compose([A.HorizontalFlip(p=1.0)]) + out = AlbumentationsOp(compose, target="mask")(_sample()) + assert np.array_equal(out.target, _mask()[:, ::-1]) + with pytest.raises(ValueError, match="seed"): + AlbumentationsOp(compose, target="mask", seed=3)(_sample()) + + def test_pipeline_rebuilds_when_transform_changes(self) -> None: + op = AlbumentationsOp(A.HorizontalFlip(p=1.0)) + first = op.pipeline + op.transform = A.HorizontalFlip(p=0.0) + assert op.pipeline is not first # the lazy cache keys on the configured objects + + def test_pil_input_accepted(self) -> None: + out = AlbumentationsOp(A.HorizontalFlip(p=1.0))(Sample(Image.fromarray(_image()), None, {})) + assert isinstance(out.input, np.ndarray) + assert np.array_equal(out.input, _image()[:, ::-1]) + + def test_boxes_mode_auto_bbox_params(self) -> None: + # When the op builds the Compose itself, bbox_params are added automatically. + op = AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="boxes") + sample = _detection_sample() + out = list(Flux(source=[sample], ops=[op]))[0] + width = _image().shape[1] + x0, _, x1, _ = sample.target["boxes"][0].tolist() + assert isinstance(out.target["boxes"], torch.Tensor) + assert out.target["boxes"].dtype == torch.float32 + assert out.target["labels"].dtype == torch.int64 + assert np.allclose(out.target["boxes"][0].tolist(), [width - x1, 1.0, width - x0, 3.0], atol=1e-4) + + def test_boxes_mode_prebuilt_compose_requires_bbox_params(self) -> None: + op = AlbumentationsOp(A.Compose([A.HorizontalFlip(p=1.0)]), target="boxes") + with pytest.raises(ValueError, match="bbox_params"): + op(_detection_sample()) + + def test_boxes_mode_requires_detection_dict(self) -> None: + op = AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="boxes") + with pytest.raises(TypeError, match="detection"): + op(Sample(_image(), "not-a-dict", {})) + + def test_confluid_native_yaml_roundtrip(self) -> None: + # The YAML surface is nested !class: nodes — no library-specific dict formats. + yaml_text = ( + "!class:sampleflux.ops.albumentations.AlbumentationsOp\n" + "target: mask\n" + "transforms:\n" + " - !class:albumentations.HorizontalFlip\n" + " p: 1.0\n" + ) + op = confluid.load(yaml_text) + out = op(_sample()) + assert np.array_equal(out.target, _mask()[:, ::-1]) + # Pipeline Parity: dump → reload → identical output. + reloaded = confluid.load(confluid.dump(op)) + out2 = reloaded(_sample()) + assert np.array_equal(out.input, out2.input) + assert np.array_equal(out.target, out2.target) + + +class TestGeneratedAlbumentationsOps: + def test_family_generated(self) -> None: + assert len(albt.__all__) > 50 + for name in ("AlbHorizontalFlip", "AlbAffine", "AlbRandomBrightnessContrast"): + assert name in albt.__all__ + + def test_flip_parity_with_raw_library(self) -> None: + out = list(Flux(source=[_sample(idx=1)], ops=[albt.AlbHorizontalFlip(p=1.0, target="mask")]))[0] + assert np.array_equal(out.input, _image()[:, ::-1]) + assert np.array_equal(out.target, _mask()[:, ::-1]) + assert out.meta == {"idx": 1} + + def test_marks_and_canvas_classification(self) -> None: + cls = albt.AlbHorizontalFlip + assert cls.__confluid_category__ == "op" + assert cls.__confluid_group__ == "augment/albumentations" + assert cls.__confluid_random__ is True + # The canvas op classifier reads vars(cls) — the base __call__ must be re-stated. + assert "__call__" in vars(cls) + assert cls.LIBRARY_CLS is A.HorizontalFlip + + def test_signature_mirrors_transform_plus_adapter_knobs(self) -> None: + import inspect + + params = list(inspect.signature(albt.AlbHorizontalFlip).parameters) + assert "p" in params + assert params[-2:] == ["target", "seed"] + docs = confluid.parse_param_docs(albt.AlbHorizontalFlip) + assert docs.get("target") # adapter knobs documented in the spliced Args block + + def test_required_param_lazy_error(self) -> None: + crop = albt.AlbRandomCrop() # zero-arg construction always works + with pytest.raises(Exception, match="height"): + crop(_sample()) # the library's own missing-argument error, raised lazily + + def test_post_construction_reconfigure_rebuilds(self) -> None: + op = albt.AlbHorizontalFlip(p=0.0) + assert np.array_equal(op(_sample()).input, _image()) # p=0 → identity + op.p = 1.0 # confluid post-construction paradigm + assert np.array_equal(op(_sample()).input, _image()[:, ::-1]) + + def test_generated_op_unwraps_in_transforms_list(self) -> None: + # A generated op wired into an adapter's transforms slot (the canvas pattern) + # unwraps to its inner library transform via raw_transform. + op = AlbumentationsOp(transforms=[albt.AlbHorizontalFlip(p=1.0)], target="mask") + out = op(_sample()) + assert np.array_equal(out.target, _mask()[:, ::-1]) + + def test_short_name_yaml_roundtrip(self) -> None: + yaml_text = "!class:AlbHorizontalFlip\np: 1.0\ntarget: mask\n" + op = confluid.load(yaml_text) + out = op(_sample()) + assert np.array_equal(out.target, _mask()[:, ::-1]) + dumped = confluid.dump(op) + assert "AlbHorizontalFlip" in dumped and "p: 1.0" in dumped + out2 = confluid.load(dumped)(_sample()) + assert np.array_equal(out.input, out2.input) + + def test_composes_inside_random_apply_and_chain(self) -> None: + # Composing ops route inner ops through core._apply_op, so the generated ops nest + # inside the gate/chain — the canonical "gate an augmentation" pattern. + from sampleflux.ops.random_apply import RandomApply + from sampleflux.ops.transform_chain import TransformChain + + chain = TransformChain( + ops=[RandomApply(op=albt.AlbHorizontalFlip(p=1.0, target="mask"), probability=1.0, random_state=0)] + ) + out = list(Flux(source=[_sample(idx=3)], ops=[chain]))[0] + assert np.array_equal(out.input, _image()[:, ::-1]) + assert np.array_equal(out.target, _mask()[:, ::-1]) + assert out.meta == {"idx": 3} + + def test_composes_inside_enable(self) -> None: + from sampleflux.ops.enable import Enable + + enable = Enable(ops=[albt.AlbHorizontalFlip(p=1.0, target="mask")]) + setattr(enable, "augment", True) # the toggle flag arrives post-construction (Confluid paradigm) + out = list(Flux(source=[_sample()], ops=[enable]))[0] + assert np.array_equal(out.input, _image()[:, ::-1]) + assert np.array_equal(out.target, _mask()[:, ::-1]) + + +class TestTorchvisionTransformOp: + v2 = pytest.importorskip("torchvision.transforms.v2") + + def test_zero_arg_construction_and_lazy_validation(self) -> None: + op = TorchvisionTransformOp() + with pytest.raises(ValueError, match="transform"): + op(_sample()) + + def test_contract_is_sample_scoped(self) -> None: + assert op_contract(TorchvisionTransformOp()).accepts == "sample" + + def test_input_only_flip_emits_chw_tensor(self) -> None: + op = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0)) + out = list(Flux(source=[_sample(idx=7)], ops=[op]))[0] + assert type(out.input) is torch.Tensor # tv_tensors subclass stripped + assert out.input.shape == (3, 4, 6) + assert np.array_equal(out.input.permute(1, 2, 0).numpy(), _image()[:, ::-1]) + assert np.array_equal(out.target, _mask()) # untouched in "none" mode + assert out.meta == {"idx": 7} + + def test_transforms_list_mask_mode_matches_albumentations(self) -> None: + # Cross-library parity: the same deterministic flip through either adapter + # yields the same pixels (layouts differ — CHW tensor vs HWC array). + tv = TorchvisionTransformOp(transforms=[self.v2.RandomHorizontalFlip(p=1.0)], target="mask")(_sample()) + alb = AlbumentationsOp(A.HorizontalFlip(p=1.0), target="mask")(_sample()) + assert np.array_equal(tv.input.permute(1, 2, 0).numpy(), alb.input) + assert np.array_equal(tv.target.numpy(), alb.target) + + def test_two_dim_input_gains_channel_axis(self) -> None: + out = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0))(Sample(_mask(), None, {})) + assert out.input.shape == (1, 4, 6) + + def test_pil_input_stays_pil(self) -> None: + out = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0))(Sample(Image.fromarray(_image()), None, {})) + assert isinstance(out.input, Image.Image) + assert np.array_equal(np.asarray(out.input), _image()[:, ::-1]) + + def test_boxes_mode_mirrors_coordinates(self) -> None: + sample = _detection_sample() + op = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0), target="boxes") + out = list(Flux(source=[sample], ops=[op]))[0] + width = _image().shape[1] + x0, _, x1, _ = sample.target["boxes"][0].tolist() + assert type(out.target["boxes"]) is torch.Tensor # BoundingBoxes subclass stripped + assert out.target["boxes"].dtype == torch.float32 + assert out.target["labels"].dtype == torch.int64 + assert out.target["boxes"][0].tolist() == [width - x1, 1.0, width - x0, 3.0] + + def test_boxes_mode_requires_detection_dict(self) -> None: + op = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0), target="boxes") + with pytest.raises(TypeError, match="detection"): + op(Sample(_image(), "not-a-dict", {})) + + def test_module_imports_without_torchvision(self) -> None: + # The ADAPTER module is entry-pointed for discovery, so importing it must NOT + # pull in torchvision (all library imports are lazy, inside __call__). The + # torchvision_transforms module deliberately DOES import it (generation), with a + # guarded fallback to zero ops when it is absent. + code = "import sys; import sampleflux.ops.torchvision; assert 'torchvision' not in sys.modules" + subprocess.run([sys.executable, "-c", code], check=True, cwd=str(Path(__file__).resolve().parents[1])) + + +class TestGeneratedTorchvisionOps: + v2 = pytest.importorskip("torchvision.transforms.v2") + + def test_family_generated(self) -> None: + import sampleflux.ops.torchvision_transforms as tvt + + assert len(tvt.__all__) > 30 + assert "TvRandomHorizontalFlip" in tvt.__all__ + assert "TvCompose" not in tvt.__all__ # containers excluded — chaining is native + + def test_flip_parity_and_marks(self) -> None: + import sampleflux.ops.torchvision_transforms as tvt + + cls = tvt.TvRandomHorizontalFlip + assert cls.__confluid_category__ == "op" + assert cls.__confluid_group__ == "augment/torchvision" + assert cls.__confluid_random__ is True + assert "__call__" in vars(cls) + out = list(Flux(source=[_sample()], ops=[cls(p=1.0, target="mask")]))[0] + assert np.array_equal(out.input.permute(1, 2, 0).numpy(), _image()[:, ::-1]) + assert np.array_equal(out.target.numpy(), _mask()[:, ::-1]) + + def test_short_name_yaml_loads(self) -> None: + import sampleflux.ops.torchvision_transforms # noqa: F401 (registers the Tv* names) + + op = confluid.load("!class:TvRandomHorizontalFlip\np: 1.0\ntarget: mask\n") + out = op(_sample()) + assert np.array_equal(out.target.numpy(), _mask()[:, ::-1]) diff --git a/tests/test_categories.py b/tests/test_categories.py index 7cfb411..25d7e3d 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -10,6 +10,7 @@ from confluid.registry import get_registry from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp +from sampleflux.ops.albumentations import AlbumentationsOp from sampleflux.ops.configure import ConfigureOp from sampleflux.ops.copy import CopyInputOp from sampleflux.ops.debug import PrintSampleOp @@ -29,6 +30,7 @@ MetadataToTargetOp, ) from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.torchvision import TorchvisionTransformOp from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource from sampleflux.storage.directory import DirectorySink @@ -86,6 +88,16 @@ def test_op_classes_tagged() -> None: assert MasksToDetectionBoxesOp.__confluid_category__ == "op" assert ConfigureOp.__confluid_category__ == "op" assert FormulaOp.__confluid_category__ == "op" + assert AlbumentationsOp.__confluid_category__ == "op" + assert TorchvisionTransformOp.__confluid_category__ == "op" + + +def test_augmentation_adapters_random_tagged() -> None: + """The library-augmentation adapters are stochastic (the wrapped library draws its own + random parameters per call), so they carry ``random=True`` — the confluid mark UIs use + to inject cache-busting (e.g. FluxStudio's ``IS_CHANGED``).""" + assert AlbumentationsOp.__confluid_random__ is True + assert TorchvisionTransformOp.__confluid_random__ is True def test_storage_sink_classes_tagged() -> None: @@ -128,6 +140,8 @@ def test_op_group_tags() -> None: assert ConvertToImageOp.__confluid_group__ == "image" assert NormalizeToUint8Op.__confluid_group__ == "image" assert SampleSinkOp.__confluid_group__ == "sink" + assert AlbumentationsOp.__confluid_group__ == "augment" + assert TorchvisionTransformOp.__confluid_group__ == "augment" def test_categories_enumerable_via_registry() -> None: @@ -185,5 +199,6 @@ def test_groups_enumerable_via_registry() -> None: "CocoToTorchVisionDetectionOp", "MasksToDetectionBoxesOp", } <= registry.list_classes(group="structure") + assert {"AlbumentationsOp", "TorchvisionTransformOp"} <= registry.list_classes(group="augment") # group × category intersect, like task × role. assert "TransformChain" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_enable.py b/tests/test_enable.py index 0c2d8f6..f3df796 100644 --- a/tests/test_enable.py +++ b/tests/test_enable.py @@ -49,6 +49,7 @@ def test_enable_disabled_passes_sample_through_untouched() -> None: counter: Dict[str, int] = {"calls": 0} wrapped = _wrap(_CountingOp(counter), visualize=False) out = wrapped(_sample()) + assert out is not None assert counter["calls"] == 0 assert "counted" not in out.meta @@ -57,6 +58,7 @@ def test_enable_enabled_invokes_inner_op() -> None: counter: Dict[str, int] = {"calls": 0} wrapped = _wrap(_CountingOp(counter), visualize=True) out = wrapped(_sample()) + assert out is not None assert counter["calls"] == 1 assert out.meta["counted"] is True assert wrapped.flag_name == "visualize" @@ -103,6 +105,7 @@ def __call__(self, sample: Sample) -> Sample: visualize=True, ) out = wrapped(_sample()) + assert out is not None assert counter_a["calls"] == 1 assert counter_b["calls"] == 1 assert out.meta["tags"] == ["first", "second"] diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index 57c9e36..8d03c02 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -13,10 +13,12 @@ from confluid import parse_param_docs # type: ignore[import-not-found] from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp +from sampleflux.ops.albumentations import AlbumentationsOp from sampleflux.ops.numpy import ConnectedComponentsOp, StandardizeOp, ThresholdOp from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp from sampleflux.ops.torch import StandardizeOp as TorchStandardizeOp from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.torchvision import TorchvisionTransformOp from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import HuggingFaceSource @@ -35,6 +37,8 @@ EncodeTargetOp, DecodeTargetOp, TransformChain, + AlbumentationsOp, + TorchvisionTransformOp, ] diff --git a/tests/test_random_apply.py b/tests/test_random_apply.py index e18e320..3f166c9 100644 --- a/tests/test_random_apply.py +++ b/tests/test_random_apply.py @@ -19,17 +19,24 @@ def test_zero_arg_construction() -> None: assert RandomApply() is not None +def _run(op: RandomApply, sample: Sample) -> Sample: + """Apply and narrow: these tests never exercise the drop (None) path.""" + out = op(sample) + assert out is not None + return out + + def test_probability_zero_never_applies() -> None: op = RandomApply(op=_BumpOp(), probability=0.0) for _ in range(20): - out = op(_s(0)) + out = _run(op, _s(0)) assert out.input == 0 def test_probability_one_always_applies() -> None: op = RandomApply(op=_BumpOp(), probability=1.0) for _ in range(20): - out = op(_s(0)) + out = _run(op, _s(0)) assert out.input == 1 @@ -61,10 +68,10 @@ def __call__(self, sample: Sample) -> Sample: fluid_op = Class(_Inner) op = RandomApply(op=fluid_op, probability=1.0) - out = op(_s(5)) + out = _run(op, _s(5)) assert out.input == 15 # second call reuses the cached flowed op - out2 = op(_s(5)) + out2 = _run(op, _s(5)) assert out2.input == 15 @@ -94,8 +101,8 @@ def test_gate_reproducible_with_seed() -> None: s = _s(0) op_a = RandomApply(op=_BumpOp(), probability=0.5, random_state=7) op_b = RandomApply(op=_BumpOp(), probability=0.5, random_state=7) - results_a = [op_a(s).input for _ in range(30)] - results_b = [op_b(s).input for _ in range(30)] + results_a = [_run(op_a, s).input for _ in range(30)] + results_b = [_run(op_b, s).input for _ in range(30)] assert results_a == results_b @@ -103,8 +110,8 @@ def test_gate_different_seeds_produce_different_sequences() -> None: s = _s(0) op_a = RandomApply(op=_BumpOp(), probability=0.5, random_state=1) op_b = RandomApply(op=_BumpOp(), probability=0.5, random_state=2) - results_a = [op_a(s).input for _ in range(50)] - results_b = [op_b(s).input for _ in range(50)] + results_a = [_run(op_a, s).input for _ in range(50)] + results_b = [_run(op_b, s).input for _ in range(50)] assert results_a != results_b From 6c10eef80e058577b5387230e6e8aca843835fc3 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 20 Jul 2026 10:52:44 +0200 Subject: [PATCH 025/102] chore: mypy conformance for the generated augmentation-op families - _augment_bridge: targeted type-ignores on dynamic base __init__/property access - albumentations_transforms / torchvision_transforms: module __getattr__ fallback so mypy accepts the generated names; raises a descriptive AttributeError for missing transforms --- sampleflux/ops/_augment_bridge.py | 10 +++++----- sampleflux/ops/albumentations_transforms.py | 11 ++++++++++- sampleflux/ops/torchvision_transforms.py | 13 ++++++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/sampleflux/ops/_augment_bridge.py b/sampleflux/ops/_augment_bridge.py index bbcf447..d58209e 100644 --- a/sampleflux/ops/_augment_bridge.py +++ b/sampleflux/ops/_augment_bridge.py @@ -40,7 +40,7 @@ def _param_specs(transform_cls: type) -> List[inspect.Parameter]: """The transform constructor's named parameters (``self`` and variadics dropped).""" - sig = inspect.signature(transform_cls.__init__) + sig = inspect.signature(transform_cls.__init__) # type: ignore[misc] return [ p for n, p in sig.parameters.items() @@ -96,12 +96,12 @@ def __init__(self: Any, **kwargs: Any) -> None: if unknown: raise TypeError(f"{op_name}: unexpected parameters {sorted(unknown)}") if seed_param: - base.__init__(self, target=kwargs.get("target", "none"), seed=kwargs.get("seed")) + base.__init__(self, target=kwargs.get("target", "none"), seed=kwargs.get("seed")) # type: ignore[misc] else: - base.__init__(self, target=kwargs.get("target", "none")) + base.__init__(self, target=kwargs.get("target", "none")) # type: ignore[misc] for pname in pnames: setattr(self, pname, kwargs.get(pname, defaults[pname])) - self._params_key: Optional[tuple] = None + self._params_key = None # Synthesized signature/annotations: static introspection (to_pydantic / FluxStudio # widgets / parse_param_docs) sees the transform's real parameters, keyword-only, with @@ -133,7 +133,7 @@ def raw_transform(self: Any) -> Any: kwargs[pname] = value return transform_cls(**kwargs) - base_pipeline = base.pipeline.fget + base_pipeline = base.pipeline.fget # type: ignore[attr-defined] def pipeline(self: Any) -> Any: # Rebuild the wrapped transform when any mirrored param changed (post-construction diff --git a/sampleflux/ops/albumentations_transforms.py b/sampleflux/ops/albumentations_transforms.py index 498915a..54f0451 100644 --- a/sampleflux/ops/albumentations_transforms.py +++ b/sampleflux/ops/albumentations_transforms.py @@ -24,7 +24,7 @@ ``AlbumentationsOp(transform=...)``. """ -from typing import List, Tuple +from typing import Any, List, Tuple from loggair import get_logger @@ -34,6 +34,15 @@ logger = get_logger(__name__) +def __getattr__(name: str) -> Any: + # Generated names live in module globals; this fallback only fires for genuinely + # missing ones — and tells mypy the dynamic attributes exist (module-__getattr__ rule). + raise AttributeError( + f"module {__name__!r} has no generated op {name!r} — " + "the albumentations transform may not exist in the installed version." + ) + + def _library_classes() -> List[Tuple[str, type]]: """The concrete, generatable albumentations transform classes, sorted by name.""" import albumentations as A diff --git a/sampleflux/ops/torchvision_transforms.py b/sampleflux/ops/torchvision_transforms.py index 3326e51..2e1edca 100644 --- a/sampleflux/ops/torchvision_transforms.py +++ b/sampleflux/ops/torchvision_transforms.py @@ -24,7 +24,7 @@ ``RandomOrder``) are deliberately NOT generated — chaining ops is native SampleFlux. """ -from typing import List, Tuple +from typing import Any, List, Tuple from loggair import get_logger @@ -33,6 +33,17 @@ logger = get_logger(__name__) + +def __getattr__(name: str) -> Any: + # Generated names live in module globals; this fallback only fires for genuinely + # missing ones — and tells mypy the dynamic attributes exist (module-__getattr__ rule). + raise AttributeError( + f"module {__name__!r} has no generated op {name!r} — torchvision may be missing " + '(install via `pip install "sampleflux[vision]"`) or the transform may not exist ' + "in the installed version." + ) + + #: v2 names not generated: containers (native chaining) + the deprecated v1 shim ToTensor. _EXCLUDED = frozenset({"Compose", "RandomApply", "RandomChoice", "RandomOrder", "ToTensor", "Transform"}) From d738eb6285431165e1856d770441c9485487c791 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 22 Jul 2026 05:49:08 +0200 Subject: [PATCH 026/102] docs: cross-link architecture decision records from user docs; expand discovery docstring - graph.md/kinds.md link the context-wiring and collate-registry rationale to docs/architecture.md - projection.md describes the LabelMap save format generically (docs never name dependents) - discovery.py module docstring expanded (schema surface + MCP end-goal wording) --- docs/graph.md | 2 +- docs/kinds.md | 2 +- docs/projection.md | 2 +- sampleflux/discovery.py | 16 +++++++++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/graph.md b/docs/graph.md index ad248f8..ce6a0ba 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -74,7 +74,7 @@ with activate(Context()): sample = op(sample) ``` -Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's `input`, `Mix` reads each cell's corresponding field. Copy discipline mirrors the stash family: stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to. +Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's `input`, `Mix` reads each cell's corresponding field. Copy discipline mirrors the stash family: stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-sample store instead of `sample.metadata` (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#the-per-sample-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). > **What about the stash family?** `sampleflux.ops.stash` (`StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp`) snapshots a field into `sample.metadata` instead of a cell. It is NOT a wiring mechanism — the context ops are — and remains only for the two jobs cells cannot do: carrying a snapshot **across a `Parallel` boundary** (metadata rides the sample through the stream split; cells deliberately raise there) and deliberately **persisting a snapshot into a sink**. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. diff --git a/docs/kinds.md b/docs/kinds.md index 42d6f08..14d2525 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -82,7 +82,7 @@ def yolo_collate(items): ... loader = DataLoader(flux, collate_fn=get_collate("yolo")) ``` -Defaults: `"sample"` (stacked input/target + list-form batched metadata — the `is_batched` convention), `"pair"` (`(stacked_inputs, stacked_targets)`), `"value"`, and the view forms `"input_meta"`/`"target_meta"`. Consumer collates (classification/segmentation/detection) register additively and keep their own conventions. +Defaults: `"sample"` (stacked input/target + list-form batched metadata — the `is_batched` convention), `"pair"` (`(stacked_inputs, stacked_targets)`), `"value"`, and the view forms `"input_meta"`/`"target_meta"`. Consumer collates (classification/segmentation/detection) register additively and keep their own conventions. The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path; the full rationale is recorded in [architecture.md](architecture.md#batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17). ## 1→N expanding ops (iterable-only pipelines) diff --git a/docs/projection.md b/docs/projection.md index 6441c15..d9f9001 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -37,7 +37,7 @@ from sampleflux import LabelMap, Flux lm = LabelMap.fit(iter_targets(train_source)) # {"bird": 0, "cat": 1, "dog": 2} lm.num_classes # 3 lm.label_names # ["bird", "cat", "dog"] (id -> name) -lm.save("class_names.json") # marainer's class_names.json format +lm.save("class_names.json") # {"class_names": [...], "num_classes": N} encoded = Flux(source=train_source, ops=[lm.encode_op()]) # targets are now ints diff --git a/sampleflux/discovery.py b/sampleflux/discovery.py index 8992cff..c1f31d3 100644 --- a/sampleflux/discovery.py +++ b/sampleflux/discovery.py @@ -12,9 +12,16 @@ * **Discovery** (callable -> JSON schema): :func:`introspect_callable` reflects a single callable into a schema (signature + docstring + the ``ACCEPTS`` / ``PRODUCES`` typespec contract), and :func:`scan_module` does the same for - every callable *defined in* a module. visual editors read these to auto-generate - ComfyUI nodes and their property panels; navigaitor builds its MCP form-spec - from the same data. + every callable *defined in* a module — a visual editor's node bridge reads + these to auto-generate canvas nodes and their property panels. + +The serialization half doubles as the workspace's generic string-callable hook +pattern (:class:`~sampleflux.core.WrappedOp` stores its ``f`` this way; consuming +packages reuse it for their own dotted-path hooks). Curated discovery (MCP +form-specs, option pickers) builds on the Confluid registry instead — which +registers classes AND builder functions, but only opt-in by name; this module is +the registration-free complement (reflect over ANY callable, no curation, plus +the callable→string dump direction the registry doesn't offer). """ import importlib @@ -109,8 +116,7 @@ def introspect_callable(func: Callable) -> Dict[str, Any]: ``*args`` / ``**kwargs``), plus the declared ``ACCEPTS`` / ``PRODUCES`` typespec contract when present. - Use: a visual editor reads this to render a node and its property-panel widgets, - and it feeds navigaitor's MCP form-spec. + Use: a visual editor reads this to render a node and its property-panel widgets. """ try: sig = inspect.signature(func) From ccbbd429e2bdccab26aadf94259b204fd0a7bdd4 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 22 Jul 2026 05:49:31 +0200 Subject: [PATCH 027/102] =?UTF-8?q?feat:=20typed-bag=20data=20model=20(Typ?= =?UTF-8?q?edSample)=20=E2=80=94=20carrier,=20transforms,=20storage,=20eng?= =?UTF-8?q?ines=20(migration=20stages=201-3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The typed bag is THE sampleflux data model (legacy Sample survives only until every consumer migrates; staged plan in the root TASKS.md — purge stage renames TypedSample back to Sample). Stage 1 — frozen typed API + core primitives: - sampleflux.bag: TypedSample (named bag of typed items, role tags input/target/aux/pred), hybrid items (Image/Mask NDArrayItem subclasses w/ attr-preserving __array_finalize__; Regions/Label dataclass wrappers), kernel-registry type dispatch (MRO-aware), Transform/ Pipeline with once-per-sample params + only= key filter - bare library transforms drop into Pipeline via the adapter coercion registry (register_adapter/coerce_transform; torchvision-v2 + albumentations matchers self-register by MRO module name — no eager imports); native HorizontalFlip DELETED (libraries cover augmentation; kernels survive as the tests/_bag_fixtures.py FixtureFlip parity fixture) - full typed surface frozen at the package top level (from sampleflux import TypedSample, ...) - primary(sample, role) + TypedSample.merge (ordered field/role union, last-wins) + rename - bag/io.py item codec registry (EncodedItem/encode_item/decode_item/register_io) extracted from interop (the Sample bridge half stays only until the purge stage) - ops/structure.py: SetRole/RenameField/DropField/CopyField/SelectFields (entry-pointed) - collate 'typed': batched TypedSample (per-field stacked payloads, attrs as lists) — the ONE batch convention replacing list-form and per_sample dict-nest - typespec.infer_field_types (per-field specs beside the legacy SampleType) Stage 2 — typed storage (one field-group layout, three backends): - HDF5/ZarrGroup/Directory write typedsample-v1 (per-field group: item type + role + native scalar attrs + payload dataset + array attrs; JSON-tagged wire format in storage/base.py split_attrs/restore_attrs — tuples survive); one carrier per store (cross-carrier append raises); backends serialize ONLY through the bag.io codec so externally-registered item types round-trip with zero storage edits - DirectorySink gains its missing matching DirectorySource; ZarrBatchSink typed path (primary payload rows + one-time uniform item template) - typed metadata scans yield nested {field: {attr: value}}; MetadataFilterSource.where addresses '.' (query._AttrView); TypedDataSource/TypedDataSink protocols Stage 3 — typed engines: - TypedSample passes VERBATIM on every Flux route (core._as_carrier — no native=True needed) and through _apply_op/_apply_op_native (bag applied verbatim; kinds binding + stored-type refresh are legacy-only); Use/Apply/Capture/_cell_field typed-aware (Apply gains key=; _cell_field on a bag = named item or primary()) - FlowGraph merge_from: typed fan-in (union via TypedSample.merge, slot-order last-wins; mutually exclusive with legacy target_from/metadata_from) lowered to the new MergeFields context op, lifted back by from_ops; bind grammar gains step[key] (bare step = primary) - branch idiom: produce -> SelectFields([new_field]) -> merge_from Docs: typed-model.md rewritten as THE model (engines/storage/query sections), architecture.md record updated, AGENTS mandates rewritten for the migration. 1126 tests green (zero legacy regressions); mypy clean from the workspace root. --- AGENTS.md | 7 +- README.md | 2 + docs/architecture.md | 436 ++++++++++++++++++++++ docs/typed-model.md | 233 ++++++++++++ examples/typed_pipeline.py | 91 +++++ pyproject.toml | 8 + sampleflux/__init__.py | 99 ++++- sampleflux/bag/__init__.py | 96 +++++ sampleflux/bag/adapters/__init__.py | 13 + sampleflux/bag/adapters/albumentations.py | 102 +++++ sampleflux/bag/adapters/torchvision.py | 143 +++++++ sampleflux/bag/dispatch.py | 82 ++++ sampleflux/bag/interop.py | 63 ++++ sampleflux/bag/io.py | 136 +++++++ sampleflux/bag/items.py | 217 +++++++++++ sampleflux/bag/sample.py | 211 +++++++++++ sampleflux/bag/transform.py | 198 ++++++++++ sampleflux/collate.py | 53 ++- sampleflux/core.py | 68 +++- sampleflux/flow.py | 144 +++++-- sampleflux/ops/context.py | 75 +++- sampleflux/ops/structure.py | 132 +++++++ sampleflux/storage/base.py | 107 +++++- sampleflux/storage/directory.py | 100 ++++- sampleflux/storage/hdf5.py | 102 ++++- sampleflux/storage/query.py | 105 +++++- sampleflux/storage/zarr.py | 133 ++++++- sampleflux/typespec.py | 22 ++ tests/_bag_fixtures.py | 77 ++++ tests/test_bag_dispatch.py | 52 +++ tests/test_bag_interop.py | 77 ++++ tests/test_bag_io.py | 93 +++++ tests/test_bag_items.py | 133 +++++++ tests/test_bag_pipeline.py | 186 +++++++++ tests/test_bag_sample.py | 114 ++++++ tests/test_bag_transform.py | 111 ++++++ tests/test_structure_ops.py | 115 ++++++ tests/test_typed_collate.py | 69 ++++ tests/test_typed_flow.py | 225 +++++++++++ tests/test_typed_storage.py | 228 +++++++++++ 40 files changed, 4571 insertions(+), 87 deletions(-) create mode 100644 docs/architecture.md create mode 100644 docs/typed-model.md create mode 100644 examples/typed_pipeline.py create mode 100644 sampleflux/bag/__init__.py create mode 100644 sampleflux/bag/adapters/__init__.py create mode 100644 sampleflux/bag/adapters/albumentations.py create mode 100644 sampleflux/bag/adapters/torchvision.py create mode 100644 sampleflux/bag/dispatch.py create mode 100644 sampleflux/bag/interop.py create mode 100644 sampleflux/bag/io.py create mode 100644 sampleflux/bag/items.py create mode 100644 sampleflux/bag/sample.py create mode 100644 sampleflux/bag/transform.py create mode 100644 sampleflux/ops/structure.py create mode 100644 tests/_bag_fixtures.py create mode 100644 tests/test_bag_dispatch.py create mode 100644 tests/test_bag_interop.py create mode 100644 tests/test_bag_io.py create mode 100644 tests/test_bag_items.py create mode 100644 tests/test_bag_pipeline.py create mode 100644 tests/test_bag_sample.py create mode 100644 tests/test_bag_transform.py create mode 100644 tests/test_structure_ops.py create mode 100644 tests/test_typed_collate.py create mode 100644 tests/test_typed_flow.py create mode 100644 tests/test_typed_storage.py diff --git a/AGENTS.md b/AGENTS.md index 780df9e..92b9c25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,9 +2,10 @@ - **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases). `Tee` threaded the sample through its branches sequentially, making it executionally identical to `TransformChain(ops=[...])` — use `TransformChain` for grouping and the context ops (`Save`/`Use`/`Mix`) for real, isolated fan-out. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom `ConfigureOp(ops=[UnstashInputOp(key)])` is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-sample side-branch (`ops` chain → `metadata[key]` + setattr) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters (fluxstudio export.py AND graphio.py) emit ONLY context ops for wiring; graphio's legacy `__taidal_stash_*` import replay was removed (pre-2026-07 stash-format ops-docs no longer import — re-export from the canvas). Do NOT reintroduce a metadata-bus twin of a context op. -- **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. +- **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. **Scope (2026-07-21):** this mandate governs the CLASSIC engine (`sampleflux.core` / `sampleflux.ops` / `sampleflux.sample`). The experimental typed-bag redesign `sampleflux.bag` (a coexisting proof of concept — see [[typed-bag-model]] below and `docs/architecture.md`) DELIBERATELY introduces a `Transform` base + typed item classes; it does not relax this rule for the classic engine, whose ops stay plain callables. +- **Typed-Bag Model Is THE Data Model — Migration In Progress (`sampleflux.bag`, promoted 2026-07-21):** `sampleflux.bag` is the redesign that steps away from the `Sample(input, target, metadata)` triple, and the user has PROMOTED it from PoC to THE sampleflux data model. The workspace migration is STAGED (see root `TASKS.md` / the migration plan): the typed world is built beside the legacy engine against the FROZEN top-level API (`from sampleflux import TypedSample, Image, Transform, primary, register_io, ...` — new/migrated code imports ONLY from the package top level, never `sampleflux.bag.*` paths, so the eventual promotion of `bag/*` to the package root changes no consumer), consumers flip one project at a time, and a final purge stage deletes the legacy `Sample`/`kinds`/augment layer and renames `TypedSample` → `Sample` workspace-wide. Until that purge stage the legacy mandates below stay in force for `sampleflux.core`/`ops`/`sample`. Model recap: a `TypedSample` is a NAMED BAG of TYPED ITEMS (each owning its metadata), transforms dispatch on item TYPE via a kernel registry (`sampleflux.bag.dispatch`, the torchvision-v2 pattern) sampling params ONCE per sample (so a flip moves image+mask+boxes together), and external libraries (torchvision `transforms.v2`, albumentations) + user types plug in via `sampleflux.bag.adapters` + one-line `@Transform.kernel(ItemType)` registrations. BARE library transforms drop straight into a `Pipeline` — `coerce_transform` wraps any element via a registered matcher/factory (`register_adapter`); the torchvision/albumentations adapters self-register a matcher (by MRO module name, no eager library import) at package load, so `Pipeline([Fourier(), v2.Normalize(...), A.GaussNoise(...)])` works with no explicit adapter wrapper (use the explicit adapter with `only=` for per-key targeting). Items are HYBRID (array items subclass `np.ndarray` w/ attr-preserving `__array_finalize__`; structured items are dataclass wrappers). `input`/`target` are ROLE TAGS on fields, not tuple positions. **Modality-neutral (mandate above):** sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and ships **NO native augmentation transforms** — geometric/photometric augmentation comes from the libraries via coercion (the former native `HorizontalFlip` was DELETED; its kernels survive only as the `tests/_bag_fixtures.py::FixtureFlip` dispatch/parity fixture), and native `Transform`s exist only where no library covers them; the signal-domain items (`Signal`/`Spectrogram`) and the `Fourier` transform live in `waivefront.bag` and register into the SAME `sampleflux.bag` registries on import — do NOT add signal-domain items/transforms here. **Typed engine primitives (2026-07-21, migration Stage 1):** `primary(sample, role)` (first field of a role — THE "the input" accessor for bind/Apply/engines) + `TypedSample.merge(*samples)` (ordered field/role union, last-listed wins on collision — the typed fan-in that replaces metadata dict-merge) + `TypedSample.rename`; the item CODEC registry `sampleflux.bag.io` (`EncodedItem`/`encode_item`/`decode_item`/`register_io` — storage backends call ONLY the codec, so externally-registered item types serialize with zero storage edits); the structure ops `sampleflux.ops.structure` (`SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, entry point `sampleflux-ops-structure` — the typed replacement for triple-slot plumbing); and the `"typed"` collate (`collate.py::typed_collate`, auto-dispatched for `TypedSample` batches) returning a BATCHED TypedSample (payloads stacked per field, per-item attrs as lists, roles preserved) — the ONE batch convention that replaces both the list-form batched metadata and the `{"per_sample": [...]}` dict-nest. **Typed storage (migration Stage 2):** all three backends write a `TypedSample` in the ONE field-group layout (`sampleflux_format="typedsample-v1"`; per sample one group per FIELD: `__item_type__`/`__role__` + plain attrs natively, payload as `data`, array attrs under `attrs/`, insertion order in `__field_order__`; structured attr values ride the JSON-tagged wire format in `storage/base.py::split_attrs`/`restore_attrs` — tuples SURVIVE) — a store holds ONE carrier (typed↔legacy append raises); backends serialize ONLY through the `bag.io` codec so external item types round-trip with zero storage edits; `DirectorySink` gained its missing matching `DirectorySource` (typed layout); `ZarrBatchSink`'s typed path appends the PRIMARY input payload + a one-time uniform item template; typed metadata scans yield NESTED `{field: {attr: value}}` and `MetadataFilterSource.where` addresses it as `.` (`query.py::_AttrView`; a Python-keyword field name is unaddressable in an expression — use `predicate`). NO legacy readers/converter for old datasets (user decision — regenerate from sources). Pins: `tests/test_typed_storage.py`. **Typed engines (migration Stage 3):** a `TypedSample` is NEVER coerced — `core._as_carrier` passes it verbatim on every Flux route (no `native=True` needed), `core._apply_op`/`_apply_op_native` apply ops to the bag verbatim (the kinds binding + `_refresh_type` are legacy-only paths), and `Use`/`Apply`/`Capture`/`_cell_field` are typed-aware (`_cell_field` on a bag = the `key`-named item or `primary()`; `Apply` gained `key=`). FlowGraph: `FlowStep.merge_from` is the TYPED fan-in (UNION of the named steps' fields+roles via `TypedSample.merge`, slot order, last-write-wins; mutually exclusive with `target_from`/`metadata_from`, which stay legacy-only — cross-carrier use raises), lowered to the new `MergeFields` context op (`ops/context.py`, `sources`/`keys`/`drop`) and lifted back by `from_ops`; `bind:` gained the field form `step[key]` (the named item; bare `step` = the primary input item), lowered to `Apply(key=...)`. The derived-field branch idiom: produce → `SelectFields([new_field])` → `merge_from` (a FULL branch bag would last-wins-overwrite shared keys — deliberate). Pins: `tests/test_typed_flow.py`. The legacy engine is otherwise untouched — the "Functional Purity" / "Sample Triplet" / "Stored Type Is Derived" mandates stay in force for `sampleflux.core`/`ops`/`sample` until the purge stage. The subpackage is named `bag`, NOT `typed`, because `sampleflux.typespec.typed` (the `@typed(...)` contract decorator) is re-exported at the package root as `sampleflux.typed` and a `typed/` submodule would shadow it. `sampleflux.bag` imports without torchvision (adapters lazy-import). Entry point `sampleflux-bag-transform`. Usage: `docs/typed-model.md`; rationale: `docs/architecture.md` → "The typed-bag model"; pins: `tests/test_bag_*.py`, `examples/typed_pipeline.py`. Follow-ups (root TASKS.md): torch-Tensor-subclass item base, confluid-native item discovery, generated `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, the `decode` path. - **`Sample.metadata` Is `dict` (single) OR `list[dict]` (batch) — Narrow via `.meta` / `.batch_meta`:** The `metadata` field is `Metadata = Union[Dict[str, Any], List[Dict[str, Any]]]`. A **single** item carries one `dict` (the normal pipeline form every source/op produces and consumes); a **batch** carries a `list` of per-item dicts (one per stacked item), produced by the collate functions (`marainer.collate.collate_fn_with_metadata`, `sonair.classification.classification_collate_fn`) when N samples are stacked into one Sample for the model/loss/predictions-sinks. `Sample.is_batched` (= `isinstance(metadata, list)`) is the single source of truth for telling them apart. Per-sample code MUST read/mutate metadata through the narrowing accessor **`sample.meta`** (returns the dict, raises `TypeError` on a batch) — `sample.meta[key]` / `sample.meta[key] = v`; batch consumers use **`sample.batch_meta`** (returns the list, raises on a single). NEVER index the raw `sample.metadata` Union directly (mypy rejects `Union[...][str]`). NOTE the batch convention is per-collate: marainer/sonair stack into the **list** form (`is_batched` True); deltaid's `segmentation_collate_fn` instead nests under a **dict** `metadata={"per_sample": [...]}` (so `is_batched` is False there — use `.meta["per_sample"]`). `describe()`/`with_type()` operate on single samples only (a batch infers / raises). Pins: `tests/test_sample.py` (batch vs single, `.meta`/`.batch_meta` guards). -- **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. +- **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. (Scope: the CLASSIC engine; the coexisting `sampleflux.bag` PoC replaces the triple with a typed bag — see the "Typed-Bag Redesign" mandate above.) In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. - **The Context Is the Graph Data Plane — Never `sample.metadata` (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its `input`, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `Mix` (fan-in; named slots read cells, empty slots keep the incoming sample; metadata merges incoming-first then slot order, `metadata_from` wins last). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches `sample.metadata` — a linear run's metadata is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`), mirroring `UnstashInputOp(copy=True, remove=True)` — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); the stash family's charter is NARROWED to the two jobs cells cannot do — carrying a snapshot ACROSS a `Parallel` boundary (metadata rides the sample; cells raise at the boundary) and deliberately PERSISTING a snapshot into a sink's metadata (2026-07-18 consolidation); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. - **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `target_from`/`metadata_from` (fan-in slots, Mix field semantics), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. @@ -27,7 +28,7 @@ - **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type Specs Live in `sampleflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). -- **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. +- **Stored Type Is Derived, Never a 4th Field:** (Classic engine; in `sampleflux.bag` type IS the item's Python class, carried per field — see the "Typed-Bag Redesign" mandate.) A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. diff --git a/README.md b/README.md index 0622020..a567e73 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ for sample in flux: | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImageOp`, `NormalizeToUint8Op`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via albumentations / torchvision `transforms.v2` — joint input+target (mask/boxes) adapters, the generated `Alb*`/`Tv*` per-transform ops, seeding, Confluid-native YAML | +| [docs/typed-model.md](docs/typed-model.md) | **Experimental** — the typed-bag model (`sampleflux.bag`): a named bag of typed items (each owning its metadata), type-dispatched transforms, torchvision/albumentations adapters, custom item types | +| [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | ## 🧭 Scope: a modality-neutral engine diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..5948415 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,436 @@ +# Architecture decisions + +The *why* behind sampleflux's non-obvious module boundaries and mechanisms. The user-facing +documentation ([README](../README.md), the per-topic `docs/*.md`) shows **how to use** each surface; +this document records **why the surface is shaped the way it is** — the context, the decision, and +the consequences — so a reader who asks "why does this module exist?" finds the answer here instead +of reverse-engineering it from git history. + +Each entry is a short decision record: **Context → Decision → Consequences → Example → What you may +change**. When a change alters one of these mechanisms, update its record in the same change (see +the workspace `AGENTS.md` → "Architecture Decisions Are Documented"). + +--- + +## Batching is two-stage; collation is a pluggable registry (`sampleflux.collate`, 2026-07-17) + +### Context + +Turning N pipeline items into one batched carrier has two distinct halves: + +1. **Grouping** — the engine yields groups of N items (`Flux.batch` / `FlowGraph.batch` yield + `list`s, and a torch `DataLoader` hands its `collate_fn` a list). +2. **Stacking** — a *collate function* turns one group into one batched carrier (stacked tensors + + batched metadata). + +The engine owns grouping; it must NOT own stacking, because stacking is task-shaped: historically +every consuming project shipped its own task collate (classification, segmentation, detection), +and **two divergent batched-metadata conventions** emerged — the list-form +`Sample(metadata=[...])` batch (`Sample.is_batched` True) versus a dict-nested +`metadata={"per_sample": [...]}` form. In addition, the multi-type carrier engine +(`Flux(native=True)`, see [kinds.md](kinds.md)) meant sampleflux itself needed stacking behavior +*keyed by carrier kind* — a `Sample`, a metadata-free pair, a bare value, and the +`InputMeta`/`TargetMeta` views each batch differently. + +### Decision + +`sampleflux/collate.py` is a **pluggable registry of collate functions keyed by representation**: +`register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`, where an omitted key +dispatches on the *detected* kind of the first item (`sampleflux.kinds.classify_carrier`). +sampleflux registers the five kind defaults (`"sample"`, `"pair"`, `"value"`, `"input_meta"`, +`"target_meta"`); consuming projects may register task aliases (`"yolo"`, `"segmentation"`, …) +**additively**. Re-registering a key deliberately overwrites (logged at debug) so a consumer can +replace a default. + +Two things were deliberately **not** done: + +- **Existing task collates were not moved here.** The registry is an addressable home consumers + can opt into, not a forced migration — consuming projects keep shipping and wiring their own + collate functions directly (e.g. via a Confluid `!ref:` to the function's dotted path). +- **The divergent metadata conventions were not unified.** The dict-nested + `{"per_sample": [...]}` convention stays with the project that owns it; unification is a + tracked follow-up in the root `TASKS.md`, not a side effect of introducing the registry. + +### Primary intended consumer: the MCP tool surface + +The open, string-keyed half of the registry exists first and foremost for **AI-callable tools** +(the workspace converges on an MCP tool surface — see the root `AGENTS.md` end-goal): a JSON tool +argument can carry `"collate": "yolo"` but never a Python function object, and a tool schema can +offer the legal values only if the set is discoverable at runtime (`registered_collates()`). The +registry is the collate layer's MCP-readiness — a stable, JSON-serializable, enumerable name per +batch layout. In ordinary Python (and in YAML via a dotted `!ref:` to the function), passing the +collate function directly remains the normal path; the registry never replaces it. + +### Consequences + +- The engine stays task-agnostic: sampleflux knows *kinds*, never classification/detection/… +- `Flux(native=True)` pipelines and the examples get correct batching per carrier kind with zero + configuration (`DataLoader(flux, collate_fn=get_collate("sample"))`). +- One addressable lookup (`get_collate("yolo")`) replaces scattered cross-package imports — once a + consumer registers. Registration happens at module import, so a key exists only after its + defining module has been imported. +- Batched metadata's list form (`Sample.is_batched`) is produced here, which is why the + `Sample.metadata` `dict | list[dict]` duality exists (see the sampleflux `AGENTS.md` metadata + mandate). +- **Current usage (as of 2026-07-20):** only the five kind defaults are registered; the live call + sites are one training example (`get_collate("sample")` as a `DataLoader` collate) and the test + pins. No consuming project registers or looks up yet — the open registration surface is capacity + held for the MCP tool surface above, and is provisional until that consumer lands. + +### Example + +```python +from torch.utils.data import DataLoader + +from sampleflux import Flux, collate, get_collate, register_collate + +flux = Flux(source=my_source, ops=[...]) + +# Kind-dispatched: Samples stack via the "sample" default (list-form batched metadata). +batch = collate([flux[0], flux[1]]) +assert batch.is_batched + +# Explicit key — the DataLoader glue. +loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("sample")) + + +# A task alias registers additively (runs when the defining module is imported). +@register_collate("yolo") +def yolo_collate(items): + ... # stack to the task's own batch layout + + +loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("yolo")) +``` + +### What you may change (and where it's documented) + +- **Plugging in your own batch layout** is the supported extension point — decorate a function with + `@register_collate("your-key")` and select it via `get_collate`/`collate`. Usage lives in + [kinds.md → the collate registry](kinds.md#multi-type-carriers--the-collate-registry-samplefluxcollate). +- **Changing a default collate's semantics** (e.g. how `"sample"` stacks, or the list-form metadata + convention) is an architectural change: every batch consumer (losses, predictions sinks, + `batch_meta` readers) depends on it. Update this record and the metadata mandate together. + +--- + +## The per-sample Context is an ambient wiring plane (`sampleflux.context`, 2026-07-17) + +### Context + +Graph-shaped pipelines — fan-out, fan-in, cross-branch values — need somewhere to hold a value +between the op that produces it and the op that consumes it. The obvious candidate, +`sample.metadata`, was rejected: metadata is the **accumulating bus that rides inside each +sample** — it persists into sinks, crosses process boundaries, and is part of the sample's +serialized identity, while wiring data is transient scaffolding that should be gone by the end of +a well-formed graph. Three constraints shaped the mechanism: ops keep the plain +`__call__(sample)` signature (no threading a context parameter through every op), the executor +stays a bare `for op in ops` loop (graphs run on the *plain sequential engine*), and a linear +pipeline's behavior — including its metadata, byte-for-byte — must be completely untouched. + +### Decision + +`sampleflux/context.py` is a **per-sample named-cell store activated ambiently**: the engine +creates one fresh `Context` per source item and activates it around the op loop via a +`contextvars.ContextVar`; the six wiring ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`Mix` in +`sampleflux.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature +change anywhere. Deliberate semantics: cells are stored **by reference** and copy-on-read is the +*reading* op's decision (`Use` deep-copies unless `drop` frees the cell = move); a missing cell on +read or delete **raises loudly** with the live-cell list (a liveness bug must never pass +silently); 1→N expansion children get `Context.copy()` (shallow — independent cell *sets*, shared +values); cells may NOT cross a stream-level op boundary (`Parallel` raises on live cells — each +inner chain gets its own contexts). A `Context` is never `@configurable` and never appears in +YAML — it is pure runtime plumbing. The public surface is two-tier by design: the `Context` class +is a package-root export, while `activate`/`current`/`require` stay module-qualified +(`sampleflux.context.…`) — reachable, but visibly plumbing. `FlowGraph` deliberately does NOT use +this module: its named-step documents give the compiler full knowledge of cell lifetimes, so it +manages its own per-sample env directly, held to the context-op semantics by the pinned +flow⇄ops execution-parity contract. + +### Consequences + +- A plain sequential `ops:` list executes a real fan-out/fan-in graph — which is exactly what + graph exporters (a visual canvas, the `flow:` compiler) lower to, so ONE executor serves both + linear and graph pipelines. +- Linear pipelines are provably untouched: no context op ⇒ the Context is created and never used; + the metadata-byte-identical invariant is pinned in `tests/test_context.py`. +- Spawn-parallelism is safe by construction: contexts are created *inside* the worker and never + pickled or shared across processes. +- Ambient state cuts both ways: running an op list containing context ops *outside* an engine + needs an explicit `with activate(Context()):` — forgetting it is a loud, actionable + `RuntimeError`, not silent misbehavior. +- Custom ops can join the wiring plane through the same `require()` seam the built-in six use — + the module being public is what keeps the wiring plane open rather than a closed set of six. + +### Example + +```python +from sampleflux import Flux +from sampleflux.ops.context import Mix, Save + +# Fan-out/fan-in on the PLAIN sequential engine: snapshot → mutate the stream → merge back. +flux = Flux( + source=my_source, + ops=[ + Save(name="clean"), # snapshot the pristine sample into a cell + my_augment_op, # the stream mutates freely + Mix(target_from="clean", drop=["clean"]), # fan-in: target from the snapshot, cell freed + ], +) + +# The same op list outside an engine needs the Context an engine would have created: +from sampleflux.context import Context, activate, require + +with activate(Context()): + for op in ops: + sample = op(sample) + +# A custom op joins the wiring plane through the same seam the built-in six use: +# require("MyOp").get("clean") / require("MyOp").put("my_cell", value) +``` + +### What you may change (and where it's documented) + +- **Writing a custom wiring op** is the supported extension point: call + `require("YourOpName")` inside `__call__`, follow the by-reference/copy-on-read discipline, and + free cells you consume. Usage of the six built-in ops lives in [graph.md](graph.md). +- **Keep the surface narrow.** Don't root-export `activate`/`current`/`require`, and don't grow + `Context` into a general blackboard — anything that should *persist with the sample* belongs on + the metadata bus, not in a cell. +- **Changing cell semantics** (by-reference storage, loud missing-cell errors, the `Parallel` + boundary rule, `copy()` shallowness) is an architectural change: the flow⇄ops parity suite and + the pinned context invariants (`tests/test_context.py`, `tests/test_flow.py`) define the + contract. Update this record and the sampleflux `AGENTS.md` context mandate together. + +--- + +## Callable↔string serialization + passive introspection (`sampleflux.discovery`, recorded 2026-07-20) + +### Context + +Two workspace mandates — *Serialization Symmetry* (every pipeline round-trips through Confluid +YAML) and *Passive Introspection* (tools discover pipeline pieces without hand-written +definitions) — need a bridge the Confluid registry deliberately does not provide. The registry is +a **curated, opt-in catalog**: classes *and* builder functions participate, but only after an +explicit `@configurable`/`register()` (the Registry Discipline mandate), keyed by +name/category/task/role, resolving *strings → callables* for config materialization. What it does +NOT do: produce a string **from** a live callable (the dump direction a bare-function value like a +mapped transform needs), resolve a callable out of a plain `.py` script or `__main__`, or walk a +module to introspect every callable *defined in it* — registered or not. + +### Decision + +`sampleflux/discovery.py` is one small stdlib-only module with **two halves**: + +- **Serialization** — `get_callable_path(fn)` → an importable `"module:qualname"` string + (resolving `__main__` to the script filename so the path survives process boundaries) and + `resolve_callable(path)` back to the live object (module import, `.py`-file load, or an + already-callable passthrough). +- **Introspection** — `introspect_callable(fn)` → a JSON-serializable schema (path, name, doc, + per-parameter type/default/required, the declared `ACCEPTS`/`PRODUCES` typespec contract), and + `scan_module(module_or_py)` applying it to every callable *defined in* a module + (`__module__`-filtered, so imports don't leak in). + +Curated discovery (MCP form-specs, task/category option pickers) deliberately does **not** use +this module — it builds on the Confluid registry, which registers classes AND builder functions, +opt-in by name. This module is the **registration-free complement**: the two surfaces answer +different questions — `scan_module` reflects over *a module, no curation required*; the registry +resolves *a curated name/category*. + +### Consequences + +- `WrappedOp` stores its callable as the string path and resolves it lazily — which is exactly + what makes it pickle across `spawn` workers and serialize into YAML verbatim. +- The dotted-path idiom became the workspace's generic **string-callable hook pattern**: + consuming packages resolve their own hook knobs (metadata encoders, exporter callables) through + `resolve_callable` instead of hand-rolling import dances. +- A visual editor's node bridge scans op/source modules and auto-generates one node (plus its + property-panel widgets) per callable — no manual node definitions anywhere. +- The `__module__ == module` filter in `scan_module` is a real contract: a class defined + elsewhere and merely *imported* into a module is invisible to it (registry-based passes exist + for that case). +- **One acknowledged overlap**: `resolve_callable`'s plain module-import branch resolves the same + importable-function targets confluid's `resolve_class` module-path branch / `!ref:` grammar can + — two spellings of one job (`"module:qualname"` here vs `"module.attr"` there). The + non-overlapping remainder (path *production* via `get_callable_path`, `.py`-file and `__main__` + handling, module scans, `ACCEPTS`/`PRODUCES` schemas) is why the module exists; whether the + resolution half should delegate to confluid is a tracked follow-up in the root `TASKS.md`. + +### Example + +```python +import numpy as np + +from sampleflux.discovery import get_callable_path, resolve_callable, scan_module + +path = get_callable_path(np.sqrt) # "numpy:sqrt" — YAML/pickle-safe identity +fn = resolve_callable(path) # back to the live callable +fn is resolve_callable(fn) # an already-callable argument passes through + +schemas = scan_module("sampleflux.ops.numpy") # one JSON schema per op defined there +``` + +### What you may change (and where it's documented) + +- **Adding a string-callable knob to your own class**: reuse `resolve_callable` (the + `WrappedOp.f` pattern) — never write a bespoke import dance. +- **The `"module:qualname"` format and the module-local scan filter are contracts** — serialized + pipelines and node bridges depend on both; changing either is an architectural change that + must update this record. + +--- + +## The engine's own callable wrappers live in `core.py` (`FilterOp`/`WrappedOp`/`JointFlux`, recorded 2026-07-20) + +### Context + +Three classes sit in `core.py` next to the `Flux` engine that look, at first glance, like they +belong elsewhere: `FilterOp` and `WrappedOp` (op-shaped, so why not `ops/`?) and `JointFlux` +(a second engine in the engine module). + +### Decision + +They stay in `core.py` because of **who constructs them and which way imports flow**. All three +are the construction targets of `Flux`'s own fluent API — `.filter(pred)` appends a `FilterOp`, +`.map(fn)` appends a `WrappedOp`, `Flux.joint([...])` wraps a `JointFlux` — so the engine itself +instantiates them. And `core.py` is the *bottom* of the op-facing layer: every composing op in +`ops/` imports `core._apply_op` (the contract-aware chokepoint); moving `FilterOp`/`WrappedOp` +into `ops/` would make `core` import from `ops` and close an import cycle. `JointFlux` is +`Flux`'s iteration-only fan-in sibling (`category="engine"`), 20 lines that exist to be +`Flux.joint`'s return value — a module of its own would be structure for structure's sake +(`FlowGraph` earns its separate module by size and its own document grammar). + +`FilterOp`/`WrappedOp` carry **no discovery category** on purpose: they wrap a *raw Python +callable*, which no GUI can wire, so they are neither canvas ops nor sources — bare +`@configurable` keeps them YAML-round-trippable while the positive category allowlist keeps them +off visual canvases. + +### Consequences + +- `ops/` stays a pure consumer of `core` — the layering is one-directional. +- `WrappedOp` is a package-root export (the public "lift a plain function" surface, and its + stored-string `f` is the reference use of the discovery serialization half); `FilterOp` is not + root-exported (normally reached via `Flux.filter`; importable as `sampleflux.core.FilterOp`). +- `JointFlux` is YAML-addressable (`!class:sampleflux.core.JointFlux()`) and canvas-composable + as an engine node; its indexable counterpart for raw sources is `ConcatSource`. + +### Example + +```python +flux = ( + Flux(source=src) + .map(np.sqrt) # appends WrappedOp(f="numpy:sqrt") + .filter(lambda s: float(s.input.max()) > 0) # appends FilterOp(p=...) +) +both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flux_a, flux_b])) +``` + +### What you may change (and where it's documented) + +- **A new engine-constructed helper** (another fluent-API target) belongs in `core.py` for the + same import-direction reason; an op users wire *directly* (YAML/canvas) belongs in `ops/` with + a category and group. +- **Do not add a discovery category to `FilterOp`/`WrappedOp`** — surfacing a raw-callable + parameter on a canvas is a dead widget; the taxonomy is pinned in `tests/test_categories.py`. + +--- + +## The typed-bag model: a named bag of typed items (`sampleflux.bag`, PoC, 2026-07-21) + +### Context + +The classic carrier is `Sample(input, target, metadata)` — a fixed 3-tuple where `metadata` is one +flat `dict` shared by the whole sample. Everything that is not literally the model input or target +rides that dict by string key: segmentation masks, `[f0,f1,t0,t1]` region lists, window locators, +`spectrogram_params`, power stats, `snr_db`, a signal's samplerate, an image's canvas size, a +label's class names. Two structural costs follow. First, **metadata has no owner** — `samplerate` +belongs to *the signal*, `canvas` to *the image*, but the flat dict severs that link. Second, **a +transform cannot move several fields together** — flipping an image and its mask and its boxes with +one shared decision is inexpressible when the fields are `input`, `target`, and `metadata["regions"]` +respectively, so today's augmentation adapters hard-code a `TargetMode = Literal["none","mask","boxes"]` +knob per op instead. `target` is also overloaded — sometimes a bare string (`"drone_x"`), sometimes a +`{boxes, labels}` dict. + +### Decision + +`sampleflux.bag` models a sample as a **named bag of typed items with per-field role tags**, and +dispatches transforms on item TYPE via a kernel registry: + +- **Items own their metadata.** An item is a typed value plus the metadata that describes *it* + (`Image(arr, layout)`, `Regions(boxes, labels, canvas)`, `Label(value, classes)`). The + realization is HYBRID: array-backed items (`Image`/`Mask`) subclass `np.ndarray` with + attribute-preserving `__array_finalize__`, so a type-agnostic op touches them as an array; + structured items (`Regions`/`Label`) are dataclass wrappers. A uniform `item_data` / `with_data` + pair hides the difference from kernels. sampleflux ships only MODALITY-NEUTRAL items; signal-domain + items (`Signal`, `Spectrogram`) live in the domain package and register into the same registry (see + "Consequences"). +- **`TypedSample` is a named bag; `input`/`target` are role TAGS, not positions.** A field carries a + role (`input`/`target`/`aux`/`pred`); `inputs()`/`targets()`/`aux()` read them at the + train/collate/sink boundary. A field changes role without moving keys. The sample is immutable + (copy-on-write), mirroring `Sample._replace`. +- **Transforms sample params ONCE, then dispatch a kernel per item type** (the torchvision-v2 + `_KERNEL_REGISTRY` pattern, structurally the same registry idea as `sampleflux.collate`). Kernels + are registered per `(transform, item type)` and resolved by MRO. Targeting is by type, with an + optional `only=[keys]` filter. +- **External libraries plug in through adapters, dropped in BARE.** A `Pipeline` COERCES each element + (`coerce_transform`): a `Transform` is used as-is; a foreign object is wrapped by whichever adapter + a matcher/factory pair claims it (`register_adapter`). The built-in torchvision-v2 and albumentations + adapters register a matcher (by MRO module name — no eager library import) at package load, so + `v2.Normalize(...)` / `A.GaussNoise(...)` go straight into a `Pipeline` with no explicit wrapper. A + plain function becomes a transform via `as_transform`; a new item type is taught to an existing + transform with one `@Transform.kernel(NewType)` registration. This keeps consumer-dialect knowledge + (how to recognise/adapt a library) OUT of the core and open for any user library. + +This is a **coexisting proof of concept**, not a replacement: it lives beside the classic engine and +changes none of it. The existing "Functional Purity", "Sample Triplet", and "Stored Type Is Derived" +mandates are scoped to the classic engine (see `AGENTS.md`), because the typed model deliberately +introduces a `Transform` base and typed item classes. + +### Consequences + +- **Cross-field consistency is free** — one sampled decision flips image + mask + boxes together, + the thing the flat-metadata model could not do. +- **Names and types coexist**, so the "torchvision uses types / albumentations uses names" split is + resolved by one container: the key is the name, the item is the type. +- **The subpackage is `bag`, not `typed`** — `sampleflux.typespec.typed` (the `@typed(...)` contract + decorator) is re-exported at the package root as `sampleflux.typed`, so a `sampleflux/typed/` + submodule would shadow it. `bag` is collision-free; `TypedSample` / dispatch / docstrings carry the + "typed" concept. +- **Batching stays in `sampleflux.collate`** (transforms are per-sample); the classic model's + `list[dict]` batch-in-metadata form is not carried into the bag model. +- **Deliberately deferred** (see root `TASKS.md`): a torch-`Tensor`-subclass item base (torch + payloads ride wrapper items for now), confluid-native item-type discovery, the generated + `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, and the `decode` path. + +### Example + +```python +from sampleflux import TypedSample, Image, Mask, Regions, Label, Pipeline +from torchvision.transforms import v2 +import albumentations as A + +sample = TypedSample( + {"image": Image(rgb), "mask": Mask(seg), "regions": Regions(boxes, canvas=(H, W)), "class": Label("drone_x")}, + roles={"mask": "target", "regions": "target", "class": "target"}, +) +out = Pipeline([ + v2.RandomHorizontalFlip(p=1.0), # Image + Mask + Regions together (one library draw) + v2.Normalize(m, s), # Image (torchvision v2, by type) — wrapped by a registered adapter + A.GaussNoise(p=1.0), # Image (albumentations, by name) — wrapped by a registered adapter +])(sample) +# image flipped+normalized+noised; mask+regions flipped consistently; out["class"] untouched. +# sampleflux ships NO native augmentation transforms — the libraries cover that via coercion. +# Signal-domain items + the Fourier transform live in the domain package and register into the +# same registries — a bare Fourier() drops into this Pipeline with no core edit. +``` + +### What you may change (and where it's documented) + +- **A new item type** — add a class + `@register_item` (usage: [typed-model.md](typed-model.md)); if + it is array-backed, subclass `NDArrayItem` and declare `_item_attrs`. +- **A new per-type behaviour for an existing transform** — register a kernel + (`@Transform.kernel(ItemType)`), no core edit. +- **Do not name the `bag` subpackage `typed`** — it shadows `sampleflux.typed` (the `@typed` + decorator). The rename rationale is pinned here and in the module docstrings. +- **Promoting this from PoC to the default model** is a workspace-wide decision that would re-scope + the classic-engine mandates and port every consumer — out of scope for the proof of concept. diff --git a/docs/typed-model.md b/docs/typed-model.md new file mode 100644 index 0000000..a33375a --- /dev/null +++ b/docs/typed-model.md @@ -0,0 +1,233 @@ +# The typed-bag model — THE sampleflux data model + +> **Status: the data model (migration in progress).** The typed bag replaces the classic +> `Sample(input, target, metadata)` triple; the legacy engine survives only until every consumer +> has migrated (staged in the root `TASKS.md`), after which it is deleted and `TypedSample` is +> renamed `Sample`. Import the typed surface from the PACKAGE TOP LEVEL +> (`from sampleflux import TypedSample, Image, Transform, ...`) — internal module paths are +> transitional. The design rationale is recorded in +> [architecture.md](architecture.md#the-typed-bag-model-a-named-bag-of-typed-items-sampleflux-bag-poc-2026-07-21). + +## Why + +In the classic model everything that is not literally the model input or target — a segmentation +mask, `[f0,f1,t0,t1]` regions, a signal's samplerate, an image's canvas size, a label's class +names — is jammed into one flat `metadata` dict keyed by string, disconnected from the value it +describes. That makes two things hard: metadata has no natural home, and a transform cannot move +several fields together consistently (flip an image → flip its mask → flip its boxes). + +The typed-bag model fixes both: **a sample is a named bag of typed items, and metadata lives on the +item it describes.** Transforms dispatch on item *type*. + +## The pieces + +### Items — typed values that own their metadata + +sampleflux is **modality-neutral**, so its core ships only generic items — images, masks, boxes, +labels. (Signal-domain items live in the domain package; see below.) + +```python +from sampleflux import Image, Mask, Regions, Label + +Image(rgb_hwc, layout="HWC") # an image knows its layout +Mask(seg_hw) # a mask shares its image's frame +Regions(boxes=[[1,1,4,4]], labels=["drone"], canvas=(8, 10)) +Label("drone_x", classes=["noise", "drone_x"]) +``` + +Items are **hybrid**: array-backed items (`Image`, `Mask`) subclass `np.ndarray`, so a +type-agnostic operation touches them as an array and their extra attributes survive numpy ops; +structured items (`Regions`, `Label`) are dataclass wrappers. A uniform payload accessor hides the +difference from kernels: + +```python +from sampleflux import item_data, with_data +item_data(Image(arr)) # -> the plain ndarray +with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved +``` + +### `TypedSample` — a named bag with role tags + +```python +from sampleflux import TypedSample + +sample = TypedSample( + {"image": Image(rgb), "regions": Regions(boxes), "class": Label("drone_x")}, + roles={"regions": "target", "class": "target"}, # default role is "input" +) +sample.inputs() # {"image": Image(...)} +sample.targets() # {"regions": Regions(...), "class": Label(...)} +sample.set_role("regions", "aux") # copy-on-write; a field's role changes without moving keys +``` + +`input` / `target` / `aux` / `pred` are **tags read at the train/collate/sink boundary**, not tuple +positions. `TypedSample` is immutable — every mutator returns a new sample. + +### Transforms — type dispatch with once-per-sample parameters + +A transform samples its parameters once, then applies a per-type kernel to each handled field. +Fields it does not handle pass through. Because the parameters are sampled **once** and shared, +image / mask / boxes move consistently — the thing the flat-metadata model could not express. A +transform may also CHANGE an item's type under the same key (e.g. the domain `Fourier` turns a +`Signal` field into a `Spectrogram` in place). + +**sampleflux ships no native augmentation transforms** — geometric/photometric augmentation comes +from torchvision `transforms.v2` / albumentations through the coercion registry below; native +transforms exist only where no library covers them (domain packages register their own). + +### Mixing libraries — one pipeline, many worlds + +torchvision `transforms.v2` dispatches by type, albumentations by keyword name. **Bare library +transforms drop straight into a `Pipeline`** — a registered adapter wraps each one automatically, and +each transform hits only the field(s) it handles: + +```python +from torchvision.transforms import v2 +import albumentations as A + +Pipeline([ + v2.RandomHorizontalFlip(p=0.5), # torchvision v2: Image + Mask + Regions together (one draw) + v2.Normalize(mean, std), # torchvision v2: Image (wrapped automatically) + A.GaussNoise(p=1.0), # albumentations: Image (wrapped automatically) +])(sample) +``` + +The coercion is a small **registry** (`sampleflux.bag.register_adapter` / `coerce_transform`): the +built-in torchvision-v2 and albumentations adapters register a matcher (by MRO module name, no eager +import) at package load. Teach a `Pipeline` about your own library's transforms with one call: + +```python +from sampleflux import register_adapter +register_adapter(lambda o: type(o).__module__.startswith("mylib"), lambda o: MyLibAdapter(o)) +``` + +For surgical control — target one field key with a library transform — construct the adapter +explicitly: `TorchvisionV2Adapter(v2.Normalize(...), only=["image"])`. + +Runnable end-to-end: [`examples/typed_pipeline.py`](../examples/typed_pipeline.py). + +## Extending it + +### A custom transform from a plain function + +```python +from sampleflux import as_transform, Image +brighten = as_transform(lambda d: d + 0.1, handles=(Image,), only=["image"]) +``` + +### A custom item type + a kernel for an existing transform — no core edit + +```python +from sampleflux import register_item +from mypkg.transforms import MyGeoTransform # any Transform subclass + +@register_item +class Keypoints: + def __init__(self, points): self.points = points + +@MyGeoTransform.kernel(Keypoints) +def _(item, params): + return move_points(item, params) +``` + +Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a +subclass transform inherits its base's kernels until it overrides them. + +### Signal-domain items live in the domain package (`waivefront.bag`) + +This is the same mechanism, applied across packages: because sampleflux is modality-neutral, the +signal-domain `Signal` / `Spectrogram` items and the `Fourier` transform (`Signal` → `Spectrogram`) +live in `waivefront.bag` and register into the SAME registries on import — so a bare `Fourier()` +drops into a `sampleflux.bag.Pipeline` alongside the generic transforms with no core edit. See +`waivefront/examples/05_typed_bag_signal.py`. + +## Engines — Flux and FlowGraph carry the typed bag + +A `TypedSample` is **never coerced**: on every `Flux` route (sequential / parallel / streamed / +`__getitem__`) and in `FlowGraph`, a typed source item passes through verbatim and each op receives +the whole bag (`Pipeline` transforms, structure ops, and the compose plane — `TransformChain`, +`RandomApply`, `Enable`, `Apply`, `Capture` — all route typed carriers correctly). + +```python +Flux(source=typed_source, ops=[v2.RandomHorizontalFlip(p=0.5), Fourier()]).to_sink(HDF5Sink(...)) +``` + +### Typed fan-in (`merge_from`) and field binds (`step[key]`) + +In a `flow:` document, the typed fan-in is **`merge_from`** — the UNION of the named steps' fields +and roles, in slot order, last-write-wins on a key collision (the typed replacement for the legacy +metadata dict-merge). The idiom for a derived-field branch: produce, `SelectFields` the new +field(s), merge: + +```yaml +flow: + start: {} + masked: {op: !class:mypkg.MakeMask(), from: start} + mask_only: {op: !class:sampleflux.ops.structure.SelectFields(keys: [mask]), from: masked} + boosted: {op: !class:mypkg.Boost(), from: start} + out: {from: boosted, merge_from: [mask_only]} +``` + +`bind:` references gain a field form: `step[key]` binds the named ITEM of that step's bag as an op +parameter; a bare `step` reference binds the step's PRIMARY input-role item +(`sampleflux.primary`). Lowering (`to_ops`) compiles `merge_from` to the `MergeFields` context op +and key-binds to `Apply(key=...)`; lifting (`from_ops`) round-trips both. `target_from` / +`metadata_from` stay legacy-`Sample`-only (a typed step using them raises; `merge_from` on a legacy +carrier likewise). + +## Storage — the typed field-group layout + +All three backends (`HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, +`DirectorySink`↔`DirectorySource`) write a `TypedSample` in ONE logical schema: per sample, one +group per FIELD carrying the item's registered type name, its role, the payload as a dataset, and +its attrs (scalars natively — queryable; arrays as sub-datasets; structured values JSON-tagged so +tuples survive). The store is stamped `sampleflux_format = "typedsample-v1"`; a store holds ONE +carrier — appending a legacy `Sample` to a typed store (or vice versa) raises. Backends never +inspect item internals — everything serializes through the `sampleflux.bag.io` codec +(`encode_item`/`decode_item`), so an externally-registered item type round-trips with zero storage +edits; `register_io(MyItem, encode=..., decode=...)` overrides the default structural codec when +needed. + +```python +sink = HDF5Sink(path="out.h5", overwrite=True) +with sink: + for sample in flux: # TypedSamples + sink.write(sample) +back = list(HDF5Source(path="out.h5")) # exact TypedSamples: fields, roles, order, tuple attrs +``` + +`ZarrBatchSink` (the uniform single-array sink) appends the PRIMARY input field's payload per row +and stores a one-time item template — per-sample attr variation needs `ZarrGroupSink`. + +### Querying typed stores without loading arrays + +The metadata scans yield the nested `{field: {attr: value}}` shape, and a `where` expression +addresses it as `.`: + +```python +fast = MetadataFilterSource(source=HDF5Source(path="out.h5"), where="signal.samplerate > 1e6") +``` + +Array-valued attrs appear as shape/dtype stubs (presence/shape testable, never loaded). A field +named like a Python keyword (e.g. `class`) can't be addressed in an expression — use the +programmatic `predicate` or a non-keyword field name. + +## Interop with the classic `Sample` + +Run a typed pipeline against the existing sources/sinks by bridging both ways. The lowering is +lossless — the whole bag is encoded in the legacy metadata while `input` / `target` still expose the +primary payloads for a legacy consumer: + +```python +from sampleflux.bag.interop import to_legacy, to_typed +legacy = to_legacy(sample) # a classic Sample; to_typed(legacy) == sample +typed = to_typed(legacy) # exact reconstruction +# adopting an arbitrary legacy dataset needs a per-dataset builder: +to_typed(raw_sample, builder=lambda s: TypedSample({"image": Image(s.input), "class": Label(s.target)})) +``` + +## What is NOT here yet (follow-ups) + +A torch-`Tensor`-subclass item base (torch payloads currently ride in wrapper items), confluid-native +item-type discovery, the generated per-transform families (`Tv*` / `Alb*`) in this namespace, +FluxStudio typed side sockets, and the `decode` (inverse) path. See the root `TASKS.md`. diff --git a/examples/typed_pipeline.py b/examples/typed_pipeline.py new file mode 100644 index 0000000..71c3693 --- /dev/null +++ b/examples/typed_pipeline.py @@ -0,0 +1,91 @@ +"""The typed-bag data model (proof of concept): a named bag of typed items, type-dispatched transforms. + +Demonstrates the modality-neutral core of the redesign that steps away from +``Sample(input, target, metadata)``: + +1. a ``TypedSample`` is a NAMED BAG of TYPED ITEMS, each owning its metadata — an ``Image`` + carries its layout, a ``Regions`` its canvas, a ``Label`` its classes; ``input`` / + ``target`` are ROLE TAGS, not fixed positions; +2. the HEADLINE — ONE pipeline of BARE library transforms (each wrapped by its registered + adapter): two torchvision ``transforms.v2`` transforms and an albumentations transform, + each hitting only the field(s) of a type it handles. sampleflux ships NO native + augmentation transforms — the libraries cover that through adapter coercion; +3. cross-field consistency — ONE library flip draw moves Image, Mask and Regions together, + the Label untouched; +4. a custom transform from a plain function (``as_transform``), no library, no core edit; +5. interop — lower to a legacy ``Sample`` and lift back losslessly. + +Signal-domain items (``Signal`` / ``Spectrogram``) and the ``Fourier`` transform are NOT here — +sampleflux is modality-neutral. They live in ``waivefront.bag`` and register into the SAME +registry; see ``waivefront/examples/typed_signal_pipeline.py`` for the signal + image mix. + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import albumentations as A +import numpy as np +from torchvision.transforms import v2 + +from sampleflux import Image, Label, Mask, Pipeline, Regions, TypedSample, as_transform +from sampleflux.bag.interop import to_legacy, to_typed + + +def make_sample(rng: np.random.Generator) -> TypedSample: + """A detection sample: an image, its mask, its boxes (targets), and a class label (target).""" + return TypedSample( + { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), + "mask": Mask(rng.random((16, 20)) > 0.5), + "regions": Regions(boxes=[[2, 3, 6, 7]], labels=["drone"], canvas=(16, 20)), + "class": Label("drone_x", classes=["noise", "drone_x"]), + }, + roles={"mask": "target", "regions": "target", "class": "target"}, + ) + + +def main() -> None: + rng = np.random.default_rng(0) + + # 1. The named typed bag with role tags. + sample = make_sample(rng) + print("sample: ", sample) + print("inputs: ", list(sample.inputs()), " targets:", list(sample.targets())) + print("image meta: ", f"layout={sample['image'].layout} regions canvas={sample['regions'].canvas}") + + # 2. HEADLINE — one pipeline of BARE library transforms; a registered adapter wraps each, + # and every transform hits only the field(s) of its type. + out = Pipeline( + [ + v2.RandomHorizontalFlip(p=1.0), # torchvision v2: Image + Mask + Regions together (one draw) + v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.25, 0.25, 0.25]), # torchvision v2: Image + A.GaussNoise(p=1.0), # albumentations: Image + ] + )(sample) + print("\n--- mixed cross-library pipeline (flip + normalize + noise) ---") + print("image ->", type(out["image"]).__name__, np.asarray(out["image"]).shape, "(flipped + normalized + noised)") + print("mask ->", type(out["mask"]).__name__, "(flipped with the image)") + print("regions->", sample["regions"].boxes, "->", [[round(v) for v in b] for b in out["regions"].boxes], "(W=20)") + print("class ->", type(out["class"]).__name__, repr(out["class"].value), "(no handler — untouched)") + assert np.array_equal(np.asarray(out["mask"]), np.asarray(sample["mask"])[:, ::-1]) + assert [round(v) for v in out["regions"].boxes[0]] == [14, 3, 18, 7] + assert out["class"].value == "drone_x" and out.roles == sample.roles + + # 3. A custom transform from a plain function — no library, no core edit. + brighten = as_transform(lambda d: d + 0.1, handles=(Image,), only=["image"]) + brightened = brighten(sample) + print("\n--- custom function transform ---") + print("image brightened:", np.allclose(np.asarray(brightened["image"]), np.asarray(sample["image"]) + 0.1)) + + # 4. Interop — lossless round-trip through the legacy Sample. + legacy = to_legacy(sample) + back = to_typed(legacy) + print("\n--- legacy interop ---") + print("legacy input:", np.asarray(legacy.input).shape, " metadata keys:", list(legacy.meta)) + print("round-trip equal:", back == sample) + assert back == sample + + print("\nOK") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index ed17d62..33bfd50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,14 @@ sampleflux-ops-debug = "sampleflux.ops.debug" sampleflux-storage-hdf5 = "sampleflux.storage.hdf5" sampleflux-storage-zarr = "sampleflux.storage.zarr" sampleflux-storage-directory = "sampleflux.storage.directory" +# The typed-bag model (sampleflux.bag): the Transform/Pipeline/coercion machinery lives in +# sampleflux.bag.transform, which imports without torchvision (adapters lazy-import their +# library). Entry-point changes need an editable reinstall before FluxStudio/navigaitor +# discovery sees the module (`aisland setup`, never --reinstall). +sampleflux-bag-transform = "sampleflux.bag.transform" +# Typed-bag structure ops (SetRole/RenameField/DropField/CopyField/SelectFields) — reshape a +# TypedSample's named fields; the typed replacement for the classic triple-slot plumbing. +sampleflux-ops-structure = "sampleflux.ops.structure" [tool.setuptools.packages.find] where = ["."] diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py index ab083ae..a75db7b 100644 --- a/sampleflux/__init__.py +++ b/sampleflux/__init__.py @@ -1,11 +1,54 @@ """ SampleFlux: Modular, functional data pipelines. + +The TYPED-BAG model (``TypedSample`` + typed items + type-dispatched ``Transform``\\ s) is THE +data model — import its surface from here (``from sampleflux import TypedSample, Image, ...``); +the internal module layout is transitional. The legacy ``Sample`` triple surface below it is +being migrated out and will be deleted once every consumer has flipped. """ +# --- the typed-bag surface (THE data model; frozen — consumers import ONLY from here) ----- +from sampleflux.bag import ( + ROLES, + EncodedField, + EncodedItem, + FunctionTransform, + Image, + Label, + Mask, + NDArrayItem, + Pipeline, + Regions, + Role, + Transform, + TypedSample, + as_transform, + coerce_transform, + decode_item, + decode_sample, + dispatch, + encode_item, + encode_sample, + get_item_type, + is_item, + item_data, + item_type_names, + item_types, + primary, + register_adapter, + register_io, + register_item, + register_kernel, + with_data, +) + +# --- shared infrastructure (carrier-agnostic) ---------------------------------------------- from sampleflux.collate import collate, get_collate, register_collate from sampleflux.context import Context from sampleflux.core import Flux, JointFlux, WrappedOp from sampleflux.flow import FlowGraph, from_ops, to_ops + +# --- LEGACY surface (the Sample triple era — dies with the purge stage) -------------------- from sampleflux.kinds import INPUT, TARGET, Input, OpContract, SampleKind, Target, classify_carrier, op_contract from sampleflux.labels import LabelMap from sampleflux.ops import RescaleOp, StandardizeOp, ToTensorOp @@ -25,19 +68,63 @@ PythonType, SampleType, UnionType, + infer_field_types, infer_sample_type, infer_type, typed, ) __all__ = [ + # ---- typed-bag surface (THE data model) ---- + "TypedSample", + "Role", + "ROLES", + "primary", + "NDArrayItem", + "Image", + "Mask", + "Regions", + "Label", + "register_item", + "item_types", + "item_type_names", + "get_item_type", + "is_item", + "item_data", + "with_data", + "Transform", + "Pipeline", + "FunctionTransform", + "as_transform", + "register_adapter", + "coerce_transform", + "dispatch", + "register_kernel", + "EncodedItem", + "EncodedField", + "register_io", + "encode_item", + "decode_item", + "encode_sample", + "decode_sample", + "infer_field_types", + # ---- shared infrastructure ---- + "Context", + "Flux", + "JointFlux", + "FlowGraph", + "from_ops", + "to_ops", + "collate", + "get_collate", + "register_collate", + "LabelMap", + # ---- legacy surface (dies with the purge stage) ---- "AnyType", "ArrayType", "ConcatSource", - "Context", "DatasetSplit", "Dim", - "FlowGraph", "INPUT", "Input", "InputMeta", @@ -48,20 +135,12 @@ "TargetMeta", "SampleKind", "classify_carrier", - "collate", - "from_ops", - "get_collate", "op_contract", - "register_collate", - "to_ops", "Dtype", "DtypeFamily", "DtypeSpec", - "Flux", "Framework", "HuggingFaceSource", - "JointFlux", - "LabelMap", "ListType", "MappingType", "ProjectionField", diff --git a/sampleflux/bag/__init__.py b/sampleflux/bag/__init__.py new file mode 100644 index 0000000..aa64805 --- /dev/null +++ b/sampleflux/bag/__init__.py @@ -0,0 +1,96 @@ +"""``sampleflux.bag`` — the typed-bag data model with type-dispatched transforms. + +A sample is a NAMED BAG of TYPED ITEMS (:class:`TypedSample`), each item owning its own +metadata; ``input``/``target`` are ROLE TAGS on fields, not tuple positions. Transforms +dispatch on item TYPE via a kernel registry, sampling their parameters once per sample so +multi-field consistency (flip image + mask + boxes together) is automatic. External libraries +(torchvision ``transforms.v2``, albumentations) drop into a :class:`Pipeline` bare — a +registered adapter wraps each — and user/domain packages register their own item types, +kernels, adapters, and storage codecs from outside (``register_item`` / ``@Transform.kernel`` +/ ``register_adapter`` / ``register_io``). + +This is THE sampleflux data model (the legacy ``Sample`` triple is being migrated out; it +survives only until every consumer has flipped). Import the public surface from the PACKAGE +TOP LEVEL (``from sampleflux import TypedSample, Image, Transform, ...``) — the ``bag`` +module path is a transitional home. See ``docs/typed-model.md`` (usage) and +``docs/architecture.md`` (rationale). +""" + +# Import the adapters for their SIDE EFFECT: each registers a coercion matcher so a bare +# torchvision v2 / albumentations transform can be dropped straight into a Pipeline. This does +# NOT import torchvision/albumentations (the adapters lazy-import their library inside method +# bodies), so `import sampleflux.bag` stays library-free — pinned by test_bag_pipeline.py. +from sampleflux.bag import adapters as _adapters # noqa: F401,E402 (registration side effect) +from sampleflux.bag.dispatch import dispatch, register_kernel, registered_kernels +from sampleflux.bag.io import ( + EncodedField, + EncodedItem, + decode_item, + decode_sample, + encode_item, + encode_sample, + register_io, +) +from sampleflux.bag.items import ( + Image, + Label, + Mask, + NDArrayItem, + Regions, + get_item_type, + is_item, + item_data, + item_type_names, + item_types, + register_item, + with_data, +) +from sampleflux.bag.sample import ROLES, Role, TypedSample, primary +from sampleflux.bag.transform import ( + FunctionTransform, + Pipeline, + Transform, + as_transform, + coerce_transform, + register_adapter, +) + +__all__ = [ + # data model + "TypedSample", + "Role", + "ROLES", + "primary", + # items + "NDArrayItem", + "Image", + "Mask", + "Regions", + "Label", + "register_item", + "item_types", + "item_type_names", + "get_item_type", + "is_item", + "item_data", + "with_data", + # transforms + "Transform", + "Pipeline", + "FunctionTransform", + "as_transform", + "register_adapter", + "coerce_transform", + # dispatch + "dispatch", + "register_kernel", + "registered_kernels", + # storage codec + "EncodedItem", + "EncodedField", + "register_io", + "encode_item", + "decode_item", + "encode_sample", + "decode_sample", +] diff --git a/sampleflux/bag/adapters/__init__.py b/sampleflux/bag/adapters/__init__.py new file mode 100644 index 0000000..886061c --- /dev/null +++ b/sampleflux/bag/adapters/__init__.py @@ -0,0 +1,13 @@ +"""Adapters that run external augmentation libraries as typed-bag transforms. + +These are deliberately NOT imported by :mod:`sampleflux.bag`'s top-level ``__init__`` — each +lazy-imports its library inside method bodies, so ``import sampleflux.bag`` stays safe on a +host without torchvision. Import an adapter directly:: + + from sampleflux.bag.adapters import TorchvisionV2Adapter, AlbumentationsAdapter +""" + +from sampleflux.bag.adapters.albumentations import AlbumentationsAdapter +from sampleflux.bag.adapters.torchvision import TorchvisionV2Adapter + +__all__ = ["TorchvisionV2Adapter", "AlbumentationsAdapter"] diff --git a/sampleflux/bag/adapters/albumentations.py b/sampleflux/bag/adapters/albumentations.py new file mode 100644 index 0000000..2881c26 --- /dev/null +++ b/sampleflux/bag/adapters/albumentations.py @@ -0,0 +1,102 @@ +"""``AlbumentationsAdapter`` — run an albumentations transform over a typed bag. + +albumentations dispatches by keyword NAME (``image=`` / ``mask=`` / ``bboxes=``) rather than by +type, so this adapter maps typed items to those named arguments, calls the transform once (one +draw applied jointly), and maps the result back. It targets ONE image field (the first, or the +one selected via ``only``), plus an optional mask and an optional regions field. + +albumentations operates on numpy HWC images and stays numpy HWC. It is a hard dependency but +imported lazily so module import stays light. +""" + +from typing import Any, List, Optional + +import numpy as np + +from sampleflux.bag.items import Image, Mask, Regions, item_data, with_data +from sampleflux.bag.sample import TypedSample +from sampleflux.bag.transform import Transform, register_adapter + + +class AlbumentationsAdapter(Transform): + """Wrap one albumentations transform (or ``A.Compose``) as a typed-bag transform. + + Args: + transform: An albumentations transform / ``A.Compose``. Validated lazily on first call. + only: Restrict to these field keys (still type-gated). + """ + + handles = (Image, Mask, Regions) + consumes = (Image,) + optional = (Mask, Regions) + produces = (Image, Mask, Regions) + + def __init__(self, transform: Optional[Any] = None, only: Optional[List[str]] = None) -> None: + super().__init__(only=only) + self.transform = transform + + def __call__(self, sample: TypedSample) -> TypedSample: + if self.transform is None: + raise ValueError("AlbumentationsAdapter: 'transform' must be set before calling.") + + img_key = self._pick(sample, Image) + if img_key is None: + return sample # albumentations needs an image; nothing to do + mask_key = self._pick(sample, Mask) + reg_key = self._pick(sample, Regions) + + kwargs: dict = {"image": np.asarray(item_data(sample[img_key]))} + if mask_key is not None: + kwargs["mask"] = np.asarray(item_data(sample[mask_key])) + if reg_key is not None: + regions = sample[reg_key] + kwargs["bboxes"] = [list(box) for box in regions.boxes] + kwargs["labels"] = list(regions.labels) if regions.labels is not None else [0] * len(regions.boxes) + + out = self._compose(need_bbox=reg_key is not None)(**kwargs) + + result = sample.replace_field(img_key, with_data(sample[img_key], out["image"])) + if mask_key is not None: + result = result.replace_field(mask_key, with_data(sample[mask_key], out["mask"])) + if reg_key is not None: + regions = sample[reg_key] + result = result.replace_field( + reg_key, + Regions( + boxes=[list(box) for box in out["bboxes"]], + labels=list(out["labels"]), + scores=regions.scores, + canvas=regions.canvas, + ), + ) + return result + + def _pick(self, sample: TypedSample, item_type: type) -> Optional[str]: + """The first field of ``item_type`` (honoring ``only``), or ``None``.""" + for key, item in sample.items(): + if self.only is not None and key not in self.only: + continue + if isinstance(item, item_type): + return key + return None + + def _compose(self, need_bbox: bool) -> Any: + """The live ``A.Compose`` — a prebuilt Compose is used as-is; a bare transform is wrapped.""" + import albumentations as A + from albumentations.core.composition import BaseCompose + + if isinstance(self.transform, BaseCompose): + return self.transform + bbox_params = A.BboxParams(format="pascal_voc", label_fields=["labels"]) if need_bbox else None + return A.Compose([self.transform], bbox_params=bbox_params) + + +def is_albumentations_transform(obj: Any) -> bool: + """True for an albumentations transform / ``Compose`` — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(obj).__mro__) + + +# Drop a bare albumentations transform straight into a Pipeline — wrapped in an AlbumentationsAdapter. +register_adapter(is_albumentations_transform, AlbumentationsAdapter) + +__all__ = ["AlbumentationsAdapter", "is_albumentations_transform"] diff --git a/sampleflux/bag/adapters/torchvision.py b/sampleflux/bag/adapters/torchvision.py new file mode 100644 index 0000000..c5fe467 --- /dev/null +++ b/sampleflux/bag/adapters/torchvision.py @@ -0,0 +1,143 @@ +"""``TorchvisionV2Adapter`` — run a torchvision ``transforms.v2`` transform over a typed bag. + +The typed bag and torchvision's ``tv_tensors`` are the SAME shape — a heterogeneous structure +of typed leaves — so this adapter is thin: it maps our items to ``tv_tensors`` +(:class:`~sampleflux.bag.items.Image`\\ →``Image``, +:class:`~sampleflux.bag.items.Mask`\\ →``Mask``, +:class:`~sampleflux.bag.items.Regions`\\ →``BoundingBoxes``), hands the WHOLE dict to the v2 +transform (v2 draws its random parameters once and applies them across every leaf, so a +geometric augmentation stays consistent across image / mask / boxes), and maps the result +back into typed items with their metadata preserved. + +torchvision is lazy-imported; this module imports without it installed (a missing install +raises a clear error pointing at the ``sampleflux[vision]`` extra). +""" + +from typing import Any, List, Optional, Tuple + +import numpy as np + +from sampleflux.bag.items import Image, Mask, Regions, item_data, with_data +from sampleflux.bag.sample import TypedSample +from sampleflux.bag.transform import Transform, register_adapter + + +def _import_v2() -> Any: + try: + from torchvision.transforms import v2 + except ImportError as exc: # pragma: no cover - exercised only without torchvision + raise ImportError( + "TorchvisionV2Adapter requires torchvision (transforms.v2 / tv_tensors). " + 'Install it via `pip install "sampleflux[vision]"`.' + ) from exc + return v2 + + +class TorchvisionV2Adapter(Transform): + """Wrap one ``transforms.v2`` transform (or ``v2.Compose``) as a typed-bag transform. + + Args: + transform: A ``transforms.v2`` transform / ``v2.Compose``. Validated lazily on first call. + only: Restrict to these field keys (still type-gated). + """ + + handles = (Image, Mask, Regions) + consumes = (Image,) + optional = (Mask, Regions) + produces = (Image, Mask, Regions) + + def __init__(self, transform: Optional[Any] = None, only: Optional[List[str]] = None) -> None: + super().__init__(only=only) + self.transform = transform + + def __call__(self, sample: TypedSample) -> TypedSample: + import torch + from torchvision import tv_tensors + + _import_v2() # raise the actionable extra hint before any torchvision use + if self.transform is None: + raise ValueError("TorchvisionV2Adapter: 'transform' must be set before calling.") + + canvas = _canvas_size(sample) + structure: dict = {} + for key, item in sample.items(): + if self.only is not None and key not in self.only: + continue + wrapped = _wrap(item, tv_tensors, torch, canvas) + if wrapped is not None: + structure[key] = wrapped + if not structure: + return sample + + out_structure = self.transform(structure) + out = sample + for key, wrapped_out in out_structure.items(): + out = out.replace_field(key, _unwrap(sample[key], wrapped_out, torch)) + return out + + +def _canvas_size(sample: TypedSample) -> Optional[Tuple[int, int]]: + """``(H, W)`` from the first Image/Mask field — the reference frame for bounding boxes.""" + for _, item in sample.items(): + if isinstance(item, (Image, Mask)): + arr = item_data(item) + if isinstance(item, Image) and getattr(item, "layout", "HWC") == "CHW" and arr.ndim == 3: + return int(arr.shape[1]), int(arr.shape[2]) + if arr.ndim >= 2: + return int(arr.shape[0]), int(arr.shape[1]) + return None + + +def _wrap(item: Any, tv_tensors: Any, torch: Any, canvas: Optional[Tuple[int, int]]) -> Any: + """Our item → a ``tv_tensors`` carrier (``None`` for a type torchvision does not handle).""" + if isinstance(item, Image): + arr = item_data(item) + tensor = torch.as_tensor(np.ascontiguousarray(arr)) + if getattr(item, "layout", "HWC") == "HWC" and tensor.ndim == 3: + tensor = tensor.permute(2, 0, 1) + if tensor.ndim == 2: + tensor = tensor.unsqueeze(0) + return tv_tensors.Image(tensor) + if isinstance(item, Mask): + return tv_tensors.Mask(torch.as_tensor(np.ascontiguousarray(item_data(item)))) + if isinstance(item, Regions): + size = item.canvas or canvas + if size is None: + raise ValueError( + "TorchvisionV2Adapter: Regions need a canvas (H, W) — set Regions.canvas or include an Image field." + ) + boxes = torch.as_tensor(np.asarray(item.boxes, dtype=np.float32).reshape(-1, 4)) + return tv_tensors.BoundingBoxes(boxes, format="XYXY", canvas_size=size) + return None + + +def _unwrap(original: Any, wrapped_out: Any, torch: Any) -> Any: + """A ``tv_tensors`` result → our item type, metadata preserved.""" + if isinstance(original, Image): + tensor = wrapped_out.as_subclass(torch.Tensor) + arr = tensor.detach().cpu().numpy() + if getattr(original, "layout", "HWC") == "HWC" and arr.ndim == 3: + arr = np.transpose(arr, (1, 2, 0)) + return with_data(original, arr) + if isinstance(original, Mask): + return with_data(original, wrapped_out.as_subclass(torch.Tensor).detach().cpu().numpy()) + if isinstance(original, Regions): + boxes = wrapped_out.as_subclass(torch.Tensor).detach().cpu().numpy().reshape(-1, 4).tolist() + return Regions(boxes=boxes, labels=original.labels, scores=original.scores, canvas=original.canvas) + return original # pragma: no cover - only wrapped types reach here + + +def is_torchvision_v2_transform(obj: Any) -> bool: + """True for a torchvision ``transforms.v2`` transform / ``Compose`` — by MRO module name. + + Inspects the object's own class MRO (which the caller already imported), so it recognises v2 + objects WITHOUT importing torchvision here; v1 ``torchvision.transforms.transforms`` objects do + not match (they don't handle ``tv_tensors``). + """ + return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(obj).__mro__) + + +# Drop a bare v2 transform straight into a Pipeline — it is wrapped in a TorchvisionV2Adapter. +register_adapter(is_torchvision_v2_transform, TorchvisionV2Adapter) + +__all__ = ["TorchvisionV2Adapter", "is_torchvision_v2_transform"] diff --git a/sampleflux/bag/dispatch.py b/sampleflux/bag/dispatch.py new file mode 100644 index 0000000..e51b90b --- /dev/null +++ b/sampleflux/bag/dispatch.py @@ -0,0 +1,82 @@ +"""The kernel registry — type dispatch for transforms (the torchvision-v2 ``_KERNEL_REGISTRY`` pattern). + +A transform does not hard-code how to handle each item type. Instead a kernel is registered +per ``(transform class, item type)`` pair, and :func:`dispatch` looks one up — walking the +item's MRO so a kernel registered for a base item type also serves its subclasses. This is +the same registry idea as :mod:`sampleflux.collate` (batching keyed by representation), +applied to per-type transform behaviour. + +Registration is open: a downstream package teaches an existing transform about a new item +type with one decorator and NO core edit — + + from mypkg.transforms import Denoise # any Transform subclass + from mypkg.items import IQSignal # any registered item type + + @Denoise.kernel(IQSignal) + def _(item, params): + return denoise_iq(item, strength=params["strength"]) + +The transform base exposes ``.kernel(item_type)`` as a thin wrapper over +:func:`register_kernel`; both are documented so either entry point works. +""" + +from typing import Any, Callable, Dict, Optional, Tuple + +__all__ = ["Kernel", "register_kernel", "get_kernel", "dispatch", "registered_kernels"] + +#: A kernel maps ``(item, params) -> item`` — the per-type behaviour of one transform. +Kernel = Callable[[Any, Dict[str, Any]], Any] + +_KERNEL_REGISTRY: Dict[Tuple[type, type], Kernel] = {} +# Memoized MRO-resolution results (``None`` = a resolved miss). Cleared on every registration. +_DISPATCH_CACHE: Dict[Tuple[type, type], Optional[Kernel]] = {} + + +def register_kernel(transform_cls: type, item_cls: type) -> Callable[[Kernel], Kernel]: + """Register a kernel for ``(transform_cls, item_cls)`` (usable as a decorator). + + Re-registering the same pair overwrites (a consumer may deliberately replace a kernel). + """ + + def _register(fn: Kernel) -> Kernel: + _KERNEL_REGISTRY[(transform_cls, item_cls)] = fn + _DISPATCH_CACHE.clear() # a new registration may change what an MRO walk resolves + return fn + + return _register + + +def get_kernel(transform_cls: type, item_cls: type) -> Optional[Kernel]: + """The kernel registered EXACTLY for ``(transform_cls, item_cls)`` (no MRO walk); ``None`` if absent.""" + return _KERNEL_REGISTRY.get((transform_cls, item_cls)) + + +def dispatch(transform_cls: type, item_cls: type) -> Optional[Kernel]: + """The kernel handling ``item_cls`` for ``transform_cls``, resolved by MRO, or ``None``. + + Resolution walks the transform's MRO (a subclass transform inherits its base's kernels + unless it overrides them) and, for each, the item's MRO (a kernel on a base item type + serves subclasses). The MOST specific transform wins; within a transform, the most + specific item type wins. ``None`` means "this transform does not handle this item" — + the caller passes the field through untouched. Results are memoized (see + :data:`_DISPATCH_CACHE`), invalidated on every :func:`register_kernel`. + """ + key = (transform_cls, item_cls) + if key in _DISPATCH_CACHE: + return _DISPATCH_CACHE[key] + resolved: Optional[Kernel] = None + for t_cls in transform_cls.__mro__: + for i_cls in item_cls.__mro__: + kernel = _KERNEL_REGISTRY.get((t_cls, i_cls)) + if kernel is not None: + resolved = kernel + break + if resolved is not None: + break + _DISPATCH_CACHE[key] = resolved + return resolved + + +def registered_kernels() -> Tuple[Tuple[str, str], ...]: + """Every registered ``(transform-name, item-name)`` pair (sorted) — for introspection / tests.""" + return tuple(sorted((t.__name__, i.__name__) for (t, i) in _KERNEL_REGISTRY)) diff --git a/sampleflux/bag/interop.py b/sampleflux/bag/interop.py new file mode 100644 index 0000000..a7572bf --- /dev/null +++ b/sampleflux/bag/interop.py @@ -0,0 +1,63 @@ +"""TEMPORARY bridge between the legacy ``Sample`` triple and :class:`TypedSample`. + +MIGRATION NOTE: this module dies in the purge stage (when legacy ``Sample`` is deleted). +The structural item codec it used to own moved to :mod:`sampleflux.bag.io` (the storage +serializer registry); this file keeps only the legacy-carrier bridge so typed pipelines can +run against not-yet-migrated Sample sources/sinks during the transition. + +The lowering is LOSSLESS — :func:`to_legacy` embeds the encoded typed bag in the legacy +metadata (under :data:`ENCODE_KEY`) while ALSO exposing the primary input / target payloads +on ``Sample.input`` / ``Sample.target`` so a legacy consumer still sees them; +:func:`to_typed` reconstructs the exact bag (``to_typed(to_legacy(x)) == x``). +""" + +from typing import Any, Callable, Dict, List, Optional, Tuple + +from sampleflux.bag.io import EncodedField, EncodedItem, decode_sample, encode_sample +from sampleflux.bag.items import item_data +from sampleflux.bag.sample import TypedSample +from sampleflux.sample import Sample + +__all__ = ["to_legacy", "to_typed", "ENCODE_KEY"] + +#: Metadata key under which :func:`to_legacy` stores the lossless typed-bag encoding. +ENCODE_KEY = "__typed__" + + +def to_legacy(sample: TypedSample) -> Sample: + """Lower a :class:`TypedSample` to a legacy ``Sample`` (lossless; see the module docstring).""" + inputs = sample.inputs() + targets = sample.targets() + legacy_input = item_data(next(iter(inputs.values()))) if inputs else None + legacy_target = item_data(next(iter(targets.values()))) if targets else None + encoded: List[Dict[str, Any]] = [ + {"key": f.key, "role": f.role, "type": f.item.type_name, "payload": f.item.payload, "attrs": f.item.attrs} + for f in encode_sample(sample) + ] + return Sample(input=legacy_input, target=legacy_target, metadata={ENCODE_KEY: {"fields": encoded}}) + + +def to_typed(sample: Sample, builder: Optional[Callable[[Sample], TypedSample]] = None) -> TypedSample: + """Lift a legacy ``Sample`` to a :class:`TypedSample`. + + A sample carrying an embedded encoding (produced by :func:`to_legacy`) is reconstructed + exactly. Otherwise ``builder`` is called to map the sample's fields to typed items; without + one, a clear error is raised (there is no universal legacy→typed mapping). + """ + meta = sample.metadata + if isinstance(meta, dict) and ENCODE_KEY in meta: + fields: Tuple[EncodedField, ...] = tuple( + EncodedField( + key=spec["key"], + role=spec["role"], + item=EncodedItem(type_name=spec["type"], payload=spec["payload"], attrs=spec["attrs"]), + ) + for spec in meta[ENCODE_KEY]["fields"] + ) + return decode_sample(fields) + if builder is not None: + return builder(sample) + raise ValueError( + "to_typed: legacy Sample has no embedded typed encoding — pass builder=... to map its " + "input/target/metadata onto typed items (per-dataset schema)." + ) diff --git a/sampleflux/bag/io.py b/sampleflux/bag/io.py new file mode 100644 index 0000000..63f56c9 --- /dev/null +++ b/sampleflux/bag/io.py @@ -0,0 +1,136 @@ +"""The item codec registry — how a typed item serializes, for EVERY storage backend. + +Storage backends never inspect item internals: they call :func:`encode_item` to get a flat +:class:`EncodedItem` (registered type name + array payload + scalar attrs) and +:func:`decode_item` to rebuild the item. The DEFAULT structural codec covers both item +shapes (an :class:`~sampleflux.bag.items.NDArrayItem` subclass → the array + its declared +attrs; a dataclass wrapper with a ``data`` field → the payload + the remaining fields), so +an externally-registered item type — a domain package's signal item, a user type — serializes +with ZERO storage-code changes. :func:`register_io` overrides the codec for types whose +structure the default cannot capture (e.g. a payload-less wrapper with non-scalar fields). + +The registered TYPE NAME (via :func:`~sampleflux.bag.items.register_item` / +:func:`~sampleflux.bag.items.get_item_type`) is the on-disk type tag — decoding requires the +item type to be registered (imported) in the reading process, exactly like the confluid +``!class:`` contract. +""" + +from dataclasses import dataclass +from dataclasses import fields as dataclass_fields +from dataclasses import is_dataclass +from typing import Any, Callable, Dict, Tuple, cast + +from sampleflux.bag.items import NDArrayItem, get_item_type, item_data +from sampleflux.bag.sample import Role, TypedSample + +__all__ = [ + "EncodedItem", + "EncodedField", + "register_io", + "encode_item", + "decode_item", + "encode_sample", + "decode_sample", +] + + +@dataclass(frozen=True) +class EncodedItem: + """One item, flattened for storage: registered type name + payload + scalar attrs.""" + + type_name: str + payload: Any # ndarray / tensor / scalar / None + attrs: Dict[str, Any] + + +@dataclass(frozen=True) +class EncodedField: + """One named field of a sample: the encoded item plus its key and role.""" + + key: str + role: Role + item: EncodedItem + + +#: Encoder: item -> (payload, attrs). Decoder: (payload, attrs) -> item. +Encoder = Callable[[Any], Tuple[Any, Dict[str, Any]]] +Decoder = Callable[[Any, Dict[str, Any]], Any] + +_CODECS: Dict[type, Tuple[Encoder, Decoder]] = {} + + +def register_io(item_cls: type, *, encode: Encoder, decode: Decoder) -> None: + """Override the codec for ``item_cls`` (exact type — no MRO walk; a codec is a per-type contract). + + ``encode(item) -> (payload, attrs)`` and ``decode(payload, attrs) -> item``. Registering + replaces any previous codec for the type. + """ + _CODECS[item_cls] = (encode, decode) + + +def encode_item(item: Any) -> EncodedItem: + """Flatten one item for storage (registered codec first, else the default structural codec).""" + codec = _CODECS.get(type(item)) + if codec is not None: + payload, attrs = codec[0](item) + return EncodedItem(type_name=type(item).__name__, payload=payload, attrs=attrs) + return EncodedItem(type_name=type(item).__name__, payload=_payload(item), attrs=_attrs(item)) + + +def decode_item(encoded: EncodedItem) -> Any: + """Rebuild an item from its encoded form (the type must be registered in this process).""" + cls = cast(Any, get_item_type(encoded.type_name)) + codec = _CODECS.get(cls) + if codec is not None: + return codec[1](encoded.payload, dict(encoded.attrs)) + if issubclass(cls, NDArrayItem): + return cls(encoded.payload, **encoded.attrs) + if _has_data_field(cls): + return cls(data=encoded.payload, **encoded.attrs) + return cls(**encoded.attrs) + + +def encode_sample(sample: TypedSample) -> Tuple[EncodedField, ...]: + """Encode every field of a sample, in insertion order.""" + return tuple( + EncodedField(key=key, role=sample.role_of(key), item=encode_item(item)) for key, item in sample.items() + ) + + +def decode_sample(fields: Tuple[EncodedField, ...]) -> TypedSample: + """Rebuild a :class:`TypedSample` from encoded fields (order preserved).""" + items: Dict[str, Any] = {} + roles: Dict[str, Role] = {} + for field in fields: + items[field.key] = decode_item(field.item) + roles[field.key] = field.role + return TypedSample(items, roles) + + +# --- the default structural codec ------------------------------------------- +def _payload(item: Any) -> Any: + """The array/data payload to store — the array for array items, ``.data`` for wrappers, else ``None``.""" + if isinstance(item, NDArrayItem): + return item_data(item) + if is_dataclass(item) and not isinstance(item, type) and _has_data_field(type(item)): + return getattr(item, "data") + return None + + +def _attrs(item: Any) -> Dict[str, Any]: + """The reconstruction attributes (everything but the payload).""" + if isinstance(item, NDArrayItem): + return {name: getattr(item, name, None) for name in type(item)._item_attrs} + if is_dataclass(item) and not isinstance(item, type): + return {f.name: getattr(item, f.name) for f in dataclass_fields(item) if f.name != "data"} + return {} + + +def _has_data_field(cls: type) -> bool: + return is_dataclass(cls) and any(f.name == "data" for f in dataclass_fields(cls)) + + +# --- optional helper: an item with a python-object payload (e.g. Regions boxes) ---- +def default_encoded_attrs(item: Any) -> Dict[str, Any]: + """The default codec's attrs view of ``item`` — reusable inside a custom encoder.""" + return _attrs(item) diff --git a/sampleflux/bag/items.py b/sampleflux/bag/items.py new file mode 100644 index 0000000..662e85c --- /dev/null +++ b/sampleflux/bag/items.py @@ -0,0 +1,217 @@ +"""Typed items — the leaves of the typed-bag model, each a value that OWNS its metadata. + +This is the answer to *"metadata belongs to input or target"*: instead of a shared flat +``Sample.metadata`` dict keyed by string, a sample is a bag of typed items and every piece +of metadata lives ON the item it describes — an :class:`Image` carries its ``layout``, a +:class:`Label` its ``classes``, a :class:`Regions` its ``canvas`` reference frame. + +The item model is HYBRID (the workspace decision): + +* **Array-backed items subclass the payload** (:class:`NDArrayItem`, an ``np.ndarray`` + subclass) so a type-agnostic operation touches them AS an array while their extra + attributes survive numpy operations (``__array_finalize__``). ``Image`` / ``Mask`` are + these. +* **Structured items are dataclass wrappers** (:class:`Regions` / :class:`Label`) — a + bounding-box set or a class label is not an array; a wrapper is also the right home for a + payload a domain package does not want to subclass (e.g. complex-IQ signal data, where + subclassing an ``np.complex64`` ndarray and preserving attributes through arithmetic is + fragile). + +This module is MODALITY-NEUTRAL — only generic items live here (images, masks, boxes, +labels). Signal-domain items (a signal, a spectrogram) live in the domain package +(``waivefront.bag``) and register into the SAME registry, per the workspace modality-neutral +mandate. That IS the extensibility story below. + +Both shapes present a uniform payload accessor via :func:`item_data` / :func:`with_data`, so +a transform kernel never has to special-case "is this a subclass or a wrapper". + +Extensibility: any type decorated with :func:`register_item` becomes a first-class item — +the dispatch registry (:mod:`sampleflux.bag.dispatch`) and the graph socket-type map can +see it. A downstream package (a signal item, a torchsig-shaped item, a SigMF recording, a +user type) adds one class + one decorator, no core edit. + +NOTE (PoC scope): array items are ``np.ndarray`` subclasses only; a torch-``Tensor``-subclass +item base (via ``__torch_function__``) is a documented follow-up — torch payloads ride in +wrapper items in the proof-of-concept. Items are registered in the local +:func:`register_item` registry rather than carried on the confluid ``@configurable`` +registry (an ``np.ndarray`` subclass builds through ``__new__``, which fights confluid's +``__init__`` validation wrap); confluid-native item discovery is a follow-up. +""" + +from dataclasses import dataclass, field, fields, is_dataclass, replace +from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, cast + +import numpy as np + +_ItemT = TypeVar("_ItemT") + +__all__ = [ + "NDArrayItem", + "Image", + "Mask", + "Regions", + "Label", + "register_item", + "item_types", + "item_type_names", + "get_item_type", + "is_item", + "item_data", + "with_data", +] + +# --------------------------------------------------------------------------- +# Item registry — the extensibility surface. A registered type is a first-class +# item the dispatch registry and the (design-only) FluxStudio socket-type map see. +# --------------------------------------------------------------------------- +_ITEM_TYPES: Dict[str, type] = {} + + +def register_item(cls: Type[Any]) -> Type[Any]: + """Register ``cls`` as a first-class item type (usable as a class decorator). + + Re-registering the same name overwrites (consumers may deliberately replace a type). + """ + _ITEM_TYPES[cls.__name__] = cls + return cls + + +def item_types() -> Tuple[type, ...]: + """Every registered item type (registration order).""" + return tuple(_ITEM_TYPES.values()) + + +def get_item_type(name: str) -> type: + """The registered item type named ``name`` (a miss names the known types).""" + try: + return _ITEM_TYPES[name] + except KeyError: + known = ", ".join(sorted(_ITEM_TYPES)) or "" + raise KeyError(f"no item type registered as {name!r} (known: {known})") from None + + +def item_type_names() -> Tuple[str, ...]: + """The registered item type NAMES (sorted) — the enumerable socket-type vocabulary.""" + return tuple(sorted(_ITEM_TYPES)) + + +def is_item(obj: Any) -> bool: + """True if ``obj`` is an instance of a registered item type.""" + types = tuple(_ITEM_TYPES.values()) + return bool(types) and isinstance(obj, types) + + +# --------------------------------------------------------------------------- +# Array-backed items — np.ndarray subclasses that preserve their extra attributes. +# --------------------------------------------------------------------------- +class NDArrayItem(np.ndarray): + """Base for array-backed items: an ``np.ndarray`` subclass whose declared extra + attributes (``_item_attrs``) survive numpy operations via ``__array_finalize__``. + + Subclasses declare their metadata attributes as ``_item_attrs`` plus a class-level + default for each:: + + class Image(NDArrayItem): + _item_attrs = ("layout",) + layout = "HWC" + + img = Image(rgb_hwc) # img.layout == "HWC" + img = Image(rgb_chw, layout="CHW") + flipped = np.flip(img, axis=1) # still an Image, flipped.layout == "CHW" + """ + + _item_attrs: Tuple[str, ...] = () + + def __new__(cls, data: Any, **attrs: Any) -> "NDArrayItem": + unknown = set(attrs) - set(cls._item_attrs) + if unknown: + raise TypeError( + f"{cls.__name__}: unexpected attributes {sorted(unknown)} (allowed: {list(cls._item_attrs)})" + ) + obj = np.asarray(data).view(cls) + for name in cls._item_attrs: + setattr(obj, name, attrs[name] if name in attrs else getattr(cls, name, None)) + return obj + + def __array_finalize__(self, obj: Any) -> None: + # Called on every construction path (view, slice, ufunc output). Carry the extra + # attributes forward from the source array (class default when absent). + if obj is None: + return + for name in getattr(type(self), "_item_attrs", ()): + setattr(self, name, getattr(obj, name, getattr(type(self), name, None))) + + +@register_item +class Image(NDArrayItem): + """An image array. ``layout`` is ``"HWC"`` (numpy convention, default) or ``"CHW"``.""" + + _item_attrs = ("layout",) + layout: str = "HWC" + + +@register_item +class Mask(NDArrayItem): + """A segmentation / activity mask array (same spatial frame as its sibling image).""" + + +# --------------------------------------------------------------------------- +# Structured items — dataclass wrappers (not arrays). +# --------------------------------------------------------------------------- +@register_item +@dataclass +class Regions: + """A set of rectangular regions / bounding boxes with optional labels and scores. + + Attributes: + boxes: A list of boxes — pixel ``[x0, y0, x1, y1]`` or signal ``[f0, f1, t0, t1]``. + labels: Optional per-box class labels. + scores: Optional per-box confidence scores. + canvas: Optional ``(H, W)`` reference frame — the coordinate system boxes live in, + so a geometric transform (flip / resize) has a self-contained frame. + """ + + boxes: List[Any] = field(default_factory=list) + labels: Optional[List[Any]] = None + scores: Optional[List[Any]] = None + canvas: Optional[Tuple[int, int]] = None + + +@register_item +@dataclass +class Label: + """A classification label plus its class vocabulary. + + Attributes: + value: The label (a class id or name). + classes: Optional ordered class vocabulary this label indexes into. + """ + + value: Any = None + classes: Optional[List[Any]] = None + + +# --------------------------------------------------------------------------- +# Uniform payload accessors — so kernels never special-case subclass vs wrapper. +# --------------------------------------------------------------------------- +def item_data(item: Any) -> Any: + """The underlying payload of an item: the plain array (array items) or ``.data`` (wrappers).""" + if isinstance(item, NDArrayItem): + return item.view(np.ndarray) + if is_dataclass(item) and any(f.name == "data" for f in fields(item)): + return getattr(item, "data") + return item + + +def with_data(item: _ItemT, new_data: Any) -> _ItemT: + """A copy of ``item`` carrying ``new_data`` as its payload, metadata preserved (same type). + + Works for both shapes: an array item is rebuilt with its declared attributes; a wrapper + with a ``data`` field is ``dataclasses.replace``\\ d. An item with no payload slot raises. + """ + if isinstance(item, NDArrayItem): + attrs = {name: getattr(item, name, None) for name in type(item)._item_attrs} + return cast(_ItemT, type(item)(new_data, **attrs)) + if is_dataclass(item) and not isinstance(item, type) and any(f.name == "data" for f in fields(item)): + return cast(_ItemT, replace(cast(Any, item), data=new_data)) + raise TypeError(f"with_data: {type(item).__name__} has no payload slot to replace") diff --git a/sampleflux/bag/sample.py b/sampleflux/bag/sample.py new file mode 100644 index 0000000..aecf95a --- /dev/null +++ b/sampleflux/bag/sample.py @@ -0,0 +1,211 @@ +"""``TypedSample`` — the named bag of typed items that replaces ``Sample(input, target, metadata)``. + +A sample is an ordered mapping ``name -> item`` (see :mod:`sampleflux.bag.items`), plus a +per-key ROLE tag. This gives every field BOTH a name (the key — the albumentations dispatch +axis) and a type (the item — the torchvision dispatch axis), and it makes ``input`` / ``target`` +ordinary tags read only at the train / collate / sink boundary rather than fixed tuple +positions. A field can change role without moving keys; auxiliary items (masks, derived +params) are simply tagged ``aux`` and excluded from both ``inputs()`` and ``targets()``. + +``TypedSample`` is immutable — every mutator returns a NEW sample (copy-on-write), mirroring +the ``Sample._replace`` idiom the legacy engine already relies on, so a transform never +aliases its input. +""" + +from typing import Any, Dict, Iterator, Mapping, Optional, Tuple + +import numpy as np +from typing_extensions import Literal, get_args + +__all__ = ["TypedSample", "Role", "ROLES", "primary"] + +#: The closed set of field roles. ``input`` / ``target`` drive the train boundary; ``aux`` is +#: a helper field (mask, derived param) in neither; ``pred`` is a model prediction. Closed +#: ``Literal`` so a typo fails at the call site and UIs enumerate the choices via ``get_args``. +Role = Literal["input", "target", "aux", "pred"] +ROLES: Tuple[str, ...] = get_args(Role) + +_DEFAULT_ROLE: Role = "input" + + +class TypedSample: + """An ordered, immutable bag of typed items with per-field role tags. + + Construct from a mapping of items (roles default to ``input``); pass ``roles`` to tag + specific keys:: + + s = TypedSample( + {"image": Image(rgb), "regions": Regions(boxes), "class": Label("drone")}, + roles={"regions": "target", "class": "target"}, + ) + s.inputs() # {"image": Image(...)} + s.targets() # {"regions": Regions(...), "class": Label(...)} + s2 = s.set_role("regions", "aux") # copy-on-write + """ + + __slots__ = ("_fields", "_roles") + + def __init__( + self, + fields: Optional[Mapping[str, Any]] = None, + roles: Optional[Mapping[str, Role]] = None, + ) -> None: + self._fields: Dict[str, Any] = dict(fields or {}) + roles = roles or {} + for key, role in roles.items(): + if key not in self._fields: + raise KeyError(f"TypedSample: role given for unknown field {key!r}") + if role not in ROLES: + raise ValueError(f"TypedSample: invalid role {role!r} for {key!r} (allowed: {list(ROLES)})") + self._roles: Dict[str, Role] = {key: roles.get(key, _DEFAULT_ROLE) for key in self._fields} + + # --- read views ------------------------------------------------------- + @property + def fields(self) -> Dict[str, Any]: + """A shallow copy of the ``name -> item`` mapping (mutating it does not touch the sample).""" + return dict(self._fields) + + @property + def roles(self) -> Dict[str, Role]: + """A shallow copy of the ``name -> role`` mapping.""" + return dict(self._roles) + + def __getitem__(self, key: str) -> Any: + return self._fields[key] + + def __contains__(self, key: object) -> bool: + return key in self._fields + + def __iter__(self) -> Iterator[str]: + return iter(self._fields) + + def __len__(self) -> int: + return len(self._fields) + + def keys(self) -> Iterator[str]: + return iter(self._fields) + + def items(self) -> Iterator[Tuple[str, Any]]: + return iter(self._fields.items()) + + def role_of(self, key: str) -> Role: + """The role tag of ``key``.""" + return self._roles[key] + + def of_role(self, role: Role) -> Dict[str, Any]: + """The ``name -> item`` fields tagged ``role`` (insertion order preserved).""" + return {key: item for key, item in self._fields.items() if self._roles[key] == role} + + def inputs(self) -> Dict[str, Any]: + """The fields tagged ``input`` — what the model consumes.""" + return self.of_role("input") + + def targets(self) -> Dict[str, Any]: + """The fields tagged ``target`` — what the loss consumes.""" + return self.of_role("target") + + def aux(self) -> Dict[str, Any]: + """The fields tagged ``aux`` — helpers in neither inputs nor targets.""" + return self.of_role("aux") + + def items_of_type(self, *types: type) -> Iterator[Tuple[str, Any]]: + """Yield ``(key, item)`` for every field whose item is an instance of one of ``types``.""" + for key, item in self._fields.items(): + if isinstance(item, types): + yield key, item + + # --- copy-on-write mutators ------------------------------------------ + def replace_field(self, key: str, item: Any) -> "TypedSample": + """A copy with ``key`` set to ``item`` (added if new; role preserved, else ``input``).""" + fields = dict(self._fields) + fields[key] = item + return TypedSample(fields, {**self._roles, key: self._roles.get(key, _DEFAULT_ROLE)}) + + def set_role(self, key: str, role: Role) -> "TypedSample": + """A copy with ``key``'s role set to ``role``.""" + if key not in self._fields: + raise KeyError(f"TypedSample.set_role: unknown field {key!r}") + if role not in ROLES: + raise ValueError(f"TypedSample.set_role: invalid role {role!r} (allowed: {list(ROLES)})") + return TypedSample(dict(self._fields), {**self._roles, key: role}) + + def drop(self, key: str) -> "TypedSample": + """A copy without ``key``.""" + fields = dict(self._fields) + roles = dict(self._roles) + fields.pop(key, None) + roles.pop(key, None) + return TypedSample(fields, roles) + + def rename(self, src: str, dst: str) -> "TypedSample": + """A copy with field ``src`` renamed to ``dst`` (role travels; position moves to the end). + + Renaming onto an existing ``dst`` replaces it (last-write-wins, consistent with + :meth:`merge`). Unknown ``src`` raises. + """ + if src not in self._fields: + raise KeyError(f"TypedSample.rename: unknown field {src!r}") + fields = dict(self._fields) + roles = dict(self._roles) + item = fields.pop(src) + role = roles.pop(src) + fields.pop(dst, None) + roles.pop(dst, None) + fields[dst] = item + roles[dst] = role + return TypedSample(fields, roles) + + # --- fan-in ------------------------------------------------------------ + @classmethod + def merge(cls, *samples: "TypedSample") -> "TypedSample": + """The ordered UNION of several samples' fields — the typed fan-in primitive. + + Fields AND their roles are united in listed order; on a key collision the + LAST-listed sample wins (value and role) — the deterministic slot-order rule that + replaces the classic metadata dict-merge. Avoid a deliberate collision by renaming + on the producing branch (:meth:`rename` / the ``RenameField`` op), not with merge + policy knobs. + """ + fields: Dict[str, Any] = {} + roles: Dict[str, Role] = {} + for sample in samples: + if not isinstance(sample, TypedSample): + raise TypeError(f"TypedSample.merge: expected TypedSample, got {type(sample).__name__}") + fields.update(sample._fields) + roles.update(sample._roles) + return cls(fields, roles) + + # --- equality / repr -------------------------------------------------- + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypedSample): + return NotImplemented + if self._roles != other._roles or list(self._fields) != list(other._fields): + return False + return all(_field_equal(self._fields[k], other._fields[k]) for k in self._fields) + + def __repr__(self) -> str: + parts = ", ".join(f"{key}={type(item).__name__}[{self._roles[key]}]" for key, item in self._fields.items()) + return f"TypedSample({parts})" + + +def primary(sample: TypedSample, role: Role = "input") -> Tuple[str, Any]: + """The FIRST field of ``role`` in insertion order, as ``(key, item)``. + + The sanctioned answer to "the input" / "the target" of a bag: engines, ``bind``, and + ``Apply``-style parameter injection use it when no explicit field key is given. Raises + ``KeyError`` (naming the sample's fields) when no field carries the role. + """ + for key, item in sample.items(): + if sample.role_of(key) == role: + return key, item + raise KeyError(f"primary: no field with role {role!r} (fields: {list(sample.keys()) or ''})") + + +def _field_equal(a: Any, b: Any) -> bool: + """Value equality that is robust to array-valued items (elementwise ``==`` is not a bool).""" + if isinstance(a, np.ndarray) or isinstance(b, np.ndarray): + return type(a) is type(b) and np.array_equal(np.asarray(a), np.asarray(b)) + try: + return bool(a == b) + except Exception: # pragma: no cover - exotic payloads fall back to identity + return a is b diff --git a/sampleflux/bag/transform.py b/sampleflux/bag/transform.py new file mode 100644 index 0000000..e88c05d --- /dev/null +++ b/sampleflux/bag/transform.py @@ -0,0 +1,198 @@ +"""``Transform`` — type-dispatched sample transforms with once-per-sample parameters. + +A transform samples its random / configured parameters ONCE per sample (:meth:`Transform.get_params`), +then walks the bag and, for each field whose item type it handles, applies the registered +kernel (:mod:`sampleflux.bag.dispatch`). Fields it does not handle pass through untouched. + +Two properties fall out of this shape for free: + +* **Cross-field consistency.** Because params are sampled once and shared, one transform + moves every spatial field with the SAME decision (a torchvision-v2 flip dropped into a + :class:`Pipeline` flips an :class:`~sampleflux.bag.items.Image`, its + :class:`~sampleflux.bag.items.Mask`, and its :class:`~sampleflux.bag.items.Regions` + together) — the thing the old flat-metadata model could not express. +* **Open extension.** A new item type is taught to an existing transform with one + ``@Transform.kernel(NewType)`` registration and no core edit. + +Targeting is by TYPE, with an optional ``only=[keys]`` filter for surgical control (touch +only the named fields even if others share a handled type). + +sampleflux ships NO native augmentation transforms — geometric/photometric augmentation +comes from the libraries (torchvision ``transforms.v2`` / albumentations) through the +adapter coercion registry below; a domain package registers its own transforms (e.g. a +signal FFT) via the same ``Transform`` + kernel machinery from outside. + +Graph annotations (``consumes`` / ``optional`` / ``produces`` — item-type tuples) describe a +transform's item-level inputs/outputs for a visual editor's typed side sockets; they are +declarative metadata, not enforced at runtime here. +""" + +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from sampleflux.bag.dispatch import Kernel, dispatch, register_kernel +from sampleflux.bag.items import item_data, with_data +from sampleflux.bag.sample import TypedSample + +__all__ = [ + "Transform", + "Pipeline", + "FunctionTransform", + "as_transform", + "register_adapter", + "coerce_transform", +] + + +class Transform: + """Base class for type-dispatched transforms (see the module docstring). + + Subclasses declare ``handles`` (the item types they process) and register a kernel per + type via ``@MyTransform.kernel(ItemType)``. Override :meth:`get_params` to sample shared + parameters once per sample. + + Args: + only: Restrict the transform to these field keys (still type-gated). ``None`` = every + field of a handled type. + """ + + #: Item types this transform processes (a field of another type passes through). + handles: Tuple[type, ...] = () + #: Graph metadata — required input item types (defaults to ``handles`` when empty). + consumes: Tuple[type, ...] = () + #: Graph metadata — optional input item types. + optional: Tuple[type, ...] = () + #: Graph metadata — item types this transform adds or changes. + produces: Tuple[type, ...] = () + + def __init__(self, only: Optional[List[str]] = None) -> None: + self.only = list(only) if only else None + + @classmethod + def kernel(cls, item_type: type) -> Callable[[Kernel], Kernel]: + """Register a kernel for ``item_type`` on this transform (decorator over :func:`register_kernel`).""" + return register_kernel(cls, item_type) + + def get_params(self, sample: TypedSample) -> Dict[str, Any]: + """Sample the shared parameters for one call. Default: no params.""" + return {} + + def __call__(self, sample: TypedSample) -> TypedSample: + params = self.get_params(sample) + out = sample + for key, item in sample.items(): + if self.only is not None and key not in self.only: + continue + kernel = dispatch(type(self), type(item)) + if kernel is None: + continue + out = out.replace_field(key, kernel(item, params)) + return out + + def decode(self, sample: TypedSample) -> TypedSample: + """The inverse transform (for visualization / back-projection). Not defined by default.""" + raise NotImplementedError(f"{type(self).__name__} defines no decode (inverse)") + + +# --------------------------------------------------------------------------- +# Adapter coercion registry — drop a FOREIGN transform (a torchvision v2 transform, +# an albumentations transform, a user library object) straight into a Pipeline and the +# right adapter wraps it. Open for extension: register a matcher + factory for any type. +# --------------------------------------------------------------------------- +#: A matcher decides whether an object is adaptable; a factory wraps it into a Transform. +AdapterMatcher = Callable[[Any], bool] +AdapterFactory = Callable[[Any], "Transform"] + +_ADAPTERS: List[Tuple[AdapterMatcher, AdapterFactory]] = [] + + +def register_adapter(matcher: AdapterMatcher, factory: AdapterFactory) -> AdapterFactory: + """Teach :func:`coerce_transform` (and thus ``Pipeline``) to adapt a foreign transform type. + + ``matcher(obj) -> bool`` recognises the objects this adapter handles (keep it import-free — + inspect ``type(obj).__mro__`` module names rather than importing the library); ``factory(obj)`` + returns a :class:`Transform` wrapping it. Later registrations win on ties (checked last-first). + + Example — make a user's library transforms droppable into a ``Pipeline``:: + + register_adapter( + lambda o: type(o).__module__.startswith("mylib"), + lambda o: MyLibAdapter(o), + ) + """ + _ADAPTERS.append((matcher, factory)) + return factory + + +def coerce_transform(obj: Any) -> "Transform": + """Return ``obj`` if it is already a :class:`Transform`, else adapt it via a registered adapter. + + Raises a clear ``TypeError`` naming the object when no adapter matches (wrap it with + :func:`as_transform` / an explicit adapter, or :func:`register_adapter`). + """ + if isinstance(obj, Transform): + return obj + for matcher, factory in reversed(_ADAPTERS): + try: + matched = matcher(obj) + except Exception: # pragma: no cover - a defensive matcher never breaks coercion + matched = False + if matched: + return factory(obj) + raise TypeError( + f"Pipeline: don't know how to adapt {type(obj).__module__}.{type(obj).__name__} into a " + "Transform. Wrap it with as_transform(...) or an adapter, or register one via " + "sampleflux.bag.register_adapter(matcher, factory)." + ) + + +class Pipeline: + """Sequential application of transforms — ``Pipeline([a, b, c])(sample)`` is ``c(b(a(sample)))``. + + Elements are COERCED (:func:`coerce_transform`): a :class:`Transform` is used as-is, and a + foreign transform (a torchvision ``transforms.v2`` transform, an albumentations transform, a + registered user type) is wrapped by its adapter automatically — so libraries drop straight in:: + + Pipeline([v2.RandomHorizontalFlip(p=0.5), v2.Normalize(mean, std), A.GaussNoise(p=1.0)])(sample) + + For surgical control (targeting one field key), construct the adapter explicitly with ``only=``. + """ + + def __init__(self, transforms: Sequence[Any]) -> None: + self.transforms: List[Transform] = [coerce_transform(t) for t in transforms] + + def __call__(self, sample: TypedSample) -> TypedSample: + for transform in self.transforms: + sample = transform(sample) + return sample + + def __repr__(self) -> str: + return f"Pipeline([{', '.join(type(t).__name__ for t in self.transforms)}])" + + +class FunctionTransform(Transform): + """A transform that applies one plain function ``fn(data) -> data`` to every handled field. + + The escape hatch for custom transforms: no kernel registration, no subclass — wrap a + function and say which item types it applies to (via :func:`as_transform`). + """ + + def __init__(self, fn: Callable[[Any], Any], handles: Sequence[type], only: Optional[List[str]] = None) -> None: + super().__init__(only=only) + self._fn = fn + self.handles = tuple(handles) + + def __call__(self, sample: TypedSample) -> TypedSample: + out = sample + for key, item in sample.items(): + if self.only is not None and key not in self.only: + continue + if isinstance(item, self.handles): + out = out.replace_field(key, with_data(item, self._fn(item_data(item)))) + return out + + +def as_transform( + fn: Callable[[Any], Any], handles: Sequence[type], only: Optional[List[str]] = None +) -> FunctionTransform: + """Wrap a plain ``fn(data) -> data`` as a :class:`FunctionTransform` over ``handles``.""" + return FunctionTransform(fn, handles, only=only) diff --git a/sampleflux/collate.py b/sampleflux/collate.py index 6ffd2a3..002784e 100644 --- a/sampleflux/collate.py +++ b/sampleflux/collate.py @@ -8,6 +8,11 @@ ``register_collate`` their task collates additively, and callers dispatch by key or by the DETECTED carrier kind (:func:`sampleflux.kinds.classify_carrier`). +The string keys primarily serve AI-callable (MCP) tool surfaces, which pass +JSON-serializable names — never function objects — and enumerate the legal values via +:func:`registered_collates`; in Python (and in YAML via a dotted ``!ref:`` to the +function), passing a collate function directly remains the normal path. + Defaults registered here: - ``"sample"`` — stacks ``input``/``target`` (torch-first, numpy fallback, else kept as a @@ -75,11 +80,17 @@ def collate(items: Sequence[Any], key: Optional[str] = None) -> Any: """Collate ``items`` into one batched carrier. ``key`` picks a registered collate explicitly; omitted, the DETECTED kind of the - first item dispatches (``sample`` / ``pair`` / ``value``). An empty batch raises. + first item dispatches — a ``TypedSample`` batch routes to ``"typed"``, everything + else through the classic carrier classifier (``sample`` / ``pair`` / ``value``). + An empty batch raises. """ + from sampleflux.bag.sample import TypedSample + if not items: raise ValueError("collate: cannot collate an empty batch") - return get_collate(key or classify_carrier(items[0]))(items) + if key is None: + key = "typed" if isinstance(items[0], TypedSample) else classify_carrier(items[0]) + return get_collate(key)(items) def _stack(values: List[Any]) -> Any: @@ -137,3 +148,41 @@ def input_meta_collate(items: Sequence[Any]) -> InputMeta: def target_meta_collate(items: Sequence[Any]) -> TargetMeta: """Default TargetMeta collate: stacked targets + the per-item metadata dicts as a list.""" return TargetMeta(_stack([item.target for item in items]), [dict(item.metadata) for item in items]) + + +@register_collate("typed") +def typed_collate(items: Sequence[Any]) -> Any: + """The typed-bag collate: N ``TypedSample``\\ s → ONE batched ``TypedSample``. + + Per field (union of keys is NOT taken — every sample must carry the same fields, a + mismatch raises): payloads are stacked via :func:`_stack` (torch → stacked tensor, + numpy → stacked array, else a list) and each declared item attr becomes a LIST of + per-item values. Array items come back as the SAME item type over the stacked payload; + wrapper items likewise (attrs as lists). Roles are preserved. This single convention + replaces both classic batch shapes (the list-form batched metadata and the + ``{"per_sample": [...]}`` dict-nest) — trainers read ``primary(batch)`` / + ``batch.targets()``. + """ + from sampleflux.bag.io import EncodedItem, decode_item, encode_item + from sampleflux.bag.sample import TypedSample + + if not items: + raise ValueError("typed_collate: cannot collate an empty batch") + first = items[0] + if not isinstance(first, TypedSample): + raise TypeError(f"typed_collate: expected TypedSample items, got {type(first).__name__}") + keys = list(first.keys()) + for i, sample in enumerate(items): + if not isinstance(sample, TypedSample) or list(sample.keys()) != keys: + raise ValueError( + f"typed_collate: item {i} fields {list(sample.keys()) if isinstance(sample, TypedSample) else '?'} " + f"do not match the batch fields {keys} — collate requires a homogeneous batch." + ) + fields: Dict[str, Any] = {} + for key in keys: + encoded = [encode_item(sample[key]) for sample in items] + type_name = encoded[0].type_name + stacked_payload = _stack([e.payload for e in encoded]) if encoded[0].payload is not None else None + batched_attrs = {name: [e.attrs.get(name) for e in encoded] for name in encoded[0].attrs} + fields[key] = decode_item(EncodedItem(type_name=type_name, payload=stacked_payload, attrs=batched_attrs)) + return TypedSample(fields, {key: first.role_of(key) for key in keys}) diff --git a/sampleflux/core.py b/sampleflux/core.py index 9ff9391..d33ea3d 100644 --- a/sampleflux/core.py +++ b/sampleflux/core.py @@ -26,6 +26,7 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger +from sampleflux.bag.sample import TypedSample from sampleflux.context import Context, activate from sampleflux.projection import ProjectionField from sampleflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, InputMeta, Pair, Sample, TargetMeta @@ -61,6 +62,18 @@ def _refresh_type(sample: Sample, op: Any) -> Sample: return sample._replace(metadata={k: v for k, v in sample.meta.items() if k not in TYPE_KEYS}) +def _as_carrier(item: Any, native: bool) -> Any: + """The stream carrier for a raw source item. + + A :class:`TypedSample` passes VERBATIM on every route — the typed carrier is + first-class and is never coerced into the legacy triple. ``native`` mode keeps any + other carrier as-is (pair/value lanes); the default coerces to the legacy ``Sample``. + """ + if native or isinstance(item, TypedSample): + return item + return Sample.from_any(item) + + def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: """Apply one op and refresh the stored type. The single op-application chokepoint shared by the sequential, parallel (via :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths. @@ -73,6 +86,11 @@ def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: """ from sampleflux.kinds import op_contract + if isinstance(sample, TypedSample): + # The TYPED carrier: transforms take the whole bag verbatim — the kinds field-scope + # binding and the reserved-key stored-type refresh are legacy-Sample concepts. + return op(sample) + contract = op_contract(op) if contract.accepts in ("sample", "any", "value") and contract.style == "packed": result = op(sample) @@ -244,6 +262,17 @@ def _check_ops_materialized(ops: List[Any]) -> None: class FilterOp: """Configurable filter operation. + The op form of :meth:`Flux.filter` — a predicate gate over the stream: the sample + passes when the predicate returns ``True`` and is dropped otherwise (``__call__`` + returns ``None``, which every engine route treats as "skip this sample"). + + Example:: + + keep_loud = FilterOp(p=lambda s: float(s.input.max()) > 0.1) + flux = Flux(source=src, ops=[keep_loud]) + # equivalently, via the fluent API (which constructs this op): + flux = Flux(source=src).filter(lambda s: float(s.input.max()) > 0.1) + Args: p: Predicate ``Sample -> bool``; the sample passes through when it returns ``True``, else is dropped. Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. @@ -263,6 +292,20 @@ def __call__(self, s: Sample) -> Optional[Sample]: class WrappedOp: """Configurable transformation wrapper with smart mapping. + The op form of :meth:`Flux.map` — lifts a plain function over one Sample slot. The + callable is ALWAYS stored as its importable ``module:function`` path (via + :mod:`sampleflux.discovery`), so the op pickles across ``spawn`` workers and + serializes into Confluid YAML verbatim; the live function resolves lazily on first + call. + + Example:: + + op = WrappedOp(f="numpy:sqrt", s="input") # dotted path — resolved lazily + flux = Flux(source=src, ops=[op]) + # equivalently, from a live callable via the fluent API (which constructs + # this op and stores np.sqrt as the string "numpy:sqrt"): + flux = Flux(source=src).map(np.sqrt) + Args: f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. @@ -331,6 +374,8 @@ def _apply_op_native(carrier: Any, op: Any) -> Any: """ from sampleflux.kinds import classify_carrier, op_contract + if isinstance(carrier, TypedSample): + return op(carrier) # the typed carrier is applied verbatim, never promoted contract = op_contract(op) if isinstance(carrier, Sample): return _apply_op(carrier, op) @@ -437,6 +482,19 @@ class JointFlux: Aggregates multiple Flux streams into a single joint stream. Each sub-flux maintains its own unique transformation chain. + The iteration-only fan-in engine behind :meth:`Flux.joint`: each sub-flux applies + its OWN op chain, so differently-processed streams concatenate lazily without + materialization. For an indexable (random-access) concatenation of raw sources, + use ``ConcatSource`` instead. + + Example:: + + clean = Flux(source=day_one, ops=[normalize]) + augmented = Flux(source=day_two, ops=[normalize, augment]) + both = JointFlux(fluxes=[clean, augmented]) # len == len(clean) + len(augmented) + # or wrapped back into an engine (equivalent fluent form): + flux = Flux.joint([clean, augmented]) # == Flux(source=JointFlux([...])) + Args: fluxes: The Flux streams to concatenate; iteration walks them in order and length is their sum. Defaults to ``None`` ⇒ an empty joint stream (zero-arg construction). @@ -594,7 +652,7 @@ def __len__(self) -> int: return len(source) return 0 - def __getitem__(self, index: int) -> Sample: + def __getitem__(self, index: int) -> Any: """Random access: get the i-th sample with ops applied. Supports three source shapes: @@ -633,7 +691,7 @@ def __getitem__(self, index: int) -> Sample: "it in ``list(...)`` before handing it to Flux." ) _check_ops_materialized(self.ops) - sample: Any = raw if self.native else Sample.from_any(raw) + sample: Any = _as_carrier(raw, self.native) with activate(Context()): for op in self.ops: result = _apply_op_native(sample, op) if self.native else _apply_op(sample, op) @@ -744,7 +802,7 @@ def _iter_streamed(self) -> Iterator[Sample]: def to_carried() -> Iterator[Optional[_Carried]]: for item in source: - yield _Carried(item if self.native else Sample.from_any(item), Context()) + yield _Carried(_as_carrier(item, self.native), Context()) def per_sample(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[_Carried]]: from sampleflux.kinds import op_contract @@ -799,7 +857,7 @@ def _iter_sequential(self) -> Iterator[Sample]: return _check_ops_materialized(self.ops) for item in source: - sample = item if self.native else Sample.from_any(item) + sample = _as_carrier(item, self.native) yield from _worker_task_multi(sample, self.ops, native=self.native) def _iter_parallel(self) -> Iterator[Sample]: @@ -815,7 +873,7 @@ def _iter_parallel(self) -> Iterator[Sample]: with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: futures = [] for item in source: - sample = item if self.native else Sample.from_any(item) + sample = _as_carrier(item, self.native) futures.append(executor.submit(_worker_task_multi, sample, self.ops, self.native)) for future in futures: diff --git a/sampleflux/flow.py b/sampleflux/flow.py index 139e6de..d9fef07 100644 --- a/sampleflux/flow.py +++ b/sampleflux/flow.py @@ -51,12 +51,13 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, Mix, Save, Use, _read_output +from sampleflux.bag.sample import TypedSample, primary +from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Mix, Save, Use, _read_output from sampleflux.sample import Sample logger = get_logger(__name__) -RESERVED_STEP_KEYS = ("from", "target_from", "metadata_from", "bind") +RESERVED_STEP_KEYS = ("from", "target_from", "metadata_from", "merge_from", "bind") """Step-grammar keys stripped from a step mapping before the op is constructed.""" __all__ = ["FlowGraph", "FlowStep", "from_ops", "parse_flow", "to_ops", "RESERVED_STEP_KEYS"] @@ -70,24 +71,37 @@ class FlowStep(NamedTuple): from_: Optional[str] # None = previous step (first step: the source sample) target_from: Optional[str] metadata_from: Optional[str] - bind: Dict[str, str] # param -> "step" | "step.attr" + bind: Dict[str, str] # param -> "step" | "step.attr" | "step[key]" + merge_from: Tuple[str, ...] = () # typed fan-in: union these steps' FIELDS, in slot order class _BindRef(NamedTuple): """A parsed ``bind:`` reference.""" step: str - attr: Optional[str] # None = the step's result; else the step op's @output attribute + attr: Optional[str] # "step.attr" = the step op's @output attribute + key: Optional[str] # "step[key]" = the named FIELD of the step's TypedSample result + + +def _split_bind_ref(ref: str) -> _BindRef: + """Split a bind reference into its three shapes: ``step`` / ``step.attr`` / ``step[key]``.""" + text = str(ref) + if text.endswith("]") and "[" in text: + head, _, inner = text[:-1].partition("[") + if head and inner and "." not in head: + return _BindRef(head, None, inner) + head, dot, attr = text.partition(".") + return _BindRef(head, attr if dot else None, None) def _parse_bind_ref(ref: str, known: Sequence[str]) -> _BindRef: - head, dot, attr = str(ref).partition(".") - if head not in known: + parsed = _split_bind_ref(ref) + if parsed.step not in known: raise ValueError( f"flow: bind reference {ref!r} does not name an earlier step " f"(known steps at this point: {list(known)!r})" ) - return _BindRef(head, attr if dot else None) + return parsed def _check_reserved_collision(op: Any, step_name: str) -> None: @@ -175,6 +189,21 @@ def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[Li f"flow step {name!r}: {key}: {ref!r} does not name an EARLIER step " f"(document order is the schedule; steps so far: {seen!r})" ) + merge_raw = reserved.get("merge_from") + merge_from: Tuple[str, ...] = () + if merge_raw is not None: + merge_from = (str(merge_raw),) if isinstance(merge_raw, str) else tuple(str(r) for r in merge_raw) + for ref in merge_from: + if ref not in seen: + raise ValueError( + f"flow step {name!r}: merge_from: {ref!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) + if target_from is not None or metadata_from is not None: + raise ValueError( + f"flow step {name!r}: merge_from (typed fan-in) and target_from/metadata_from " + "(legacy fan-in) are mutually exclusive on one step" + ) bind_raw = reserved.get("bind") or {} if not isinstance(bind_raw, dict): raise TypeError(f"flow step {name!r}: bind must be a mapping of param -> step[.output]") @@ -193,6 +222,7 @@ def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[Li target_from=None if target_from is None else str(target_from), metadata_from=None if metadata_from is None else str(metadata_from), bind=bind, + merge_from=merge_from, ) ) seen.append(name) @@ -223,8 +253,10 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T readers[step.target_from].append((i, "target")) if step.metadata_from is not None: readers[step.metadata_from].append((i, "meta")) + for ref in step.merge_from: + readers[ref].append((i, "merge")) for ref in step.bind.values(): - parsed = _BindRef(*ref.partition(".")[::2]) if "." in ref else _BindRef(ref, None) + parsed = _split_bind_ref(ref) if parsed.attr is None: readers[parsed.step].append((i, "bind")) readers[outputs].append((len(steps), "out")) @@ -317,7 +349,7 @@ def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": # -- execution --------------------------------------------------------- - def _run(self, seed: Sample) -> Optional[Sample]: + def _run(self, seed: Any) -> Optional[Any]: """Run one sample through the steps; ``None`` = filtered (an op returned None).""" steps, outputs = self._ensure_parsed() readers = _result_readers(steps, outputs) @@ -344,10 +376,34 @@ def read_result(name: str, *, copy: bool) -> Any: base = read_result(prev, copy=False) else: base = seed - sample = Sample.from_any(base) + # The TYPED carrier passes through verbatim; everything else coerces to Sample. + sample: Any = base if isinstance(base, TypedSample) else Sample.from_any(base) + + # 2a. typed fan-in: UNION the merge_from steps' fields (slot order, last wins) + if step.merge_from: + if not isinstance(sample, TypedSample): + raise TypeError( + f"flow step {step.name!r}: merge_from is the TYPED fan-in but the carrier is " + f"{type(sample).__name__} — use target_from/metadata_from for legacy Samples." + ) + merged = [sample] + for ref in step.merge_from: + value = read_result(ref, copy=True) + if not isinstance(value, TypedSample): + raise TypeError( + f"flow step {step.name!r}: merge_from step {ref!r} holds " + f"{type(value).__name__}, expected a TypedSample" + ) + merged.append(value) + sample = TypedSample.merge(*merged) - # 2. fan-in slots (Mix semantics) + # 2b. legacy fan-in slots (Mix semantics) if step.target_from is not None or step.metadata_from is not None: + if isinstance(sample, TypedSample): + raise TypeError( + f"flow step {step.name!r}: target_from/metadata_from are the LEGACY fan-in " + "but the carrier is a TypedSample — use merge_from." + ) metadata = dict(sample.meta) target = sample.target if step.target_from is not None: @@ -378,24 +434,27 @@ def read_result(name: str, *, copy: bool) -> Any: "Run expanding pipelines through the Flux engine (iterable-only)." ) for param, ref in step.bind.items(): - if "." in ref: - head, _, attr = ref.partition(".") - producer = next(s for s in steps if s.name == head) - value = _read_output(producer.op, attr) + parsed = _split_bind_ref(ref) + if parsed.attr is not None: + producer = next(s for s in steps if s.name == parsed.step) + value = _read_output(producer.op, parsed.attr) if value is _MISSING: raise AttributeError( f"flow step {step.name!r}: bind {param}={ref!r} — " - f"step {head!r} op has no @output attribute {attr!r}" + f"step {parsed.step!r} op has no @output attribute {parsed.attr!r}" ) else: - value = read_result(ref, copy=False) - if isinstance(value, Sample): + value = read_result(parsed.step, copy=False) + if isinstance(value, TypedSample): + # "step[key]" = the named field; bare "step" = the primary input. + value = value[parsed.key] if parsed.key else primary(value)[1] + elif isinstance(value, Sample): value = value.input setattr(op, param, value) result = op(sample) if result is None: return None - sample = cast(Sample, result) + sample = result env[step.name] = sample prev = step.name @@ -424,7 +483,8 @@ def _iter_samples(self) -> Iterator[Sample]: return assert self.source is not None for item in self.source: - result = self._run(Sample.from_any(item)) + seed = item if isinstance(item, TypedSample) else Sample.from_any(item) + result = self._run(seed) if result is not None: yield result @@ -448,7 +508,7 @@ def __len__(self) -> int: return len(self.source) return 0 - def __getitem__(self, index: int) -> Sample: + def __getitem__(self, index: int) -> Any: if self.source is None: raise TypeError("FlowGraph source is None — cannot index.") if hasattr(self.source, "__getitem__"): @@ -458,7 +518,8 @@ def __getitem__(self, index: int) -> Sample: f"FlowGraph source {type(self.source).__name__} does not support indexing; " "wrap it in a list or use iteration." ) - result = self._run(Sample.from_any(raw)) + seed = raw if isinstance(raw, TypedSample) else Sample.from_any(raw) + result = self._run(seed) if result is None: raise IndexError(f"Sample {index} filtered out by the flow") return result @@ -529,11 +590,11 @@ def to_ops(steps: Union[Sequence[FlowStep], Dict[str, Any]], outputs: str = "") attr_refs: Dict[str, List[str]] = {} for step in parsed: for ref in step.bind.values(): - if "." in ref: - head, _, attr = ref.partition(".") - attr_refs.setdefault(head, []) - if attr not in attr_refs[head]: - attr_refs[head].append(attr) + parsed_ref = _split_bind_ref(ref) + if parsed_ref.attr is not None: + attr_refs.setdefault(parsed_ref.step, []) + if parsed_ref.attr not in attr_refs[parsed_ref.step]: + attr_refs[parsed_ref.step].append(parsed_ref.attr) def take_cell(name: str) -> Tuple[str, bool]: """(cell, is_last_read) — decrement the read counter.""" @@ -547,7 +608,16 @@ def take_cell(name: str) -> Tuple[str, bool]: cell, last = take_cell(step.from_) ops.append(Use(name=cell, drop=last)) - # 2. fan-in slots + # 2. fan-in slots — typed union (MergeFields) or the legacy Mix slots + if step.merge_from: + merge_drops: List[str] = [] + merge_cells: List[str] = [] + for ref in step.merge_from: + cell, last = take_cell(ref) + merge_cells.append(cell) + if last: + merge_drops.append(cell) + ops.append(MergeFields(sources=merge_cells, drop=merge_drops)) if step.target_from is not None or step.metadata_from is not None: drops: List[str] = [] kwargs: Dict[str, Any] = {} @@ -567,14 +637,15 @@ def take_cell(name: str) -> Tuple[str, bool]: emitted: Optional[Any] = step.op if emitted is not None: for param, ref in step.bind.items(): - if "." in ref: + parsed_ref = _split_bind_ref(ref) + if parsed_ref.attr is not None: cell = attr_cells[ref] cell_reads_left.setdefault(cell, 1) cell_reads_left[cell] -= 1 emitted = Apply(op=emitted, param=param, source=cell, drop=cell_reads_left[cell] <= 0) else: - cell, last = take_cell(ref) - emitted = Apply(op=emitted, param=param, source=cell, drop=last) + cell, last = take_cell(parsed_ref.step) + emitted = Apply(op=emitted, param=param, source=cell, key=parsed_ref.key or "", drop=last) captures = attr_refs.get(step.name, []) if captures: for attr in captures: @@ -616,7 +687,7 @@ def take_cell(name: str) -> Tuple[str, bool]: # --------------------------------------------------------------------------- -_CONTEXT_OP_CLASSES = (Save, Use, Drop, Apply, Capture, Mix) +_CONTEXT_OP_CLASSES = (Save, Use, Drop, Apply, Capture, Mix, MergeFields) def _ctx_view(raw: Any) -> Optional[type]: @@ -738,6 +809,11 @@ def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: pending.update(mix_grammar) pending["__mix_pending__"] = True continue + if view is MergeFields: + sources = [cell_ref(str(c)) for c in (_ctx_field(raw, "sources", None) or [])] + pending["merge_from"] = sources + pending["__mix_pending__"] = True + continue if view is Drop: continue # liveness is recomputed on lowering @@ -750,7 +826,9 @@ def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: for attr, cell in _capture_items(op).items(): captures[cell] = attr else: - bind[str(_ctx_field(op, "param", ""))] = cell_ref(str(_ctx_field(op, "source", ""))) + ref = cell_ref(str(_ctx_field(op, "source", ""))) + apply_key = str(_ctx_field(op, "key", "") or "") + bind[str(_ctx_field(op, "param", ""))] = f"{ref}[{apply_key}]" if apply_key else ref op = _ctx_field(op, "op") pending.pop("__mix_pending__", None) if bind: diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py index 36835c7..1e7de0d 100644 --- a/sampleflux/ops/context.py +++ b/sampleflux/ops/context.py @@ -19,6 +19,7 @@ from confluid import configurable, flow from confluid.fluid import Fluid +from sampleflux.bag.sample import TypedSample, primary from sampleflux.context import require from sampleflux.sample import Sample @@ -46,8 +47,16 @@ def _read_output(op: Any, name: str) -> Any: return _MISSING -def _cell_field(value: Any, field: str) -> Any: - """A cell's contribution to a Sample field: the Sample's own field, or the raw value verbatim.""" +def _cell_field(value: Any, field: str, key: str = "") -> Any: + """A cell's contribution to a value slot. + + A legacy ``Sample`` cell contributes its named field; a ``TypedSample`` cell + contributes the ``key``-named item when ``key`` is given, else its PRIMARY input-role + item (:func:`~sampleflux.bag.sample.primary` — the sanctioned "the input" accessor); + a raw cell value is used verbatim. + """ + if isinstance(value, TypedSample): + return value[key] if key else primary(value)[1] if isinstance(value, Sample): return getattr(value, field) return value @@ -96,7 +105,7 @@ def __init__(self, name: str = "", drop: bool = False) -> None: self.name = str(name) self.drop = bool(drop) - def __call__(self, sample: Sample) -> Sample: + def __call__(self, sample: Any) -> Any: if not self.name: raise ValueError("Use: 'name' (the context cell to read) is required") ctx = require("Use") @@ -105,6 +114,8 @@ def __call__(self, sample: Sample) -> Sample: ctx.delete(self.name) else: value = deepcopy(value) + if isinstance(value, TypedSample): + return value # the typed carrier passes through verbatim (never coerced) return Sample.from_any(value) @@ -148,6 +159,7 @@ class Apply: op: The op to configure and apply; required at call time, validated lazily. param: Attribute name on ``op`` to set with the cell value; required at call time. source: Context cell holding the value; required at call time, validated lazily. + key: For a TypedSample cell — the named field to contribute. Blank (default) = the primary input field. drop: When True, free the source cell after reading it. """ @@ -156,15 +168,17 @@ def __init__( op: Optional[object] = None, param: str = "", source: str = "", + key: str = "", drop: bool = False, ) -> None: # Lazy / zero-arg: store config only; op/param/source are validated at first call. self.op = op self.param = str(param) self.source = str(source) + self.key = str(key) self.drop = bool(drop) - def __call__(self, sample: Sample) -> Optional[Sample]: + def __call__(self, sample: Any) -> Optional[Any]: if self.op is None: raise ValueError("Apply: an 'op' to configure and apply is required") if not self.param: @@ -176,7 +190,7 @@ def __call__(self, sample: Sample) -> Optional[Sample]: value = ctx.get(self.source) if self.drop: ctx.delete(self.source) - value = _cell_field(value, "input") + value = _cell_field(value, "input", key=self.key) op = cast(Any, self.op) setattr(op, self.param, value) # _apply_op = the engine's contract-aware chokepoint, so a field-scoped wrapped op @@ -321,3 +335,54 @@ def __call__(self, sample: Sample) -> Sample: ctx.delete(cell_name) return Sample(input=mixed_input, target=mixed_target, metadata=metadata) + + +@configurable(category="op", group="structure") +class MergeFields: + """Typed fan-in: UNION the named cells' fields into the incoming :class:`TypedSample`. + + The typed replacement for :class:`Mix`'s metadata dict-merge: each source cell (a + ``TypedSample`` saved by an earlier branch) contributes its FIELDS and ROLES, united in + listed order with last-write-wins on a key collision (the deterministic slot-order rule; + avoid a deliberate collision by renaming on the producing branch — + ``sampleflux.ops.structure.RenameField``). ``keys`` selects a subset of a source's + fields before the union. + + Args: + sources: Context cells (earlier branch results) to union into the incoming sample, in order. + keys: Restrict the union to these field keys across all sources. Empty (default) = every field. + drop: Context cells to free after merging (defaults to none). + """ + + def __init__( + self, + sources: Optional[List[str]] = None, + keys: Optional[List[str]] = None, + drop: Optional[List[str]] = None, + ) -> None: + # Lazy / zero-arg: store config only; cells are resolved at first call. + self.sources = list(sources) if sources else [] + self.keys = list(keys) if keys else [] + self.drop = list(drop) if drop else [] + + def __call__(self, sample: TypedSample) -> TypedSample: + if not self.sources: + raise ValueError("MergeFields: 'sources' (the context cells to union) is required") + if not isinstance(sample, TypedSample): + raise TypeError( + f"MergeFields: the incoming carrier is {type(sample).__name__}, expected TypedSample — " + "typed fan-in unions named fields (legacy Sample fan-in is Mix)." + ) + ctx = require("MergeFields") + merged = sample + for cell_name in self.sources: + value = ctx.get(cell_name) + if not isinstance(value, TypedSample): + raise TypeError(f"MergeFields: cell {cell_name!r} holds {type(value).__name__}, expected a TypedSample") + if self.keys: + keep = [k for k in self.keys if k in value] + value = TypedSample({k: value[k] for k in keep}, {k: value.role_of(k) for k in keep}) + merged = TypedSample.merge(merged, value) + for cell_name in self.drop: + ctx.delete(cell_name) + return merged diff --git a/sampleflux/ops/structure.py b/sampleflux/ops/structure.py new file mode 100644 index 0000000..3c7a28a --- /dev/null +++ b/sampleflux/ops/structure.py @@ -0,0 +1,132 @@ +"""Structure ops for the typed bag — reshape a :class:`~sampleflux.bag.sample.TypedSample`'s fields. + +The typed analogue of the classic triple-slot plumbing (``MetadataToTargetOp``, the stash/swap +family): where the old model moved values between the fixed ``input``/``target`` slots and the +shared metadata dict, the bag model just RENAMES, RETAGS, COPIES, or DROPS named fields. Each op +is a thin copy-on-write wrapper over a ``TypedSample`` mutator — no payload is touched. + +All ops are lazy / zero-arg constructible (config validated in ``__call__``) and +``@configurable(category="op", group="structure")`` so they surface as canvas nodes. +""" + +from typing import List, Optional + +from confluid import configurable +from typing_extensions import get_args + +from sampleflux.bag.sample import ROLES, Role, TypedSample + +__all__ = ["SetRole", "RenameField", "DropField", "CopyField", "SelectFields"] + + +@configurable(category="op", group="structure") +class SetRole: + """Retag a field's role (``input`` / ``target`` / ``aux`` / ``pred``) without moving it. + + The typed replacement for the classic "metadata value becomes the target" op — in the bag + model a field's role is a tag, so promotion is a retag, not a move. + + Args: + key: The field to retag. + role: The new role — one of ``input`` / ``target`` / ``aux`` / ``pred``. + """ + + def __init__(self, key: str = "", role: Role = "input") -> None: + self.key = key + self.role = role + + def __call__(self, sample: TypedSample) -> TypedSample: + if not self.key: + raise ValueError("SetRole: 'key' (the field to retag) is required") + if self.role not in get_args(Role): + raise ValueError(f"SetRole: invalid role {self.role!r} (allowed: {list(ROLES)})") + return sample.set_role(self.key, self.role) + + +@configurable(category="op", group="structure") +class RenameField: + """Rename a field (role travels with it). Renaming onto an existing key replaces it. + + The sanctioned way to avoid a deliberate fan-in collision: rename on the producing branch + BEFORE the merge, instead of a merge-policy knob. + + Args: + src: The field to rename. + dst: The new field name. + """ + + def __init__(self, src: str = "", dst: str = "") -> None: + self.src = src + self.dst = dst + + def __call__(self, sample: TypedSample) -> TypedSample: + if not self.src or not self.dst: + raise ValueError("RenameField: both 'src' and 'dst' are required") + return sample.rename(self.src, self.dst) + + +@configurable(category="op", group="structure") +class DropField: + """Remove a field from the bag (e.g. free a heavy Signal after its Spectrogram is derived). + + Args: + key: The field to remove. Missing keys raise unless ``missing_ok``. + missing_ok: Silently pass through when the field is absent (default False). + """ + + def __init__(self, key: str = "", missing_ok: bool = False) -> None: + self.key = key + self.missing_ok = missing_ok + + def __call__(self, sample: TypedSample) -> TypedSample: + if not self.key: + raise ValueError("DropField: 'key' (the field to remove) is required") + if self.key not in sample: + if self.missing_ok: + return sample + raise KeyError(f"DropField: unknown field {self.key!r} (fields: {list(sample.keys())})") + return sample.drop(self.key) + + +@configurable(category="op", group="structure") +class CopyField: + """Duplicate a field under a new name (same item object; items are treated as immutable). + + Args: + src: The field to copy. + dst: The name of the copy. An existing ``dst`` is replaced. + role: Optional role for the copy; ``None`` keeps the source field's role. + """ + + def __init__(self, src: str = "", dst: str = "", role: Optional[Role] = None) -> None: + self.src = src + self.dst = dst + self.role = role + + def __call__(self, sample: TypedSample) -> TypedSample: + if not self.src or not self.dst: + raise ValueError("CopyField: both 'src' and 'dst' are required") + if self.src not in sample: + raise KeyError(f"CopyField: unknown field {self.src!r} (fields: {list(sample.keys())})") + out = sample.replace_field(self.dst, sample[self.src]) + return out.set_role(self.dst, self.role if self.role is not None else sample.role_of(self.src)) + + +@configurable(category="op", group="structure") +class SelectFields: + """Keep ONLY the named fields (order = the given order); everything else is dropped. + + Args: + keys: The fields to keep. Unknown keys raise (a silent miss hides a typo). + """ + + def __init__(self, keys: Optional[List[str]] = None) -> None: + self.keys = list(keys) if keys else [] + + def __call__(self, sample: TypedSample) -> TypedSample: + if not self.keys: + raise ValueError("SelectFields: 'keys' (the fields to keep) is required") + missing = [k for k in self.keys if k not in sample] + if missing: + raise KeyError(f"SelectFields: unknown fields {missing} (fields: {list(sample.keys())})") + return TypedSample({k: sample[k] for k in self.keys}, {k: sample.role_of(k) for k in self.keys}) diff --git a/sampleflux/storage/base.py b/sampleflux/storage/base.py index 37115fd..677b5f2 100644 --- a/sampleflux/storage/base.py +++ b/sampleflux/storage/base.py @@ -1,9 +1,18 @@ -from typing import Any, Iterator, Protocol, runtime_checkable +import json +from typing import Any, Dict, Iterator, Protocol, Tuple, runtime_checkable +import numpy as np import torch +from sampleflux.bag.sample import TypedSample from sampleflux.sample import Sample +#: Root-attribute format tag stamped on stores written in the typed field-group layout. +TYPED_FORMAT = "typedsample-v1" + +#: Prefix marking a JSON-encoded structured attr value (list/tuple/dict/None) in plain attrs. +_JSON_MARK = "__json__:" + def to_numpy(data: Any) -> Any: """Convert a torch tensor to a numpy array for array-storage backends (HDF5 / Zarr). @@ -18,7 +27,7 @@ def to_numpy(data: Any) -> Any: @runtime_checkable class DataSource(Protocol): - """Minimum contract for a SampleFlux data source.""" + """Minimum contract for a SampleFlux data source (LEGACY carrier — dies with the purge stage).""" def __iter__(self) -> Iterator[Sample]: """Iterate over samples in the source.""" @@ -31,7 +40,7 @@ def __len__(self) -> int: @runtime_checkable class DataSink(Protocol): - """Minimum contract for a SampleFlux data sink.""" + """Minimum contract for a SampleFlux data sink (LEGACY carrier — dies with the purge stage).""" def write(self, sample: Sample) -> None: """Write a single sample to the sink.""" @@ -42,6 +51,98 @@ def flush(self) -> None: ... +@runtime_checkable +class TypedDataSource(Protocol): + """Minimum contract for a typed-bag data source.""" + + def __iter__(self) -> Iterator[TypedSample]: + """Iterate over typed samples in the source.""" + ... + + def __len__(self) -> int: + """Total number of samples available.""" + ... + + +@runtime_checkable +class TypedDataSink(Protocol): + """Minimum contract for a typed-bag data sink.""" + + def write(self, sample: TypedSample) -> None: + """Write a single typed sample to the sink.""" + ... + + def flush(self) -> None: + """Ensure all pending writes are committed to storage.""" + ... + + +# -------------------------------------------------------------------------------------- +# The shared attr wire-format for the typed field-group layout (HDF5 attrs / Zarr .zattrs +# / directory JSON all speak it): scalars stay native (queryable), array values become +# separate datasets, and structured values (list/tuple/dict/None) ride a JSON string with +# TUPLE TAGGING so a round-trip preserves tuple-ness (Regions.canvas == (H, W), not [H, W]). +# -------------------------------------------------------------------------------------- +def split_attrs(attrs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Split an item's attrs into ``(plain, arrays)`` for storage. + + ``plain`` holds natively-storable scalars/strings plus JSON-marked structured values; + ``arrays`` holds ndarray/Tensor attr values (stored as their own datasets). + """ + plain: Dict[str, Any] = {} + arrays: Dict[str, Any] = {} + for key, value in attrs.items(): + if isinstance(value, (np.ndarray, torch.Tensor)): + arrays[key] = to_numpy(value) + elif isinstance(value, np.generic): + plain[key] = value.item() + elif isinstance(value, (bool, int, float, str)): + plain[key] = value + else: + plain[key] = _JSON_MARK + json.dumps(_tag_json(value)) + return plain, arrays + + +def restore_attrs(plain: Dict[str, Any], arrays: Dict[str, Any]) -> Dict[str, Any]: + """Rebuild an item's attrs dict from :func:`split_attrs`' two halves.""" + attrs: Dict[str, Any] = {} + for key, value in plain.items(): + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, bytes): # h5py may hand string attrs back as bytes + value = value.decode("utf-8") + if isinstance(value, str) and value.startswith(_JSON_MARK): + attrs[key] = _untag_json(json.loads(value[len(_JSON_MARK) :])) + else: + attrs[key] = value + attrs.update(arrays) + return attrs + + +def _tag_json(value: Any) -> Any: + """JSON-safe view of ``value`` with tuples tagged (``{"__tuple__": [...]}``) so they survive.""" + if isinstance(value, tuple): + return {"__tuple__": [_tag_json(v) for v in value]} + if isinstance(value, list): + return [_tag_json(v) for v in value] + if isinstance(value, dict): + return {k: _tag_json(v) for k, v in value.items()} + if isinstance(value, np.generic): + return value.item() + return value + + +def _untag_json(value: Any) -> Any: + """Reverse :func:`_tag_json` (restores tuples).""" + if isinstance(value, dict): + if set(value.keys()) == {"__tuple__"}: + return tuple(_untag_json(v) for v in value["__tuple__"]) + return {k: _untag_json(v) for k, v in value.items()} + if isinstance(value, list): + return [_untag_json(v) for v in value] + return value + + class Storage: """Base class for storage backends providing context manager support.""" diff --git a/sampleflux/storage/directory.py b/sampleflux/storage/directory.py index 7881af2..d395a49 100644 --- a/sampleflux/storage/directory.py +++ b/sampleflux/storage/directory.py @@ -1,11 +1,17 @@ +import json from pathlib import Path -from typing import Union +from typing import Any, Dict, Iterator, Union import confluid import numpy as np -from sampleflux.sample import Sample -from sampleflux.storage.base import DataSink, Storage +from sampleflux.bag.io import EncodedItem, decode_item, encode_item +from sampleflux.bag.sample import TypedSample +from sampleflux.storage.base import DataSink, Storage, restore_attrs, split_attrs, to_numpy + +#: Typed-layout filenames inside each per-sample directory. +_FIELDS_JSON = "fields.json" +_FIELDS_NPZ = "fields.npz" # category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @@ -30,8 +36,13 @@ def open(self) -> "DirectorySink": self.path.mkdir(parents=True, exist_ok=True) return self - def write(self, sample: Sample) -> None: + def write(self, sample: Any) -> None: """Write a sample to its own subdirectory.""" + if isinstance(sample, TypedSample): + self.open() + self._write_typed(sample) + return + # Use a zero-padded index for sorting sample_dir = self.path / f"{self._counter:06d}" sample_dir.mkdir(parents=True, exist_ok=True) @@ -57,5 +68,86 @@ def write(self, sample: Sample) -> None: self._counter += 1 + def _write_typed(self, sample: TypedSample) -> None: + """One sample in the typed field-group layout: ``fields.json`` + ``fields.npz``. + + ``fields.json`` describes every field (order, item type, role, plain attrs); + ``fields.npz`` carries the array halves — payloads keyed by field name, array-valued + attrs keyed ``.``. Every item serializes through the + :mod:`sampleflux.bag.io` codec, so externally-registered item types round-trip with + no storage edits. + """ + sample_dir = self.path / f"{self._counter:06d}" + sample_dir.mkdir(parents=True, exist_ok=True) + + spec: Dict[str, Any] = {"sampleflux_format": "typedsample-v1", "fields": []} + payloads: Dict[str, Any] = {} + for key, item in sample.items(): + encoded = encode_item(item) + plain, arrays = split_attrs(encoded.attrs) + spec["fields"].append( + { + "key": key, + "type": encoded.type_name, + "role": sample.role_of(key), + "attrs": plain, + "array_attrs": sorted(arrays), + "has_payload": encoded.payload is not None, + } + ) + if encoded.payload is not None: + payloads[key] = np.asarray(to_numpy(encoded.payload)) + for name, value in arrays.items(): + payloads[f"{key}.{name}"] = np.asarray(value) + + (sample_dir / _FIELDS_JSON).write_text(json.dumps(spec, indent=2)) + if payloads: + np.savez(sample_dir / _FIELDS_NPZ, **payloads) + self._counter += 1 + def flush(self) -> None: pass # Filesystem handles immediate writes + + +@confluid.configurable +class DirectorySource(Storage): + """Read typed samples written by :class:`DirectorySink` (one ``fields.json`` + ``fields.npz`` per sample). + + The matching source of the sink's TYPED layout (one directory per sample, sorted by the + zero-padded name, so read order matches write order). + + Args: + path: Root directory written by DirectorySink. + """ + + def __init__(self, path: Union[str, Path] = "") -> None: + # Lazy / zero-arg: store config only; the directory is scanned lazily on iteration. + self.path = Path(path) + + def _sample_dirs(self) -> list: + if not self.path.exists(): + raise FileNotFoundError(f"DirectorySource: {self.path} does not exist") + return sorted(p for p in self.path.iterdir() if p.is_dir() and (p / _FIELDS_JSON).exists()) + + def __iter__(self) -> Iterator[TypedSample]: + for sample_dir in self._sample_dirs(): + yield self._read(sample_dir) + + def __len__(self) -> int: + return len(self._sample_dirs()) + + @staticmethod + def _read(sample_dir: Path) -> TypedSample: + spec = json.loads((sample_dir / _FIELDS_JSON).read_text()) + npz_path = sample_dir / _FIELDS_NPZ + payloads = dict(np.load(npz_path, allow_pickle=False)) if npz_path.exists() else {} + fields: Dict[str, Any] = {} + roles: Dict[str, Any] = {} + for entry in spec["fields"]: + key = entry["key"] + arrays = {name: payloads[f"{key}.{name}"] for name in entry["array_attrs"]} + attrs = restore_attrs(dict(entry["attrs"]), arrays) + payload = payloads[key] if entry["has_payload"] else None + fields[key] = decode_item(EncodedItem(type_name=entry["type"], payload=payload, attrs=attrs)) + roles[key] = entry["role"] + return TypedSample(fields, roles) diff --git a/sampleflux/storage/hdf5.py b/sampleflux/storage/hdf5.py index 7335839..84d59aa 100644 --- a/sampleflux/storage/hdf5.py +++ b/sampleflux/storage/hdf5.py @@ -1,5 +1,6 @@ +import json from pathlib import Path -from typing import Iterator, Optional, Union +from typing import Any, Dict, Iterator, Optional, Union import h5py import numpy as np @@ -7,11 +8,38 @@ from confluid import configurable from loggair import get_logger +from sampleflux.bag.io import EncodedItem, decode_item, encode_item +from sampleflux.bag.sample import TypedSample from sampleflux.sample import Sample -from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy +from sampleflux.storage.base import TYPED_FORMAT, DataSink, DataSource, Storage, restore_attrs, split_attrs, to_numpy logger = get_logger("sampleflux.storage.hdf5") +#: Reserved field-group attr names in the typed layout (never item attrs). +_TYPE_ATTR = "__item_type__" +_ROLE_ATTR = "__role__" +_ORDER_ATTR = "__field_order__" + + +def _read_typed_sample(group: h5py.Group) -> TypedSample: + """Decode one ``sNNNNNN`` sample group of the typed field-group layout.""" + order = json.loads(group.attrs[_ORDER_ATTR]) + fields: Dict[str, Any] = {} + roles: Dict[str, Any] = {} + for name in order: + fgrp = group[name] + plain = {k: v for k, v in fgrp.attrs.items() if k not in (_TYPE_ATTR, _ROLE_ATTR)} + arrays: Dict[str, Any] = {} + agrp = fgrp.get("attrs") + if isinstance(agrp, h5py.Group): + for key, dset in agrp.items(): + arrays[key] = dset[()] + payload = fgrp["data"][()] if "data" in fgrp else None + attrs = restore_attrs(dict(plain), arrays) + fields[name] = decode_item(EncodedItem(type_name=str(fgrp.attrs[_TYPE_ATTR]), payload=payload, attrs=attrs)) + roles[name] = str(fgrp.attrs[_ROLE_ATTR]) + return TypedSample(fields, roles) + @configurable class HDF5Source(Storage, DataSource): @@ -40,11 +68,22 @@ def close(self) -> None: self._file.close() self._file = None - def __iter__(self) -> Iterator[Sample]: + @property + def is_typed(self) -> bool: + """True when the file carries the typed field-group layout (``sampleflux_format`` root attr).""" + self.open() + return self._file is not None and self._file.attrs.get("sampleflux_format") == TYPED_FORMAT + + def __iter__(self) -> Iterator[Any]: self.open() if self._file is None: return + if self.is_typed: + for name in sorted(k for k in self._file.keys() if k.startswith("s")): + yield _read_typed_sample(self._file[name]) + return + prefixes = sorted([k.split("_data")[0] for k in self._file.keys() if k.endswith("_data")]) for pref in prefixes: @@ -64,6 +103,8 @@ def __len__(self) -> int: self.open() if self._file is None: return 0 + if self.is_typed: + return len([k for k in self._file.keys() if k.startswith("s")]) return len([k for k in self._file.keys() if k.endswith("_data")]) def iter_metadata(self) -> "Iterator[tuple[str, dict]]": @@ -108,11 +149,20 @@ def close(self) -> None: self._file.close() self._file = None - def write(self, sample: Sample) -> None: + def write(self, sample: Any) -> None: self.open() if self._file is None: return + if isinstance(sample, TypedSample): + self._write_typed(sample) + return + if self._file.attrs.get("sampleflux_format") == TYPED_FORMAT: + raise TypeError( + "HDF5Sink: this file carries the typed field-group layout — cannot append a legacy " + "Sample to it (one carrier per file)." + ) + prefix = f"{self._counter:05d}" # Convert tensors to numpy for h5py @@ -155,6 +205,50 @@ def write(self, sample: Sample) -> None: self._counter += 1 + def _write_typed(self, sample: TypedSample) -> None: + """One sample in the typed field-group layout — see ``docs/typed-model.md`` (storage). + + Layout: root attr ``sampleflux_format = "typedsample-v1"``; per sample a group + ``sNNNNNN`` (attr ``__field_order__`` preserves insertion order) holding one subgroup + per FIELD with attrs ``__item_type__``/``__role__`` + the item's plain attrs, the + payload as ``data``, and array-valued attrs as datasets under ``attrs/``. Every item + serializes through the :mod:`sampleflux.bag.io` codec, so externally-registered item + types round-trip with no storage edits. + """ + assert self._file is not None + if self._counter == 0 and "sampleflux_format" not in self._file.attrs: + if any(k.endswith("_data") for k in self._file.keys()): + raise TypeError( + "HDF5Sink: this file carries the legacy Sample layout — cannot append a " + "TypedSample to it (one carrier per file)." + ) + self._file.attrs["sampleflux_format"] = TYPED_FORMAT + elif self._file.attrs.get("sampleflux_format") != TYPED_FORMAT: + raise TypeError( + "HDF5Sink: this file carries the legacy Sample layout — cannot append a " + "TypedSample to it (one carrier per file)." + ) + + group = self._file.create_group(f"s{self._counter:06d}") + group.attrs[_ORDER_ATTR] = json.dumps(list(sample.keys())) + for key, item in sample.items(): + encoded = encode_item(item) + fgrp = group.create_group(key) + fgrp.attrs[_TYPE_ATTR] = encoded.type_name + fgrp.attrs[_ROLE_ATTR] = sample.role_of(key) + plain, arrays = split_attrs(encoded.attrs) + for name, value in plain.items(): + fgrp.attrs[name] = value + if encoded.payload is not None: + payload = np.asarray(to_numpy(encoded.payload)) + kwargs = {"compression": self.compression} if self.compression and payload.ndim > 0 else {} + fgrp.create_dataset("data", data=payload, **kwargs) + for name, value in arrays.items(): + arr = np.asarray(value) + kwargs = {"compression": self.compression} if self.compression and arr.ndim > 0 else {} + fgrp.create_dataset(f"attrs/{name}", data=arr, **kwargs) + self._counter += 1 + def flush(self) -> None: if self._file: self._file.flush() diff --git a/sampleflux/storage/query.py b/sampleflux/storage/query.py index 5dc35db..3af333f 100644 --- a/sampleflux/storage/query.py +++ b/sampleflux/storage/query.py @@ -20,18 +20,51 @@ scans ever become hot.) """ +import json from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, cast, runtime_checkable import h5py from confluid import configurable from loggair import get_logger +from sampleflux.bag.io import encode_item +from sampleflux.bag.sample import TypedSample from sampleflux.ops.formula import _FORMULA_NAMESPACE from sampleflux.sample import Sample +from sampleflux.storage.base import TYPED_FORMAT, restore_attrs logger = get_logger("sampleflux.storage.query") -__all__ = ["MetadataFilterSource", "SupportsMetadataScan", "scan_hdf5_metadata", "scan_zarr_metadata"] +__all__ = [ + "MetadataFilterSource", + "SupportsMetadataScan", + "scan_hdf5_metadata", + "scan_zarr_metadata", + "typed_sample_metadata", +] + + +class _AttrView(dict): + """A metadata sub-dict that ALSO answers attribute access — so a typed scan's per-field + attrs evaluate naturally in a ``where`` expression (``signal.samplerate > 1e6``) while + staying a plain dict for programmatic predicates.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: # pragma: no cover - mirrors normal attribute-miss semantics + raise AttributeError(name) from exc + + +def _viewed(metadata: Dict[str, Any]) -> Dict[str, Any]: + """Wrap dict-valued entries in :class:`_AttrView` (one level — the typed field/attr shape).""" + return {k: _AttrView(v) if isinstance(v, dict) else v for k, v in metadata.items()} + + +def typed_sample_metadata(sample: TypedSample) -> Dict[str, Dict[str, Any]]: + """A live sample's queryable metadata: ``{field: {attr: value}}`` (attrs via the io codec, + payloads untouched) — the same nested shape the typed storage scans yield.""" + return {key: dict(encode_item(item).attrs) for key, item in sample.items()} @runtime_checkable @@ -44,13 +77,30 @@ def iter_metadata(self) -> Iterator[Tuple[str, Dict[str, Any]]]: def scan_hdf5_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: - """Scan an ``HDF5Sink`` file's metadata: dataset attrs + array-metadata SHAPE/DTYPE stubs. + """Scan an ``HDF5Sink`` file's metadata WITHOUT loading payload arrays. - Array-valued metadata (datasets under ``{prefix}_meta/``) is represented by a stub - string ``""`` — queries can test presence/shape without a - single array read. + Legacy layout: dataset attrs + array-metadata SHAPE/DTYPE stubs (a stub string + ``""`` — queries can test presence/shape without an array + read). Typed field-group layout: per sample the NESTED shape ``{field: {attr: value}}`` + (plain attrs decoded; array-valued attrs as stubs) — a ``where`` expression addresses it + as ``"."`` (e.g. ``"signal.samplerate > 1e6"``). """ with h5py.File(str(path), "r") as handle: + if handle.attrs.get("sampleflux_format") == TYPED_FORMAT: + for name in sorted(k for k in handle.keys() if k.startswith("s")): + group = handle[name] + nested: Dict[str, Any] = {} + for field in json.loads(group.attrs["__field_order__"]): + fgrp = group[field] + plain = {k: v for k, v in fgrp.attrs.items() if k not in ("__item_type__", "__role__")} + attrs = restore_attrs(dict(plain), {}) + agrp = fgrp.get("attrs") + if isinstance(agrp, h5py.Group): + for key, dset in agrp.items(): + attrs[key] = f"" + nested[field] = attrs + yield name, nested + return prefixes = sorted(k.split("_data")[0] for k in handle.keys() if k.endswith("_data")) for prefix in prefixes: metadata: Dict[str, Any] = dict(handle[f"{prefix}_data"].attrs) @@ -62,10 +112,28 @@ def scan_hdf5_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: def scan_zarr_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: - """Scan a ``ZarrGroupSink`` store's metadata: each sample group's ``.zattrs``.""" + """Scan a ``ZarrGroupSink`` store's metadata: ``.zattrs`` only, no payload arrays. + + Typed field-group stores yield the same NESTED ``{field: {attr: value}}`` shape as the + HDF5 scan (array-valued attrs as name stubs). + """ import zarr root = zarr.open_group(str(path), mode="r") + if root.attrs.get("sampleflux_format") == TYPED_FORMAT: + for name in sorted(root.group_keys()): + group = cast(Any, root[name]) + nested: Dict[str, Any] = {} + for field in json.loads(group.attrs["__field_order__"]): + fgrp = group[field] + plain = {k: v for k, v in dict(fgrp.attrs).items() if k not in ("__item_type__", "__role__")} + attrs = restore_attrs(plain, {}) + if "attrs" in fgrp: + for key in fgrp["attrs"].array_keys(): + attrs[key] = f"" + nested[field] = attrs + yield name, nested + return for name in sorted(root.group_keys()): yield name, dict(root[name].attrs) @@ -75,16 +143,23 @@ def _where_predicate(where: str) -> Callable[[Dict[str, Any]], bool]: The expression evaluates in the FormulaOp restricted namespace (``math.*`` + ``abs``/``min``/``max``/``round``/``pow``, no builtins) with the metadata KEYS bound - as variables — e.g. ``"snr_db > 10 and drone == 'DJI'"``. A missing key (NameError) - means the sample does not match (logged at trace-equivalent debug); any other - evaluation error raises (a malformed expression must fail loudly). + as variables — e.g. ``"snr_db > 10 and drone == 'DJI'"`` (legacy flat metadata) or + ``"signal.samplerate > 1e6"`` (a typed scan's per-field attrs). A missing key/attr + (NameError/AttributeError) means the sample does not match (logged at debug); any + other evaluation error raises (a malformed expression must fail loudly). + + NOTE: a field named like a Python keyword (e.g. ``class``) cannot be addressed in an + expression — query such fields via the programmatic ``predicate`` (metadata is plain + nested dicts there), or give queryable fields non-keyword names. """ def _predicate(metadata: Dict[str, Any]) -> bool: - namespace = {**_FORMULA_NAMESPACE, **metadata} + # Dict-valued entries (the typed scans' per-field attr dicts) evaluate through + # _AttrView so "." reads naturally; flat legacy metadata is untouched. + namespace = {**_FORMULA_NAMESPACE, **_viewed(metadata)} try: return bool(eval(where, {"__builtins__": {}}, namespace)) # noqa: S307 - restricted namespace - except NameError as exc: + except (NameError, AttributeError, KeyError) as exc: logger.debug(f"MetadataFilterSource: where={where!r} — {exc}; sample treated as non-matching") return False except Exception as exc: @@ -143,7 +218,13 @@ def matches(self) -> List[int]: f"MetadataFilterSource: {type(self.source).__name__} has no iter_metadata — " "falling back to full-iteration filtering (arrays load for every sample)." ) - self._matches = [i for i, sample in enumerate(self.source) if self._match(dict(sample.meta))] + self._matches = [ + i + for i, sample in enumerate(self.source) + if self._match( + typed_sample_metadata(sample) if isinstance(sample, TypedSample) else dict(sample.meta) + ) + ] return self._matches def __iter__(self) -> Iterator[Sample]: diff --git a/sampleflux/storage/zarr.py b/sampleflux/storage/zarr.py index c1dc8d7..d7151cf 100644 --- a/sampleflux/storage/zarr.py +++ b/sampleflux/storage/zarr.py @@ -1,13 +1,42 @@ +import json from pathlib import Path -from typing import Iterator, List, Optional, Union, cast +from typing import Any, Dict, Iterator, List, Optional, Union, cast import confluid import numpy as np import torch import zarr +from sampleflux.bag.io import EncodedItem, decode_item, encode_item +from sampleflux.bag.sample import TypedSample from sampleflux.sample import Sample -from sampleflux.storage.base import DataSink, DataSource, Storage, to_numpy +from sampleflux.storage.base import TYPED_FORMAT, DataSink, DataSource, Storage, restore_attrs, split_attrs, to_numpy + +#: Reserved field-group attr names in the typed layout (never item attrs). +_TYPE_ATTR = "__item_type__" +_ROLE_ATTR = "__role__" +_ORDER_ATTR = "__field_order__" + + +def _read_typed_group(grp: "zarr.Group") -> TypedSample: + """Decode one ``sample_NNNNNN`` group of the typed field-group layout.""" + order = json.loads(str(grp.attrs[_ORDER_ATTR])) + fields: Dict[str, Any] = {} + roles: Dict[str, Any] = {} + for name in order: + fgrp = cast(zarr.Group, grp[name]) + fattrs = dict(fgrp.attrs) + plain = {k: v for k, v in fattrs.items() if k not in (_TYPE_ATTR, _ROLE_ATTR)} + arrays: Dict[str, Any] = {} + if "attrs" in fgrp: + agrp = cast(zarr.Group, fgrp["attrs"]) + for key in agrp.array_keys(): + arrays[key] = np.asarray(cast(zarr.Array, agrp[key])[:]) + payload = np.asarray(cast(zarr.Array, fgrp["data"])[:]) if "data" in fgrp.array_keys() else None + attrs = restore_attrs(plain, arrays) + fields[name] = decode_item(EncodedItem(type_name=str(fattrs[_TYPE_ATTR]), payload=payload, attrs=attrs)) + roles[name] = str(fattrs[_ROLE_ATTR]) + return TypedSample(fields, roles) # category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @@ -33,10 +62,20 @@ def open(self) -> "ZarrGroupSink": pass return self - def write(self, sample: Sample) -> None: + def write(self, sample: Any) -> None: self.open() if self._root is None: raise RuntimeError("Zarr group not open") + + if isinstance(sample, TypedSample): + self._write_typed(sample) + return + if self._root.attrs.get("sampleflux_format") == TYPED_FORMAT: + raise TypeError( + "ZarrGroupSink: this store carries the typed field-group layout — cannot append a " + "legacy Sample to it (one carrier per store)." + ) + # Use require_group to handle existing nodes safely name = f"sample_{self._counter:06d}" grp = self._root.require_group(name) @@ -55,6 +94,35 @@ def write(self, sample: Sample) -> None: self._counter += 1 + def _write_typed(self, sample: TypedSample) -> None: + """One sample in the typed field-group layout (the Zarr twin of HDF5Sink._write_typed).""" + assert self._root is not None + existing_format = self._root.attrs.get("sampleflux_format") + if existing_format is None: + if any(True for _ in self._root.group_keys()) and self._counter == 0: + raise TypeError( + "ZarrGroupSink: this store carries the legacy Sample layout — cannot append a " + "TypedSample to it (one carrier per store)." + ) + self._root.attrs["sampleflux_format"] = TYPED_FORMAT + elif existing_format != TYPED_FORMAT: + raise TypeError(f"ZarrGroupSink: unknown store format {existing_format!r}") + + grp = self._root.require_group(f"sample_{self._counter:06d}") + grp.attrs[_ORDER_ATTR] = json.dumps(list(sample.keys())) + for key, item in sample.items(): + encoded = encode_item(item) + fgrp = grp.require_group(key) + fgrp.attrs[_TYPE_ATTR] = encoded.type_name + fgrp.attrs[_ROLE_ATTR] = sample.role_of(key) + plain, arrays = split_attrs(encoded.attrs) + fgrp.attrs.update(plain) + if encoded.payload is not None: + fgrp.create_array("data", data=np.asarray(to_numpy(encoded.payload)), overwrite=True) + for name, value in arrays.items(): + fgrp.create_array(f"attrs/{name}", data=np.asarray(value), overwrite=True) + self._counter += 1 + def flush(self) -> None: pass # pragma: no cover @@ -94,10 +162,20 @@ def open(self) -> "ZarrGroupSource": def close(self) -> None: self._root = None - def __iter__(self) -> Iterator[Sample]: + @property + def is_typed(self) -> bool: + """True when the store carries the typed field-group layout (``sampleflux_format`` root attr).""" + self.open() + return self._root is not None and self._root.attrs.get("sampleflux_format") == TYPED_FORMAT + + def __iter__(self) -> Iterator[Any]: self.open() if self._root is None: return + if self.is_typed: + for name in sorted(self._root.group_keys()): + yield _read_typed_group(cast(zarr.Group, self._root[name])) + return for name in sorted(self._root.group_keys()): grp = cast(zarr.Group, self._root[name]) data = cast(zarr.Array, grp[self.sample_key])[:] @@ -155,10 +233,39 @@ def open(self) -> "ZarrBatchSink": ) return self - def write(self, sample: Sample) -> None: + def write(self, sample: Any) -> None: self.open() if self._data_arr is None: raise RuntimeError("Zarr array not open") + + if isinstance(sample, TypedSample): + # The batch sink stores ONE uniform array: the PRIMARY input field's payload per + # row, plus a one-time item template (type/field/attrs of the FIRST sample) so the + # source can rebuild typed rows. Uniform-batch by design — per-sample attr + # variation does not fit a single stacked array; use ZarrGroupSink for that. + from sampleflux.bag.sample import primary + + key, item = primary(sample) + encoded = encode_item(item) + if "sampleflux_format" not in self._data_arr.attrs: + plain, arrays = split_attrs(encoded.attrs) + if arrays: + raise TypeError( + "ZarrBatchSink: array-valued item attrs do not fit the single-array batch " + "layout — use ZarrGroupSink." + ) + self._data_arr.attrs.update( + {"sampleflux_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} + ) + self._data_arr.append([np.asarray(to_numpy(encoded.payload))], axis=0) + self._counter += 1 + return + if self._data_arr.attrs.get("sampleflux_format") == TYPED_FORMAT: + raise TypeError( + "ZarrBatchSink: this store carries the typed layout — cannot append a legacy " + "Sample to it (one carrier per store)." + ) + # Append to the primary array # Zarr handles the resizing and chunking internally self._data_arr.append([sample.input], axis=0) @@ -198,10 +305,24 @@ def open(self) -> "ZarrBatchSource": def close(self) -> None: self._data_arr = None - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Any]: self.open() if self._data_arr is None: return + attrs = dict(self._data_arr.attrs) + if attrs.get("sampleflux_format") == TYPED_FORMAT: + # Typed batch rows: rebuild each row as the stored item type under the stored + # field key (uniform template — see ZarrBatchSink.write). + field = str(attrs["__field__"]) + type_name = str(attrs[_TYPE_ATTR]) + item_attrs = restore_attrs( + {k: v for k, v in attrs.items() if k not in ("sampleflux_format", _TYPE_ATTR, "__field__")}, {} + ) + for i in range(self._data_arr.shape[0]): + payload = np.asarray(self._data_arr[i]) + item = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=item_attrs)) + yield TypedSample({field: item}) + return for i in range(self._data_arr.shape[0]): yield Sample(input=torch.from_numpy(np.asarray(self._data_arr[i]))) diff --git a/sampleflux/typespec.py b/sampleflux/typespec.py index 808a6a7..b4cdd4d 100644 --- a/sampleflux/typespec.py +++ b/sampleflux/typespec.py @@ -96,6 +96,7 @@ "typed", "accepts", "compatible", + "infer_field_types", "infer_type", "infer_sample_type", "canonical_dtype", @@ -719,6 +720,27 @@ def infer_sample_type(sample: "Sample") -> SampleType: return SampleType(input=infer_type(sample.input), target=infer_type(sample.target)) +def infer_field_types(sample: Any) -> Dict[str, TypeSpec]: + """Per-field type inference for a typed bag: ``{field key: TypeSpec of the item's payload}``. + + The typed-bag analogue of :func:`infer_sample_type` — one spec per NAMED field instead of + the fixed input/target pair. The spec describes the item's PAYLOAD (via + :func:`~sampleflux.bag.items.item_data`, so an array item and a data-bearing wrapper both + report their array); a payload-less structured item reports its Python type. Visual + editors use this to type per-field sockets and pickers. + """ + from sampleflux.bag.items import item_data + from sampleflux.bag.sample import TypedSample + + if not isinstance(sample, TypedSample): + raise TypeError(f"infer_field_types: expected a TypedSample, got {type(sample).__name__}") + specs: Dict[str, TypeSpec] = {} + for key, item in sample.items(): + payload = item_data(item) + specs[key] = PythonType(type(item).__qualname__) if payload is item else infer_type(payload) + return specs + + # -------------------------------------------------------------------------------------------------- # JSON (de)serialization # -------------------------------------------------------------------------------------------------- diff --git a/tests/_bag_fixtures.py b/tests/_bag_fixtures.py new file mode 100644 index 0000000..4846704 --- /dev/null +++ b/tests/_bag_fixtures.py @@ -0,0 +1,77 @@ +"""Test-local typed-bag fixtures. + +``FixtureFlip`` is the former native ``HorizontalFlip`` kept ONLY as a test fixture: sampleflux +ships no native augmentation transforms (geometric/photometric augmentation comes from +torchvision v2 / albumentations through adapter coercion), but the kernel-dispatch machinery +(once-per-sample params, per-type kernels, MRO resolution, ``only=`` filter) still needs a +fully native transform to pin — and the fixture doubles as the ADAPTER-PARITY reference (a +`v2.RandomHorizontalFlip(p=1.0)` through the adapter must move image/mask/boxes exactly like +this native implementation does). +""" + +from typing import Any, Dict, List, Optional + +import numpy as np + +from sampleflux import Image, Mask, Regions, Transform, TypedSample, item_data, with_data + + +class FixtureFlip(Transform): + """Horizontal flip with ONE shared decision across Image + Mask + Regions (test fixture).""" + + handles = (Image, Mask, Regions) + consumes = (Image,) + optional = (Mask, Regions) + produces = (Image, Mask, Regions) + + def __init__(self, p: float = 0.5, only: Optional[List[str]] = None) -> None: + super().__init__(only=only) + self.p = p + + def get_params(self, sample: TypedSample) -> Dict[str, Any]: + do = float(np.random.random()) < self.p + return {"do": do, "width": _reference_width(sample)} + + +@FixtureFlip.kernel(Image) +def _flip_image(item: Image, params: Dict[str, Any]) -> Image: + if not params["do"]: + return item + axis = 2 if getattr(item, "layout", "HWC") == "CHW" else 1 + return with_data(item, np.flip(item_data(item), axis=axis).copy()) + + +@FixtureFlip.kernel(Mask) +def _flip_mask(item: Mask, params: Dict[str, Any]) -> Mask: + if not params["do"]: + return item + return with_data(item, np.flip(item_data(item), axis=1).copy()) + + +@FixtureFlip.kernel(Regions) +def _flip_regions(item: Regions, params: Dict[str, Any]) -> Regions: + if not params["do"]: + return item + width = params.get("width") or (item.canvas[1] if item.canvas else None) + if width is None: + raise ValueError("FixtureFlip: no reference width to flip Regions") + boxes = [[width - box[2], box[1], width - box[0], box[3]] for box in item.boxes] + return Regions(boxes=boxes, labels=item.labels, scores=item.scores, canvas=item.canvas) + + +def _reference_width(sample: TypedSample) -> Optional[int]: + """The horizontal extent to flip boxes against — from the first Image/Mask, or a Regions canvas.""" + for _, item in sample.items(): + if isinstance(item, Image): + arr = item_data(item) + axis = 2 if getattr(item, "layout", "HWC") == "CHW" else 1 + if arr.ndim > axis: + return int(arr.shape[axis]) + if isinstance(item, Mask): + arr = item_data(item) + if arr.ndim >= 2: + return int(arr.shape[1]) + for _, item in sample.items(): + if isinstance(item, Regions) and item.canvas: + return int(item.canvas[1]) + return None diff --git a/tests/test_bag_dispatch.py b/tests/test_bag_dispatch.py new file mode 100644 index 0000000..38ef288 --- /dev/null +++ b/tests/test_bag_dispatch.py @@ -0,0 +1,52 @@ +"""The kernel registry — exact + MRO dispatch, transform-MRO inheritance, override, cache.""" + +from typing import Any, Dict + +from sampleflux import Image, Label, Mask, Regions, Transform +from sampleflux.bag.dispatch import dispatch, get_kernel, register_kernel, registered_kernels +from tests._bag_fixtures import FixtureFlip + + +class TestDispatch: + def test_exact_hit(self) -> None: + assert get_kernel(FixtureFlip, Image) is not None + assert dispatch(FixtureFlip, Image) is get_kernel(FixtureFlip, Image) + + def test_miss_returns_none(self) -> None: + # FixtureFlip has no Label kernel — a Label field passes through. + assert dispatch(FixtureFlip, Label) is None + assert get_kernel(FixtureFlip, Label) is None + + def test_item_mro_walk(self) -> None: + class SubMask(Mask): + pass + + # No kernel for SubMask, but its base Mask has one — the MRO walk resolves it. + assert get_kernel(FixtureFlip, SubMask) is None + assert dispatch(FixtureFlip, SubMask) is get_kernel(FixtureFlip, Mask) + + def test_transform_mro_inheritance(self) -> None: + class TunedFlip(FixtureFlip): + pass + + # A subclass transform inherits its base's kernels until it overrides them. + assert dispatch(TunedFlip, Image) is get_kernel(FixtureFlip, Image) + + def test_override_wins_and_invalidates_cache(self) -> None: + class T(Transform): + pass + + assert dispatch(T, Image) is None # populate the cache with a miss + + @register_kernel(T, Image) + def _kernel(item: Any, params: Dict[str, Any]) -> Any: + return item + + assert dispatch(T, Image) is _kernel # cache was cleared on registration + + def test_registered_kernels_lists_pairs(self) -> None: + pairs = registered_kernels() + assert ("FixtureFlip", "Image") in pairs + assert ("FixtureFlip", "Regions") in pairs + assert dispatch(FixtureFlip, Label) is None # FixtureFlip does not handle Label + assert dispatch(FixtureFlip, Regions) is get_kernel(FixtureFlip, Regions) diff --git a/tests/test_bag_interop.py b/tests/test_bag_interop.py new file mode 100644 index 0000000..71b06a7 --- /dev/null +++ b/tests/test_bag_interop.py @@ -0,0 +1,77 @@ +"""Legacy ``Sample`` <-> ``TypedSample`` bridge — lossless round-trip, builder path, errors. + +Uses the modality-neutral core items plus a small test-local data-bearing wrapper (the shape a +signal item takes) so the wrapper round-trip is covered without importing a domain package. +""" + +from dataclasses import dataclass + +import numpy as np +import pytest + +from sampleflux.bag.interop import ENCODE_KEY, to_legacy, to_typed +from sampleflux.bag.items import Image, Label, Regions, register_item +from sampleflux.bag.sample import TypedSample +from sampleflux.sample import Sample + + +@register_item +@dataclass +class _WrapBlob: + data: object = None + tag: str = "x" + + +def _sample() -> TypedSample: + return TypedSample( + { + "image": Image(np.arange(48, dtype=np.float32).reshape(4, 4, 3), layout="HWC"), + "blob": _WrapBlob(np.arange(16, dtype=np.float32), tag="sig"), + "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(4, 8)), + "class": Label("drone_x", classes=["noise", "drone_x"]), + }, + roles={"regions": "target", "class": "target"}, + ) + + +class TestRoundTrip: + def test_lossless(self) -> None: + s = _sample() + assert to_typed(to_legacy(s)) == s + + def test_legacy_exposes_input_target(self) -> None: + legacy = to_legacy(_sample()) + assert isinstance(legacy, Sample) + assert np.asarray(legacy.input).shape == (4, 4, 3) # first input field payload (the image) + assert ENCODE_KEY in legacy.meta + + def test_reconstructs_item_types_and_meta(self) -> None: + back = to_typed(to_legacy(_sample())) + assert isinstance(back["image"], Image) and back["image"].layout == "HWC" + assert isinstance(back["blob"], _WrapBlob) and back["blob"].tag == "sig" + assert isinstance(back["regions"], Regions) and back["regions"].canvas == (4, 8) + assert back.role_of("class") == "target" + + def test_no_input_or_target(self) -> None: + s = TypedSample({"aux": _WrapBlob(np.ones(4))}, roles={"aux": "aux"}) + legacy = to_legacy(s) + assert legacy.input is None and legacy.target is None + assert to_typed(legacy) == s + + +class TestBuilderPath: + def test_builder_used_when_no_encoding(self) -> None: + raw = Sample(input=np.zeros((4, 4, 3)), target="cat", metadata={}) + + def builder(sample: Sample) -> TypedSample: + return TypedSample( + {"image": Image(sample.input), "class": Label(sample.target)}, + roles={"class": "target"}, + ) + + typed = to_typed(raw, builder=builder) + assert isinstance(typed["image"], Image) and typed["class"].value == "cat" + + def test_no_encoding_no_builder_raises(self) -> None: + with pytest.raises(ValueError, match="no embedded typed encoding"): + to_typed(Sample(input=np.zeros(3), target=None, metadata={})) diff --git a/tests/test_bag_io.py b/tests/test_bag_io.py new file mode 100644 index 0000000..33368f1 --- /dev/null +++ b/tests/test_bag_io.py @@ -0,0 +1,93 @@ +"""The item codec registry (``sampleflux.bag.io``) — default structural codec, overrides, samples.""" + +from dataclasses import dataclass + +import numpy as np +import pytest + +from sampleflux import ( + EncodedItem, + Image, + Label, + Regions, + TypedSample, + decode_item, + decode_sample, + encode_item, + encode_sample, + register_io, + register_item, +) + + +@register_item +@dataclass +class _IoBlob: + """A data-bearing wrapper (the shape a domain signal item takes).""" + + data: object = None + rate: float = 1.0 + + +class TestDefaultCodec: + def test_array_item_round_trip(self) -> None: + img = Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW") + enc = encode_item(img) + assert enc.type_name == "Image" and enc.attrs == {"layout": "CHW"} + back = decode_item(enc) + assert isinstance(back, Image) and back.layout == "CHW" + assert np.array_equal(np.asarray(back), np.asarray(img)) + + def test_wrapper_item_round_trip(self) -> None: + blob = _IoBlob(np.ones(4), rate=48000.0) + enc = encode_item(blob) + assert enc.type_name == "_IoBlob" and enc.attrs == {"rate": 48000.0} + assert np.array_equal(enc.payload, np.ones(4)) + back = decode_item(enc) + assert isinstance(back, _IoBlob) and back.rate == 48000.0 + + def test_payloadless_item_round_trip(self) -> None: + lab = Label("drone", classes=["a", "drone"]) + enc = encode_item(lab) + assert enc.payload is None and enc.attrs == {"value": "drone", "classes": ["a", "drone"]} + back = decode_item(enc) + assert isinstance(back, Label) and back.value == "drone" and back.classes == ["a", "drone"] + + def test_unknown_type_name_raises(self) -> None: + with pytest.raises(KeyError, match="no item type registered"): + decode_item(EncodedItem(type_name="Nope", payload=None, attrs={})) + + +class TestRegisteredCodec: + def test_override_wins_and_round_trips(self) -> None: + @register_item + class Compact: # a type the default codec cannot capture + def __init__(self, values: list) -> None: + self.values = values + + register_io( + Compact, + encode=lambda item: (np.asarray(item.values), {}), + decode=lambda payload, attrs: Compact(list(np.asarray(payload))), + ) + enc = encode_item(Compact([1, 2, 3])) + assert enc.type_name == "Compact" and np.array_equal(enc.payload, [1, 2, 3]) + back = decode_item(enc) + assert isinstance(back, Compact) and back.values == [1, 2, 3] + + +class TestSampleCodec: + def test_sample_round_trip_fields_roles_order(self) -> None: + s = TypedSample( + { + "image": Image(np.zeros((2, 2, 3), dtype=np.float32)), + "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), + "class": Label("x"), + }, + roles={"regions": "target", "class": "target"}, + ) + fields = encode_sample(s) + assert [f.key for f in fields] == ["image", "regions", "class"] + assert [f.role for f in fields] == ["input", "target", "target"] + back = decode_sample(fields) + assert back == s diff --git a/tests/test_bag_items.py b/tests/test_bag_items.py new file mode 100644 index 0000000..58a7fa5 --- /dev/null +++ b/tests/test_bag_items.py @@ -0,0 +1,133 @@ +"""Typed items — array-subclass attribute preservation, wrappers, payload accessors, registry. + +Only the MODALITY-NEUTRAL core items live in sampleflux (Image / Mask / Regions / Label). The +data-bearing-wrapper and multi-attribute-array paths (which the signal-domain items in +``waivefront.bag`` exercise for real) are covered here with small test-local item types, so the +core stays tested without importing a domain package. +""" + +from dataclasses import dataclass + +import numpy as np +import pytest + +from sampleflux.bag.items import ( + Image, + Label, + Mask, + NDArrayItem, + Regions, + get_item_type, + is_item, + item_data, + item_type_names, + item_types, + register_item, + with_data, +) + + +@register_item +@dataclass +class _Blob: + """A test-local data-bearing wrapper item (the shape a signal item takes).""" + + data: object = None + tag: str = "x" + + +class _Multi(NDArrayItem): + """A test-local array item with two extra attributes (a spectrogram-like shape).""" + + _item_attrs = ("a", "b") + a: int = 1 + b: object = None + + +class TestArrayItems: + def test_default_and_explicit_attr(self) -> None: + assert Image(np.zeros((2, 3, 3))).layout == "HWC" + assert Image(np.zeros((3, 2, 3)), layout="CHW").layout == "CHW" + + def test_attr_survives_numpy_ops(self) -> None: + img = Image(np.arange(2 * 3 * 3).reshape(2, 3, 3), layout="HWC") + flipped = np.flip(img, axis=1) + assert isinstance(flipped, Image) and flipped.layout == "HWC" + doubled = img * 2 + assert isinstance(doubled, Image) and doubled.layout == "HWC" + assert isinstance(img[0], Image) # slicing keeps the subclass + attr + + def test_multiple_attrs_survive_ufunc(self) -> None: + item = _Multi(np.zeros((4, 8)), a=7, b={"n": 8}) + assert item.a == 7 and item.b == {"n": 8} + shifted = item + 1 # attrs carried through the ufunc + assert isinstance(shifted, _Multi) and shifted.a == 7 and shifted.b == {"n": 8} + + def test_unknown_attr_rejected(self) -> None: + with pytest.raises(TypeError, match="unexpected attributes"): + Image(np.zeros((2, 2, 3)), colorspace="rgb") + + def test_mask_has_no_extra_attrs(self) -> None: + assert isinstance(Mask(np.zeros((4, 4))), NDArrayItem) + + +class TestWrapperItems: + def test_wrapper_fields(self) -> None: + blob = _Blob(np.ones(8), tag="sig") + assert blob.tag == "sig" and np.asarray(blob.data).sum() == 8 + + def test_zero_arg_construction(self) -> None: + # Wrappers build with no args (fields defaulted) — the workspace lazy/zero-arg convention. + assert _Blob().data is None and Regions().boxes == [] and Label().value is None + + +class TestPayloadAccessors: + def test_item_data_array(self) -> None: + img = Image(np.arange(4).reshape(2, 2)) + data = item_data(img) + assert type(data) is np.ndarray and np.array_equal(data, [[0, 1], [2, 3]]) + + def test_item_data_wrapper(self) -> None: + assert np.array_equal(item_data(_Blob(np.ones(3))), np.ones(3)) + + def test_item_data_no_payload_returns_self(self) -> None: + reg = Regions(boxes=[[0, 0, 1, 1]]) + assert item_data(reg) is reg # no `.data` slot — returns the item + + def test_with_data_array_preserves_attrs(self) -> None: + img = Image(np.zeros((2, 2, 3)), layout="CHW") + rebuilt = with_data(img, np.ones((2, 2, 3))) + assert isinstance(rebuilt, Image) and rebuilt.layout == "CHW" and rebuilt.sum() == 12 + + def test_with_data_wrapper_preserves_meta(self) -> None: + rebuilt = with_data(_Blob(np.zeros(4), tag="t"), np.ones(4)) + assert isinstance(rebuilt, _Blob) and rebuilt.tag == "t" and np.asarray(rebuilt.data).sum() == 4 + + def test_with_data_without_payload_raises(self) -> None: + with pytest.raises(TypeError, match="no payload slot"): + with_data(Regions(boxes=[]), [[0, 0, 1, 1]]) + + +class TestRegistry: + def test_builtins_registered(self) -> None: + names = item_type_names() + for name in ("Image", "Mask", "Regions", "Label"): + assert name in names + assert Image in item_types() + + def test_get_item_type_and_miss(self) -> None: + assert get_item_type("Image") is Image + with pytest.raises(KeyError, match="no item type registered"): + get_item_type("Nope") + + def test_is_item(self) -> None: + assert is_item(Image(np.zeros((1, 1, 3)))) and is_item(Label()) + assert not is_item(np.zeros((2, 2))) and not is_item(42) + + def test_register_custom_type(self) -> None: + @register_item + class Keypoints: # a user type — one class + one decorator, no core edit + def __init__(self, points: list) -> None: + self.points = points + + assert "Keypoints" in item_type_names() and get_item_type("Keypoints") is Keypoints diff --git a/tests/test_bag_pipeline.py b/tests/test_bag_pipeline.py new file mode 100644 index 0000000..009b0db --- /dev/null +++ b/tests/test_bag_pipeline.py @@ -0,0 +1,186 @@ +"""The headline cross-library mixed pipeline + adapter behavior + import safety.""" + +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +from sampleflux.bag import Image, Label, Mask, Pipeline, Regions, TypedSample +from tests._bag_fixtures import FixtureFlip + + +class TestImportSafety: + def test_typed_imports_without_torchvision(self) -> None: + # The top-level package must import without torchvision (adapters lazy-import their + # library inside method bodies), so discovery stays safe on hosts missing it. + code = "import sys; import sampleflux.bag; assert 'torchvision' not in sys.modules" + subprocess.run([sys.executable, "-c", code], check=True, cwd=str(Path(__file__).resolve().parents[1])) + + +class TestTorchvisionAdapter: + v2 = pytest.importorskip("torchvision.transforms.v2") + + def test_normalize_touches_only_image(self) -> None: + from sampleflux.bag.adapters import TorchvisionV2Adapter + + s = TypedSample( + {"image": Image(np.ones((4, 5, 3), dtype=np.float32)), "class": Label("x")}, + roles={"class": "target"}, + ) + out = TorchvisionV2Adapter(self.v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]))(s) + assert isinstance(out["image"], Image) and out["image"].layout == "HWC" + assert np.asarray(out["image"]).shape == (4, 5, 3) + assert np.allclose(np.asarray(out["image"]), 1.0) # (1-0.5)/0.5 + assert out["class"].value == "x" + + def test_flip_moves_image_mask_boxes_together(self) -> None: + from sampleflux.bag.adapters import TorchvisionV2Adapter + + s = TypedSample( + { + "image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32)), + "mask": Mask(np.arange(6 * 8).reshape(6, 8).astype(np.int64)), + "regions": Regions(boxes=[[1, 1, 4, 4]], canvas=(6, 8)), + } + ) + out = TorchvisionV2Adapter(self.v2.RandomHorizontalFlip(p=1.0))(s) + assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(s["mask"])[:, ::-1]) + assert out["regions"].boxes == [[4.0, 1.0, 7.0, 4.0]] # W=8: x -> W-x + + def test_missing_transform_raises(self) -> None: + from sampleflux.bag.adapters import TorchvisionV2Adapter + + with pytest.raises(ValueError, match="must be set"): + TorchvisionV2Adapter()(TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) + + def test_no_handled_field_is_noop(self) -> None: + from sampleflux.bag.adapters import TorchvisionV2Adapter + + s = TypedSample({"class": Label("x")}) + assert TorchvisionV2Adapter(self.v2.RandomHorizontalFlip(p=1.0))(s) == s + + +class TestAlbumentationsAdapter: + def test_gaussnoise_touches_only_image(self) -> None: + import albumentations as A + + from sampleflux.bag.adapters import AlbumentationsAdapter + + s = TypedSample( + {"image": Image(np.full((6, 6, 3), 0.5, dtype=np.float32)), "class": Label("x")}, + roles={"class": "target"}, + ) + out = AlbumentationsAdapter(A.GaussNoise(p=1.0))(s) + assert isinstance(out["image"], Image) + assert not np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])) + assert out["class"].value == "x" + + def test_bboxes_wrapped_and_returned(self) -> None: + import albumentations as A + + from sampleflux.bag.adapters import AlbumentationsAdapter + + s = TypedSample( + { + "image": Image(np.random.rand(10, 12, 3).astype(np.float32)), + "regions": Regions(boxes=[[2, 3, 6, 7]], labels=[1], canvas=(10, 12)), + } + ) + out = AlbumentationsAdapter(A.HorizontalFlip(p=1.0))(s) + assert out["regions"].boxes[0] == pytest.approx([6.0, 3.0, 10.0, 7.0], abs=1e-4) # W=12 + assert out["regions"].labels == [1] + + def test_missing_transform_raises(self) -> None: + from sampleflux.bag.adapters import AlbumentationsAdapter + + with pytest.raises(ValueError, match="must be set"): + AlbumentationsAdapter()(TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) + + +class TestMixedPipeline: + def test_cross_library_bare_transforms(self) -> None: + # BARE library transforms drop straight into the Pipeline — the registered adapters + # wrap them; no explicit TorchvisionV2Adapter(...) / AlbumentationsAdapter(...). Two + # torchvision v2 transforms and an albumentations transform mix in one pipeline; each + # hits only the field(s) of a type it handles, and the flip moves image+mask+boxes with + # one library draw. + v2 = pytest.importorskip("torchvision.transforms.v2") + import albumentations as A + + rng = np.random.default_rng(0) + sample = TypedSample( + { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), + "mask": Mask(rng.random((16, 20)) > 0.5), + "regions": Regions(boxes=[[2, 3, 6, 7]], labels=["a"], canvas=(16, 20)), + "class": Label("drone_x", classes=["noise", "drone_x"]), + }, + roles={"mask": "target", "regions": "target", "class": "target"}, + ) + out = Pipeline( + [ + v2.RandomHorizontalFlip(p=1.0), # torchvision v2: Image + Mask + Regions together + v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.25, 0.25, 0.25]), # torchvision v2: Image + A.GaussNoise(p=1.0), # albumentations: Image + ] + )(sample) + assert not np.array_equal(np.asarray(out["image"]), np.asarray(sample["image"])) # flipped+normalized+noised + assert np.array_equal(np.asarray(out["mask"]), np.asarray(sample["mask"])[:, ::-1]) # flipped with the image + assert out["regions"].boxes[0] == pytest.approx([14, 3, 18, 7], abs=1e-4) # W=20: x -> W-x + assert out["class"].value == "drone_x" # label rode through untouched + assert out.roles == sample.roles # role tags preserved end-to-end + + +class TestCoercion: + def test_native_transform_passes_through(self) -> None: + from sampleflux.bag import coerce_transform + + flip = FixtureFlip() + assert coerce_transform(flip) is flip + + def test_torchvision_bare_transform_coerced(self) -> None: + v2 = pytest.importorskip("torchvision.transforms.v2") + + from sampleflux.bag import coerce_transform + from sampleflux.bag.adapters.torchvision import TorchvisionV2Adapter, is_torchvision_v2_transform + + norm = v2.Normalize(mean=[0.0], std=[1.0]) + assert is_torchvision_v2_transform(norm) + assert isinstance(coerce_transform(norm), TorchvisionV2Adapter) + + def test_albumentations_bare_transform_coerced(self) -> None: + import albumentations as A + + from sampleflux.bag import coerce_transform + from sampleflux.bag.adapters.albumentations import AlbumentationsAdapter, is_albumentations_transform + + noise = A.GaussNoise(p=1.0) + assert is_albumentations_transform(noise) + assert isinstance(coerce_transform(noise), AlbumentationsAdapter) + + def test_unknown_object_raises(self) -> None: + from sampleflux.bag import coerce_transform + + with pytest.raises(TypeError, match="don't know how to adapt"): + coerce_transform(object()) + + def test_register_custom_adapter(self) -> None: + # A user library object becomes droppable into a Pipeline with one register_adapter call. + from sampleflux.bag import FunctionTransform, coerce_transform, register_adapter + from sampleflux.bag.transform import Transform + + class MyLibDouble: # a foreign object, not a Transform + pass + + def factory(_obj: object) -> Transform: + return FunctionTransform(lambda d: d * 2, handles=(Image,)) + + register_adapter(lambda o: isinstance(o, MyLibDouble), factory) + + s = TypedSample({"image": Image(np.ones((2, 2, 3)))}) + out = Pipeline([MyLibDouble()])(s) + assert np.allclose(np.asarray(out["image"]), 2.0) + assert isinstance(coerce_transform(MyLibDouble()), FunctionTransform) diff --git a/tests/test_bag_sample.py b/tests/test_bag_sample.py new file mode 100644 index 0000000..f8738ec --- /dev/null +++ b/tests/test_bag_sample.py @@ -0,0 +1,114 @@ +"""``TypedSample`` — role tags, views, copy-on-write mutators, array-safe equality.""" + +import numpy as np +import pytest + +from sampleflux.bag.items import Image, Label, Regions +from sampleflux.bag.sample import ROLES, TypedSample + + +def _sample() -> TypedSample: + return TypedSample( + {"image": Image(np.ones(4)), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")}, + roles={"regions": "target", "class": "target"}, + ) + + +class TestRolesAndViews: + def test_default_role_is_input(self) -> None: + s = TypedSample({"a": Image(np.zeros((1, 1, 3))), "b": Label()}) + assert s.roles == {"a": "input", "b": "input"} + + def test_inputs_targets_aux(self) -> None: + s = _sample() + assert list(s.inputs()) == ["image"] + assert list(s.targets()) == ["regions", "class"] + assert s.aux() == {} + + def test_of_role_and_role_of(self) -> None: + s = _sample() + assert s.role_of("class") == "target" + assert list(s.of_role("input")) == ["image"] + + def test_items_of_type(self) -> None: + s = _sample() + assert [k for k, _ in s.items_of_type(Image)] == ["image"] + assert [k for k, _ in s.items_of_type(Image, Label)] == ["image", "class"] + + def test_roles_closed_set(self) -> None: + assert set(ROLES) == {"input", "target", "aux", "pred"} + + +class TestConstruction: + def test_role_for_unknown_field_raises(self) -> None: + with pytest.raises(KeyError, match="unknown field"): + TypedSample({"a": Label()}, roles={"b": "target"}) + + def test_invalid_role_raises(self) -> None: + with pytest.raises(ValueError, match="invalid role"): + TypedSample({"a": Label()}, roles={"a": "output"}) # type: ignore[dict-item] + + +class TestCopyOnWrite: + def test_set_role_returns_new(self) -> None: + s = _sample() + s2 = s.set_role("regions", "aux") + assert s2.role_of("regions") == "aux" + assert s.role_of("regions") == "target" # original untouched + + def test_set_role_unknown_and_invalid(self) -> None: + s = _sample() + with pytest.raises(KeyError): + s.set_role("nope", "aux") + with pytest.raises(ValueError): + s.set_role("class", "bogus") # type: ignore[arg-type] + + def test_replace_field_preserves_role(self) -> None: + s = _sample() + s2 = s.replace_field("class", Label("y")) + assert s2["class"].value == "y" and s2.role_of("class") == "target" + assert s["class"].value == "x" + + def test_replace_new_field_defaults_input(self) -> None: + s = _sample().replace_field("image", Image(np.zeros((2, 2, 3)))) + assert "image" in s and s.role_of("image") == "input" + + def test_drop(self) -> None: + s = _sample().drop("class") + assert "class" not in s and list(s.keys()) == ["image", "regions"] + + +class TestMappingProtocol: + def test_len_iter_contains_getitem(self) -> None: + s = _sample() + assert len(s) == 3 and "image" in s and list(iter(s)) == ["image", "regions", "class"] + assert isinstance(s["image"], Image) + + def test_fields_and_roles_are_copies(self) -> None: + s = _sample() + s.fields["image"] = None + s.roles["image"] = "target" + assert isinstance(s["image"], Image) and s.role_of("image") == "input" + + +class TestEquality: + def test_equal_with_array_fields(self) -> None: + a = TypedSample({"img": Image(np.zeros((2, 2, 3)))}) + b = TypedSample({"img": Image(np.zeros((2, 2, 3)))}) + assert a == b + + def test_unequal_arrays(self) -> None: + a = TypedSample({"img": Image(np.zeros((2, 2, 3)))}) + b = TypedSample({"img": Image(np.ones((2, 2, 3)))}) + assert a != b + + def test_unequal_roles_or_keys(self) -> None: + a = TypedSample({"x": Label("v")}) + assert a != TypedSample({"x": Label("v")}, roles={"x": "target"}) + assert a != TypedSample({"y": Label("v")}) + + def test_not_a_sample(self) -> None: + assert (TypedSample({"x": Label()}) == 5) is False + + def test_repr(self) -> None: + assert "Image[input]" in repr(_sample()) diff --git a/tests/test_bag_transform.py b/tests/test_bag_transform.py new file mode 100644 index 0000000..e23196a --- /dev/null +++ b/tests/test_bag_transform.py @@ -0,0 +1,111 @@ +"""Transforms — type dispatch, once-per-sample params, cross-field consistency, only filter. + +Native-kernel machinery is pinned via the test fixture ``FixtureFlip`` (sampleflux ships no +native augmentation transforms — libraries cover that through adapter coercion). +""" + +import numpy as np +import pytest + +from sampleflux import Image, Label, Mask, Pipeline, Regions, Transform, TypedSample, as_transform +from tests._bag_fixtures import FixtureFlip + + +def _seg() -> TypedSample: + return TypedSample( + { + "image": Image(np.arange(8 * 10 * 3).reshape(8, 10, 3).astype(np.float32)), + "mask": Mask(np.arange(8 * 10).reshape(8, 10)), + "regions": Regions(boxes=[[1, 1, 4, 4]], labels=["a"], canvas=(8, 10)), + "class": Label("a"), + } + ) + + +class TestKernelDispatchMachinery: + def test_cross_field_consistency(self) -> None: + out = FixtureFlip(p=1.0)(_seg()) + seg = _seg() + assert np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(seg["mask"])[:, ::-1]) + assert out["regions"].boxes == [[6, 1, 9, 4]] # W=10: x -> W-x + assert out["class"].value == "a" # no handler — untouched + + def test_p_zero_is_identity(self) -> None: + out = FixtureFlip(p=0.0)(_seg()) + assert np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert out["regions"].boxes == [[1, 1, 4, 4]] + + def test_only_filter(self) -> None: + out = FixtureFlip(p=1.0, only=["image"])(_seg()) + assert not np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(_seg()["mask"])) # mask skipped + assert out["regions"].boxes == [[1, 1, 4, 4]] # regions skipped + + def test_image_layout_chw(self) -> None: + s = TypedSample({"image": Image(np.arange(3 * 4 * 5).reshape(3, 4, 5), layout="CHW")}) + out = FixtureFlip(p=1.0)(s) + assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, :, ::-1]) + + def test_regions_uses_canvas_without_image(self) -> None: + s = TypedSample({"regions": Regions(boxes=[[2, 0, 5, 3]], canvas=(8, 10))}) + assert FixtureFlip(p=1.0)(s)["regions"].boxes == [[5, 0, 8, 3]] + + def test_regions_without_reference_width_raises(self) -> None: + s = TypedSample({"regions": Regions(boxes=[[2, 0, 5, 3]])}) # no image, no canvas + with pytest.raises(ValueError, match="no reference width"): + FixtureFlip(p=1.0)(s) + + def test_params_sampled_once(self) -> None: + # A partial-probability flip must be all-or-nothing across fields (shared decision), + # never per-field independent draws. + seg = _seg() + for _ in range(25): + out = FixtureFlip(p=0.5)(seg) + image_flipped = not np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])) + regions_flipped = out["regions"].boxes != seg["regions"].boxes + assert image_flipped == regions_flipped + + +class TestAdapterParity: + def test_v2_flip_matches_native_fixture(self) -> None: + # The bare-library path must move image/mask/boxes EXACTLY like the native fixture — + # this is the guarantee that let sampleflux drop its native flip for the library one. + v2 = pytest.importorskip("torchvision.transforms.v2") + seg = _seg() + native = FixtureFlip(p=1.0)(seg) + adapted = Pipeline([v2.RandomHorizontalFlip(p=1.0)])(seg) + assert np.array_equal(np.asarray(adapted["image"]), np.asarray(native["image"])) + assert np.array_equal(np.asarray(adapted["mask"]), np.asarray(native["mask"])) + assert adapted["regions"].boxes[0] == pytest.approx(native["regions"].boxes[0], abs=1e-4) + assert adapted["class"].value == native["class"].value == "a" + + +class TestPipelineAndFunction: + def test_pipeline_is_sequential(self) -> None: + s = TypedSample({"x": Image(np.ones((2, 2, 3), dtype=np.float32))}) + double = as_transform(lambda d: d * 2, handles=(Image,)) + out = Pipeline([double, double])(s) + assert np.allclose(np.asarray(out["x"]), 4.0) + + def test_function_transform_only_filter(self) -> None: + s = TypedSample({"a": Image(np.ones((2, 2, 3))), "b": Image(np.ones((2, 2, 3)))}) + out = as_transform(lambda d: d + 1, handles=(Image,), only=["a"])(s) + assert np.allclose(np.asarray(out["a"]), 2.0) and np.allclose(np.asarray(out["b"]), 1.0) + + def test_pipeline_repr(self) -> None: + assert "FixtureFlip" in repr(Pipeline([FixtureFlip()])) + + +class TestBaseTransform: + def test_default_get_params_and_passthrough(self) -> None: + # A transform with no kernels leaves every field alone. + s = TypedSample({"x": Label("v")}) + assert Transform()(s) == s + + def test_decode_not_implemented(self) -> None: + with pytest.raises(NotImplementedError, match="no decode"): + FixtureFlip().decode(TypedSample({"image": Image(np.zeros((2, 2, 3)))})) + + def test_zero_arg_construction(self) -> None: + assert FixtureFlip().p == 0.5 and Transform().only is None diff --git a/tests/test_structure_ops.py b/tests/test_structure_ops.py new file mode 100644 index 0000000..a9fa405 --- /dev/null +++ b/tests/test_structure_ops.py @@ -0,0 +1,115 @@ +"""Typed structure ops — SetRole/RenameField/DropField/CopyField/SelectFields + primary/merge.""" + +import numpy as np +import pytest + +from sampleflux import Image, Label, Regions, TypedSample, primary +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole + + +def _sample() -> TypedSample: + return TypedSample( + {"image": Image(np.zeros((2, 2, 3))), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")}, + roles={"regions": "target", "class": "target"}, + ) + + +class TestSetRole: + def test_retags(self) -> None: + out = SetRole(key="regions", role="aux")(_sample()) + assert out.role_of("regions") == "aux" and out.role_of("class") == "target" + + def test_lazy_validation(self) -> None: + assert SetRole().key == "" # zero-arg constructible + with pytest.raises(ValueError, match="'key'"): + SetRole()(_sample()) + # An invalid Literal is rejected at CONSTRUCTION (confluid schema enforcement) … + with pytest.raises(Exception, match="input_value='bogus'"): + SetRole(key="class", role="bogus") # type: ignore[arg-type] + # … and the defensive __call__ re-check guards post-construction mutation. + op = SetRole(key="class") + op.role = "bogus" # type: ignore[assignment] + with pytest.raises(ValueError, match="invalid role"): + op(_sample()) + + +class TestRenameField: + def test_renames_role_travels(self) -> None: + out = RenameField(src="regions", dst="boxes")(_sample()) + assert "regions" not in out and out.role_of("boxes") == "target" + + def test_rename_onto_existing_replaces(self) -> None: + out = RenameField(src="class", dst="image")(_sample()) + assert isinstance(out["image"], Label) and out.role_of("image") == "target" + + def test_validation(self) -> None: + with pytest.raises(ValueError, match="both 'src' and 'dst'"): + RenameField()(_sample()) + with pytest.raises(KeyError, match="unknown field"): + RenameField(src="nope", dst="x")(_sample()) + + +class TestDropField: + def test_drops(self) -> None: + out = DropField(key="class")(_sample()) + assert "class" not in out and list(out.keys()) == ["image", "regions"] + + def test_missing_raises_unless_ok(self) -> None: + with pytest.raises(KeyError, match="unknown field"): + DropField(key="nope")(_sample()) + assert DropField(key="nope", missing_ok=True)(_sample()) == _sample() + + +class TestCopyField: + def test_copies_with_source_role(self) -> None: + out = CopyField(src="regions", dst="regions_backup")(_sample()) + assert out["regions_backup"] is out["regions"] and out.role_of("regions_backup") == "target" + + def test_copy_with_explicit_role(self) -> None: + out = CopyField(src="regions", dst="regions_aux", role="aux")(_sample()) + assert out.role_of("regions_aux") == "aux" + + def test_validation(self) -> None: + with pytest.raises(ValueError, match="both 'src' and 'dst'"): + CopyField()(_sample()) + with pytest.raises(KeyError, match="unknown field"): + CopyField(src="nope", dst="x")(_sample()) + + +class TestSelectFields: + def test_keeps_only_and_orders(self) -> None: + out = SelectFields(keys=["class", "image"])(_sample()) + assert list(out.keys()) == ["class", "image"] and out.role_of("class") == "target" + + def test_validation(self) -> None: + with pytest.raises(ValueError, match="'keys'"): + SelectFields()(_sample()) + with pytest.raises(KeyError, match="unknown fields"): + SelectFields(keys=["image", "nope"])(_sample()) + + +class TestPrimaryAndMerge: + def test_primary_by_role(self) -> None: + s = _sample() + assert primary(s)[0] == "image" + assert primary(s, "target")[0] == "regions" # first target in insertion order + + def test_primary_missing_role_raises(self) -> None: + with pytest.raises(KeyError, match="no field with role 'pred'"): + primary(_sample(), "pred") + + def test_merge_union_last_wins(self) -> None: + a = TypedSample({"x": Label("a"), "shared": Label("from_a")}) + b = TypedSample({"y": Label("b"), "shared": Label("from_b")}, roles={"shared": "target"}) + m = TypedSample.merge(a, b) + assert list(m.keys()) == ["x", "shared", "y"] # union keeps first-seen position + assert m["shared"].value == "from_b" and m.role_of("shared") == "target" # last wins, role travels + + def test_merge_rejects_non_sample(self) -> None: + with pytest.raises(TypeError, match="expected TypedSample"): + TypedSample.merge(_sample(), "nope") # type: ignore[arg-type] + + def test_configurable_marks(self) -> None: + for cls in (SetRole, RenameField, DropField, CopyField, SelectFields): + assert getattr(cls, "__confluid_category__", None) == "op" + assert getattr(cls, "__confluid_group__", None) == "structure" diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py new file mode 100644 index 0000000..798ef45 --- /dev/null +++ b/tests/test_typed_collate.py @@ -0,0 +1,69 @@ +"""The typed collate — batched TypedSample convention (golden shapes consumers rely on).""" + +from dataclasses import dataclass + +import numpy as np +import pytest +import torch + +from sampleflux import Image, Label, Mask, TypedSample, collate, get_collate, register_item + + +@register_item +@dataclass +class _CollateBlob: + data: object = None + rate: float = 1.0 + + +def _sample(i: int) -> TypedSample: + return TypedSample( + { + "image": Image(np.full((4, 5, 3), float(i), dtype=np.float32)), + "mask": Mask(np.full((4, 5), i, dtype=np.int64)), + "class": Label(i, classes=["a", "b", "c"]), + }, + roles={"mask": "target", "class": "target"}, + ) + + +class TestTypedCollate: + def test_golden_shapes(self) -> None: + # THE batch convention consumers rely on: batched TypedSample, payloads stacked + # per field, per-item attrs as lists, roles preserved. + batch = collate([_sample(0), _sample(1), _sample(2)]) + assert isinstance(batch, TypedSample) + assert np.asarray(batch["image"]).shape == (3, 4, 5, 3) # stacked payload + assert np.asarray(batch["mask"]).shape == (3, 4, 5) + assert batch["class"].value == [0, 1, 2] # per-item attrs become lists + assert batch["class"].classes == [["a", "b", "c"]] * 3 + assert batch.roles == {"image": "input", "mask": "target", "class": "target"} + + def test_auto_dispatch_and_explicit_key(self) -> None: + samples = [_sample(0), _sample(1)] + auto = collate(samples) # TypedSample batch routes to "typed" automatically + explicit = get_collate("typed")(samples) + assert isinstance(auto, TypedSample) and isinstance(explicit, TypedSample) + assert np.array_equal(np.asarray(auto["image"]), np.asarray(explicit["image"])) + + def test_torch_payloads_stack_to_tensor(self) -> None: + samples = [ + TypedSample({"sig": _CollateBlob(torch.ones(8) * i, rate=float(i))}, roles={"sig": "input"}) + for i in range(2) + ] + batch = collate(samples) + assert isinstance(batch["sig"].data, torch.Tensor) and batch["sig"].data.shape == (2, 8) + assert batch["sig"].rate == [0.0, 1.0] + + def test_heterogeneous_batch_raises(self) -> None: + odd = TypedSample({"other": Label("x")}) + with pytest.raises(ValueError, match="do not match the batch fields"): + collate([_sample(0), odd]) + + def test_empty_batch_raises(self) -> None: + with pytest.raises(ValueError, match="empty batch"): + get_collate("typed")([]) + + def test_non_typed_items_raise(self) -> None: + with pytest.raises(TypeError, match="expected TypedSample"): + get_collate("typed")([1, 2, 3]) diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py new file mode 100644 index 0000000..1b66b9e --- /dev/null +++ b/tests/test_typed_flow.py @@ -0,0 +1,225 @@ +"""Typed FlowGraph — merge_from fan-in, step[key] bind, typed carriers through Flux, parity.""" + +from typing import Any, Dict, List, Optional + +import numpy as np +import pytest + +from sampleflux import FlowGraph, Flux, Image, Label, Mask, Transform, TypedSample, to_ops +from sampleflux.flow import from_ops, parse_flow +from sampleflux.ops.context import MergeFields +from sampleflux.ops.structure import RenameField, SetRole + + +class _AddOffset(Transform): + """Adds a configurable offset to every Image payload (bind target).""" + + handles = (Image,) + + def __init__(self, offset: float = 0.0, only: Optional[List[str]] = None) -> None: + super().__init__(only=only) + self.offset = offset + + def __call__(self, sample: TypedSample) -> TypedSample: + out = sample + for key, item in sample.items(): + if isinstance(item, Image) and (self.only is None or key in self.only): + out = out.replace_field(key, Image(np.asarray(item) + self.offset, layout=item.layout)) + return out + + +class _MakeMask(Transform): + """Derives a Mask field from the first Image (a branch producer).""" + + def __call__(self, sample: TypedSample) -> TypedSample: + image = next(item for item in sample.fields.values() if isinstance(item, Image)) + out = sample.replace_field("mask", Mask(np.asarray(image)[..., 0] > 0.5)) + return out.set_role("mask", "target") + + +def _seed(value: float = 0.0) -> TypedSample: + return TypedSample( + {"image": Image(np.full((2, 3, 3), value, dtype=np.float32)), "label": Label("x")}, + roles={"label": "target"}, + ) + + +class TestTypedFlowGraph: + def test_linear_typed_flow(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"plus": _AddOffset(offset=2.0)}) + (out,) = list(graph) + assert isinstance(out, TypedSample) and np.allclose(np.asarray(out["image"]), 3.0) + + def test_merge_from_union(self) -> None: + # Fork: derive a mask on a branch, SELECT the new field, union it back into the main + # stream. (Selecting is the idiom — a full branch bag would also carry its own + # 'image', and last-wins would overwrite the boosted one.) + from sampleflux.ops.structure import SelectFields + + flow = { + "start": {}, + "masked": {"op": _MakeMask(), "from": "start"}, + "mask_only": {"op": SelectFields(keys=["mask"]), "from": "masked"}, + "boosted": {"op": _AddOffset(offset=1.0), "from": "start"}, + "out": {"from": "boosted", "merge_from": ["mask_only"]}, + } + graph = FlowGraph(source=[_seed(0.75)], flow=flow, outputs="out") + (out,) = list(graph) + assert np.allclose(np.asarray(out["image"]), 1.75) # the boosted branch's image survives + assert "mask" in out and out.role_of("mask") == "target" # the selected branch field + assert out["label"].value == "x" + + def test_merge_collision_last_wins(self) -> None: + # Both branches carry 'image'; the merge source is listed LAST -> its image wins. + flow = { + "start": {}, + "a": {"op": _AddOffset(offset=1.0), "from": "start"}, + "b": {"op": _AddOffset(offset=5.0), "from": "start"}, + "out": {"from": "a", "merge_from": ["b"]}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out")) + assert np.allclose(np.asarray(out["image"]), 5.0) # b (last) wins over a + + def test_rename_avoids_collision(self) -> None: + flow = { + "start": {}, + "a": {"op": _AddOffset(offset=1.0), "from": "start"}, + "b_renamed": {"op": RenameField(src="image", dst="image_b"), "from": "start"}, + "out": {"from": "a", "merge_from": ["b_renamed"]}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out")) + assert np.allclose(np.asarray(out["image"]), 1.0) # branch a intact + assert "image_b" in out # branch b united under its renamed key + + def test_step_key_bind(self) -> None: + # bind offset := the 'probe' step's image payload mean is NOT expressible without a + # value op — bind the FIELD instead and let the op read it: offset receives the + # Image item from probe via step[image]. + class _OffsetFromItem(Transform): + def __init__(self, item: Any = None) -> None: + super().__init__() + self.item = item + + def __call__(self, sample: TypedSample) -> TypedSample: + offset = float(np.asarray(self.item).mean()) + out = sample + for key, value in sample.items(): + if isinstance(value, Image): + out = out.replace_field(key, Image(np.asarray(value) + offset, layout=value.layout)) + return out + + flow = { + "start": {}, + "probe": {"op": _AddOffset(offset=2.0), "from": "start"}, # image becomes 2.0 + "final": {"op": _OffsetFromItem(), "from": "start", "bind": {"item": "probe[image]"}}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) + assert np.allclose(np.asarray(out["image"]), 2.0) # 0.0 + mean(2.0) + + def test_bare_step_bind_is_primary(self) -> None: + class _CapturePrimary(Transform): + def __init__(self, item: Any = None) -> None: + super().__init__() + self.item = item + + def __call__(self, sample: TypedSample) -> TypedSample: + assert isinstance(self.item, Image) # primary input-role field of the bound step + return sample + + flow = { + "start": {}, + "probe": {"op": _AddOffset(offset=1.0), "from": "start"}, + "final": {"op": _CapturePrimary(), "from": "start", "bind": {"item": "probe"}}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) + assert isinstance(out, TypedSample) + + def test_typed_step_with_legacy_fanin_raises(self) -> None: + flow = { + "start": {}, + "a": {"op": _AddOffset(offset=1.0), "from": "start"}, + "out": {"from": "a", "target_from": "start"}, + } + graph = FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out") + with pytest.raises(TypeError, match="LEGACY fan-in"): + list(graph) + + def test_merge_and_legacy_fanin_mutually_exclusive(self) -> None: + with pytest.raises(ValueError, match="mutually exclusive"): + parse_flow({"a": {}, "b": {"from": "a", "merge_from": ["a"], "target_from": "a"}}) + + def test_merge_from_forward_ref_raises(self) -> None: + with pytest.raises(ValueError, match="EARLIER step"): + parse_flow({"a": {"merge_from": ["b"]}, "b": {}}) + + +class TestTypedLoweringParity: + def _flow(self) -> Dict[str, Any]: + from sampleflux.ops.structure import SelectFields + + return { + "start": {}, + "masked": {"op": _MakeMask(), "from": "start"}, + "mask_only": {"op": SelectFields(keys=["mask"]), "from": "masked"}, + "boosted": {"op": _AddOffset(offset=1.0), "from": "start"}, + "out": {"from": "boosted", "merge_from": ["mask_only"]}, + } + + def test_to_ops_runs_on_flux(self) -> None: + # The lowered flat op list (MergeFields wiring) matches the native FlowGraph result. + steps, outputs = parse_flow(self._flow()) + native = list(FlowGraph(source=[_seed(0.25)], flow=self._flow(), outputs="out")) + lowered = list(Flux(source=[_seed(0.25)], ops=to_ops(steps, outputs))) + assert len(native) == len(lowered) == 1 + assert native[0] == lowered[0] + + def test_round_trip_from_ops(self) -> None: + steps, outputs = parse_flow(self._flow()) + ops = to_ops(steps, outputs) + assert any(isinstance(op, MergeFields) for op in ops) + lifted, lifted_out = from_ops(ops) + relowered = to_ops(*parse_flow(lifted, lifted_out)) + native = list(Flux(source=[_seed(0.5)], ops=relowered)) + assert len(native) == 1 and "mask" in native[0] + + def test_key_bind_round_trip(self) -> None: + class _Reader(Transform): + def __init__(self, item: Any = None) -> None: + super().__init__() + self.item = item + + def __call__(self, sample: TypedSample) -> TypedSample: + return sample.replace_field("echo", self.item) + + flow = { + "start": {}, + "probe": {"op": _AddOffset(offset=3.0), "from": "start"}, + "final": {"op": _Reader(), "from": "start", "bind": {"item": "probe[image]"}}, + } + steps, outputs = parse_flow(flow) + ops = to_ops(steps, outputs) + lifted, _ = from_ops(ops) + # the key-bind grammar survives the round trip + final_step = lifted["final"] if "final" in lifted else list(lifted.values())[-1] + assert isinstance(final_step, dict) and final_step["bind"]["item"].endswith("[image]") + (out,) = list(Flux(source=[_seed(0.0)], ops=ops)) + assert np.allclose(np.asarray(out["echo"]), 3.0) + + +class TestTypedThroughFlux: + def test_default_flux_carries_typed_verbatim(self) -> None: + # No native=True needed: a TypedSample source item is NEVER coerced to legacy Sample. + flux = Flux(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) + (out,) = list(flux) + assert isinstance(out, TypedSample) and np.allclose(np.asarray(out["image"]), 2.0) + + def test_getitem_typed(self) -> None: + flux = Flux(source=[_seed(1.0), _seed(2.0)], ops=[SetRole(key="image", role="aux")]) + assert flux[1].role_of("image") == "aux" + + def test_compose_ops_route_typed(self) -> None: + from sampleflux.ops.transform_chain import TransformChain + + flux = Flux(source=[_seed(1.0)], ops=[TransformChain(ops=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])]) + (out,) = list(flux) + assert np.allclose(np.asarray(out["image"]), 4.0) diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py new file mode 100644 index 0000000..0186f45 --- /dev/null +++ b/tests/test_typed_storage.py @@ -0,0 +1,228 @@ +"""Typed field-group storage — HDF5/Zarr/Directory round-trips, carrier guards, typed queries.""" + +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pytest + +from sampleflux import Image, Label, Regions, Sample, TypedSample, register_item +from sampleflux.storage.base import restore_attrs, split_attrs +from sampleflux.storage.directory import DirectorySink, DirectorySource +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.query import MetadataFilterSource, scan_hdf5_metadata, scan_zarr_metadata +from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource + + +@register_item +@dataclass +class _StoreSig: + """An externally-registered data-bearing item (the shape a domain signal item takes).""" + + data: object = None + samplerate: float = 1.0 + mask: object = None # an ARRAY-valued attr — exercises the attrs/ dataset path + + +def _samples() -> list: + # Ragged across samples: different box counts, one field with an array-valued attr. + s0 = TypedSample( + { + "image": Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW"), + "sig": _StoreSig(np.arange(8, dtype=np.float32), samplerate=20e6, mask=np.array([1, 0, 1], dtype=np.uint8)), + "regions": Regions(boxes=[[0, 0, 1, 1], [1, 1, 2, 2]], labels=["a", "b"], canvas=(2, 2)), + "label": Label("drone", classes=["x", "drone"]), + }, + roles={"regions": "target", "label": "target", "sig": "aux"}, + ) + s1 = TypedSample( + { + "image": Image(np.ones((2, 2, 3), dtype=np.float32)), + "sig": _StoreSig(np.zeros(4, dtype=np.float32), samplerate=1e6, mask=np.array([0], dtype=np.uint8)), + "regions": Regions(boxes=[[0, 0, 2, 2]], labels=["c"], canvas=(2, 2)), + "label": Label("x", classes=["x", "drone"]), + }, + roles={"regions": "target", "label": "target", "sig": "aux"}, + ) + return [s0, s1] + + +def _assert_round_trip(back: list, expect: list) -> None: + assert len(back) == len(expect) + for got, want in zip(back, expect): + assert list(got.keys()) == list(want.keys()) # insertion order preserved + assert got.roles == want.roles + assert got["image"].layout == want["image"].layout + assert np.array_equal(np.asarray(got["image"]), np.asarray(want["image"])) + assert got["sig"].samplerate == want["sig"].samplerate + assert np.array_equal(np.asarray(got["sig"].data), np.asarray(want["sig"].data)) + assert np.array_equal(np.asarray(got["sig"].mask), np.asarray(want["sig"].mask)) # array attr + assert got["regions"].boxes == want["regions"].boxes # ragged boxes + assert got["regions"].canvas == want["regions"].canvas # tuple preserved + assert isinstance(got["regions"].canvas, tuple) + assert got["label"].value == want["label"].value and got["label"].classes == want["label"].classes + + +class TestAttrWireFormat: + def test_split_restore_round_trip(self) -> None: + attrs = { + "s": "text", + "i": 3, + "f": 1.5, + "b": True, + "none": None, + "tup": (2, 3), + "nested": {"a": [1, (2, 3)]}, + "arr": np.arange(4), + } + plain, arrays = split_attrs(attrs) + assert list(arrays) == ["arr"] + back = restore_attrs(plain, arrays) + assert back["s"] == "text" and back["i"] == 3 and back["f"] == 1.5 and back["b"] is True + assert back["none"] is None + assert back["tup"] == (2, 3) and isinstance(back["tup"], tuple) + assert back["nested"] == {"a": [1, (2, 3)]} + assert np.array_equal(back["arr"], np.arange(4)) + + def test_numpy_scalars_become_python(self) -> None: + plain, _ = split_attrs({"x": np.float32(2.5)}) + assert restore_attrs(plain, {})["x"] == 2.5 + + +class TestHDF5Typed: + def test_round_trip(self, tmp_path: Path) -> None: + path = tmp_path / "t.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + for s in _samples(): + sink.write(s) + sink.flush() + source = HDF5Source(path=path) + with source: + assert source.is_typed and len(source) == 2 + _assert_round_trip(list(source), _samples()) + + def test_carrier_guards_both_directions(self, tmp_path: Path) -> None: + typed_path = tmp_path / "typed.h5" + sink = HDF5Sink(path=typed_path, overwrite=True) + with sink: + sink.write(_samples()[0]) + appender = HDF5Sink(path=typed_path) + with appender: + with pytest.raises(TypeError, match="typed field-group layout"): + appender.write(Sample(input=np.zeros(3))) + + legacy_path = tmp_path / "legacy.h5" + legacy = HDF5Sink(path=legacy_path, overwrite=True) + with legacy: + legacy.write(Sample(input=np.zeros(3), metadata={"k": 1})) + appender2 = HDF5Sink(path=legacy_path) + with appender2: + with pytest.raises(TypeError, match="legacy Sample layout"): + appender2.write(_samples()[0]) + + def test_legacy_path_unchanged(self, tmp_path: Path) -> None: + path = tmp_path / "legacy.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + sink.write(Sample(input=np.arange(4, dtype=np.float32), target=1, metadata={"snr_db": 12.0})) + sink.flush() + source = HDF5Source(path=path) + with source: + assert not source.is_typed + (back,) = list(source) + assert isinstance(back, Sample) and back.meta["snr_db"] == 12.0 + + +class TestZarrTyped: + def test_group_round_trip(self, tmp_path: Path) -> None: + path = str(tmp_path / "g.zarr") + sink = ZarrGroupSink(path=path) + sink.open() + for s in _samples(): + sink.write(s) + source = ZarrGroupSource(path=path) + assert source.is_typed and len(source) == 2 + _assert_round_trip(list(source), _samples()) + + def test_group_carrier_guard(self, tmp_path: Path) -> None: + path = str(tmp_path / "g.zarr") + sink = ZarrGroupSink(path=path) + sink.open() + sink.write(_samples()[0]) + with pytest.raises(TypeError, match="typed field-group layout"): + sink.write(Sample(input=np.zeros(3))) + + def test_batch_typed_rows(self, tmp_path: Path) -> None: + path = str(tmp_path / "b.zarr") + sink = ZarrBatchSink(path=path, shape=[2, 2, 3], dtype="float32", overwrite=True) + sink.open() + for s in _samples(): + sink.write(s) # appends the PRIMARY input field's payload + source = ZarrBatchSource(path=path) + rows = list(source) + assert len(rows) == 2 and all(isinstance(r, TypedSample) for r in rows) + assert isinstance(rows[0]["image"], Image) and rows[0]["image"].layout == "CHW" # uniform template + assert np.asarray(rows[1]["image"]).shape == (2, 2, 3) + + +class TestDirectoryTyped: + def test_round_trip(self, tmp_path: Path) -> None: + path = tmp_path / "dir" + sink = DirectorySink(path=path) + sink.open() + for s in _samples(): + sink.write(s) + source = DirectorySource(path=path) + assert len(source) == 2 + _assert_round_trip(list(source), _samples()) + + def test_missing_root_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + len(DirectorySource(path=tmp_path / "nope")) + + +class TestTypedQueries: + def test_hdf5_scan_is_nested_and_payload_free(self, tmp_path: Path) -> None: + path = tmp_path / "q.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + for s in _samples(): + sink.write(s) + scans = list(scan_hdf5_metadata(path)) + assert len(scans) == 2 + _, meta = scans[0] + assert meta["sig"]["samplerate"] == 20e6 # plain attr decoded + assert meta["image"]["layout"] == "CHW" + assert "shape" in str(meta["sig"]["mask"]) # array attr is a STUB, not the array + + def test_zarr_scan_nested(self, tmp_path: Path) -> None: + path = str(tmp_path / "q.zarr") + sink = ZarrGroupSink(path=path) + sink.open() + for s in _samples(): + sink.write(s) + scans = list(scan_zarr_metadata(path)) + assert scans[1][1]["sig"]["samplerate"] == 1e6 + + def test_where_field_attr_expression(self, tmp_path: Path) -> None: + path = tmp_path / "w.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + for s in _samples(): + sink.write(s) + source = HDF5Source(path=path) + source.open() + fast = MetadataFilterSource(source=source, where="sig.samplerate > 1e7") + assert len(fast) == 1 + (match,) = list(fast) + assert isinstance(match, TypedSample) and match["sig"].samplerate == 20e6 + + def test_full_iteration_fallback_on_typed_samples(self) -> None: + # A plain list source (no iter_metadata protocol) of TypedSamples still filters. + filt = MetadataFilterSource(source=_samples(), where="image.layout == 'CHW'") + assert len(filt) == 1 + + def test_missing_attr_is_non_match(self) -> None: + filt = MetadataFilterSource(source=_samples(), where="sig.nonexistent > 0") + assert len(filt) == 0 From cc8c3f318e5aca1302370993ebfc5a6a262b4ca3 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 22 Jul 2026 18:01:40 +0200 Subject: [PATCH 028/102] =?UTF-8?q?feat(sampleflux):=20typed-native=20ops?= =?UTF-8?q?=20=E2=80=94=20ConvertToImage/Threshold/ConnectedComponents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive typed-bag twins of three generic ops so a typed pipeline can go array -> Image -> Mask -> Regions with no legacy Sample (legacy *Op untouched). 1160 passed, mypy clean (108 files), flake8 clean. - ops/image.py: ConvertToImage(Transform) — array item -> Image field (reuses value_to_image) - ops/numpy.py: Threshold(Transform) -> Mask; ConnectedComponents(Transform) -> Regions with (row_min,row_max,col_min,col_max) bin-box tuples (the connected-components contract) - bag/items.py: Regions.extras dict (per-box parallel arrays / region-set measurements) - docs/architecture.md: record for native typed transforms that change a field's TYPE - tests/test_typed_generic_ops.py: parity vs legacy + the array->Image->Mask->Regions chain --- docs/architecture.md | 97 ++++++++++++ sampleflux/bag/items.py | 4 + sampleflux/ops/image.py | 87 +++++++++++ sampleflux/ops/numpy.py | 162 ++++++++++++++++++++ tests/test_typed_generic_ops.py | 253 ++++++++++++++++++++++++++++++++ 5 files changed, 603 insertions(+) create mode 100644 tests/test_typed_generic_ops.py diff --git a/docs/architecture.md b/docs/architecture.md index 5948415..a5b8a4a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -434,3 +434,100 @@ out = Pipeline([ decorator). The rename rationale is pinned here and in the module docstrings. - **Promoting this from PoC to the default model** is a workspace-wide decision that would re-scope the classic-engine mandates and port every consumer — out of scope for the proof of concept. + +--- + +## Native typed transforms that change a field's TYPE (`ConvertToImage`/`Threshold`/`ConnectedComponents`, 2026-07-22) + +### Context + +Two shapes of typed transform exist. The first is the augmentation shape the base `Transform` +was built for: it `handles` an item type and, per handled field, applies a registered kernel that +returns *the same type* (a flip returns a flipped `Image`), so `image`, `mask`, and `boxes` move +together and library transforms (torchvision v2 / albumentations) drop in through the adapter +coercion registry. The workspace deliberately ships **no** native transforms of that shape — +libraries cover it. + +But a running detection/segmentation front-end needs a different shape: **read one field, write a +field of a DIFFERENT type**. Turning a numeric array into a displayable image, thresholding an +array into a boolean mask, and labelling that mask into a set of bin boxes are each a *type +change* (`array → Image`, `array → Mask`, `Mask → Regions`), not an in-place per-type edit. No +library provides them, and the legacy classic-engine ops that do (`ConvertToImageOp`, +`ThresholdOp`, `ConnectedComponentsOp`) operate on the `Sample(input, target, metadata)` triple, +which the typed world does not carry. Without typed equivalents a `TypedSample` pipeline could not +reach `Regions` from a raw array — the critical path for typed detection was blocked. + +### Decision + +Add native typed **twins** that subclass `Transform` and OVERRIDE `__call__` (rather than register +a kernel), reading one field and writing a different-typed item — the same shape the domain +package's `Spectrogram` twin (`Signal → Spectrogram`) already established: + +- A twin declares `handles` / `consumes` / `produces` **truthfully** as graph metadata (e.g. + `ConnectedComponents`: `consumes=(Mask,)`, `produces=(Regions,)`), but does its work in + `__call__`, not through the kernel-dispatch loop — kernel dispatch is for same-type per-field + edits, and a type change has one input field and one output field. +- The source field is resolved by a small `_find_*` helper: an explicit `field=` name, else the + first item of the natural type (a `Mask` for `ConnectedComponents`) or the first array-bearing + item — every miss raises a `ValueError` naming the sample's fields. +- The output is written with `sample.replace_field(output, item)` + `sample.set_role(output, role)` + (copy-on-write), and the role is chosen semantically: the working image is `input`, a threshold + mask and raw connected-component boxes are `aux` (intermediates, and specifically NOT `pred` — + that role is reserved for a detector's output). +- Each twin **reuses its legacy op's math verbatim** so the numbers are pinned identical: + `ConvertToImage` calls the shared `_render_rgb`/`_bound_longest_side` render core; + `ConnectedComponents` calls the shared `connected_component_bboxes` helper; `Threshold` + delegates to a legacy `ThresholdOp` instance run on a shim `Sample`. The twins are STRICTLY + ADDITIVE — the legacy ops are untouched, because many consumers still use them via the `Sample` + path. + +The generic connected-components output format is a hard contract: `Regions.boxes` is a list of +`(row_min, row_max, col_min, col_max)` inclusive integer tuples (**row bounds first, then column +bounds**). A downstream back-projection reads exactly that order to map bins to a world / signal +coordinate frame, so the tuple order is load-bearing, not incidental. + +### Consequences + +- A `TypedSample` carrying a raw 2-D array runs `ConvertToImage → Threshold → ConnectedComponents` + end-to-end and arrives at a `Regions` field with no legacy `Sample` anywhere — the typed + detection/segmentation front-end is unblocked. +- Parity is free and provable: because each twin reuses the legacy math, a twin's output is + byte-identical to a legacy run on the equivalent `Sample` (pinned in + `tests/test_typed_generic_ops.py`). +- `ConvertToImage` does NOT republish `image_width_px` / `image_height_px` (the legacy op wrote + them into the shared metadata dict). The `Image` item's array SHAPE carries the pixel + dimensions, and the typed model has no shared dict to write into — a consumer reads the dims off + the payload. +- `Threshold`'s `{meta_key}` expression grammar has no typed home (an item owns its own metadata; + there is no shared sample dict), so only numeric literals and `$ENV` bounds resolve in the twin; + a `{key}` bound raises loudly. Literal dB thresholds — the critical path — are unaffected. +- The twins carry `category="op"` + `group="image"`/`"numpy"`, so they are discoverable exactly + like the legacy ops (their modules were already entry-pointed; a class added to a registered + module needs no new entry point). + +### Example + +```python +from sampleflux import TypedSample, Mask +from sampleflux.ops.image import ConvertToImage +from sampleflux.ops.numpy import Threshold, ConnectedComponents + +sample = TypedSample({"spec": Mask(db_spectrogram)}) # a raw 2-D array item +sample = ConvertToImage()(sample) # + Image field (role "input") +sample = Threshold(field="spec", low_level=-30.0)(sample) # + Mask field (role "aux") +sample = ConnectedComponents(field="mask")(sample) # + Regions field (role "aux") + +sample["boxes"].boxes # [(row_min, row_max, col_min, col_max), ...] — the pinned bin-box contract +``` + +### What you may change (and where it's documented) + +- **A twin's source-field resolution or output role** — keep the `_find_*` → `replace_field` → + `set_role` shape and a loud `ValueError` on a miss; `aux` vs `pred` is a semantic choice + (raw detections are `aux`). +- **The `(row_min, row_max, col_min, col_max)` bin-box order is a contract** — a back-projection + depends on it; changing it is an architectural change that must update this record and every + consumer. +- **Do not modify the legacy ops or reimplement their math in a twin** — a twin reuses the legacy + math so parity is guaranteed; the twins are additive and the legacy `Sample`-path consumers must + keep working. diff --git a/sampleflux/bag/items.py b/sampleflux/bag/items.py index 662e85c..057c55b 100644 --- a/sampleflux/bag/items.py +++ b/sampleflux/bag/items.py @@ -169,12 +169,16 @@ class Regions: scores: Optional per-box confidence scores. canvas: Optional ``(H, W)`` reference frame — the coordinate system boxes live in, so a geometric transform (flip / resize) has a self-contained frame. + extras: Auxiliary PER-BOX parallel arrays and region-set measurements keyed by name + (e.g. per-box durations/bandwidths/power readings) — item-scoped metadata that + travels WITH the boxes it describes. """ boxes: List[Any] = field(default_factory=list) labels: Optional[List[Any]] = None scores: Optional[List[Any]] = None canvas: Optional[Tuple[int, int]] = None + extras: Dict[str, Any] = field(default_factory=dict) @register_item diff --git a/sampleflux/ops/image.py b/sampleflux/ops/image.py index eff0486..733341b 100644 --- a/sampleflux/ops/image.py +++ b/sampleflux/ops/image.py @@ -26,6 +26,10 @@ from loggair import get_logger from PIL import Image, ImageDraw +from sampleflux.bag.items import Image as ImageItem +from sampleflux.bag.items import NDArrayItem, item_data +from sampleflux.bag.sample import TypedSample +from sampleflux.bag.transform import Transform from sampleflux.sample import Sample from sampleflux.typespec import ArrayType as _ArrayType from sampleflux.typespec import PythonType, SampleType, UnionType @@ -704,9 +708,92 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=self.normalize_to_uint8(arr, self.vmin, self.vmax)) +@configurable(category="op", group="image") +class ConvertToImage(Transform): + """Typed twin of :class:`ConvertToImageOp` — an array-bearing field → an ``Image`` item. + + The typed-bag counterpart of :class:`ConvertToImageOp`: instead of rendering + ``sample.input`` into a PIL image in place, it reads an array-bearing field from a + :class:`~sampleflux.TypedSample` and writes a fresh :class:`~sampleflux.Image` item + (HWC ``uint8`` RGB) under ``output``, tagged with the ``input`` role (it is the + pipeline's working image). Any other field passes through untouched. + + Rendering is byte-identical to the legacy op — it reuses the SAME + :func:`value_to_image` core (:func:`_render_rgb` → optional flip → resize): a 2-D map is + colormapped, a 3-D array treated as an image, a boolean mask becomes 0/255, floats are + min-max normalized. Sizing matches the legacy op: + + * ``width`` and ``height`` both > 0 → resize to exactly that raster; + * otherwise → bound the longest side by ``max_size``, preserving aspect. + + Unlike the legacy op it does NOT publish ``image_width_px`` / ``image_height_px`` — the + ``Image`` item's array SHAPE carries the pixel dimensions, so a downstream consumer + (e.g. a back-projection) reads them straight off the payload; there is no shared + metadata dict to publish into in the typed model. + + Args: + colormap: Colormap applied to 2-D maps — a supported ``Colormap`` name (``"gray"`` = greyscale). + width: Exact output width in pixels; resize to ``(width, height)`` when both width and height are > 0. + height: Exact output height in pixels; resize to ``(width, height)`` when both width and height are > 0. + max_size: When ``width``/``height`` aren't both set, bound the longest side to this many pixels (aspect kept). + flip_vertical: Mirror the image top-to-bottom (e.g. spectrogram row 0 = f_min → display f_max at the top). + field: Name of the source field to render; blank (default) picks the first array-bearing item in the bag. + output: Name of the field the ``Image`` item is written to (added if new); its role is set to ``input``. + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (ImageItem,) + + def __init__( + self, + colormap: Colormap = "gray", + width: int = 0, + height: int = 0, + max_size: int = 512, + flip_vertical: bool = False, + field: str = "", + output: str = "image", + ) -> None: + super().__init__() + self.colormap: Colormap = colormap + self.width = int(width) + self.height = int(height) + self.max_size = int(max_size) + self.flip_vertical = bool(flip_vertical) + self.field = field + self.output = output + + def _find_source(self, sample: TypedSample) -> Any: + """Resolve the payload to render (``self.field`` or the first array-bearing item).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError(f"ConvertToImage: field {self.field!r} not in sample (fields: {list(sample.keys())})") + return item_data(sample[self.field]) + for _key, item in sample.items(): + arr = _coerce_to_ndarray(item_data(item)) + if arr is not None and arr.ndim in (2, 3): + return item_data(item) + raise ValueError(f"ConvertToImage: no array-bearing field in sample (fields: {list(sample.keys())})") + + def __call__(self, sample: TypedSample) -> TypedSample: + rgb = _render_rgb(self._find_source(sample), self.colormap) + if self.flip_vertical: + rgb = rgb[::-1, :, :] + if self.width > 0 and self.height > 0: + out_arr = np.array( + Image.fromarray(rgb).resize((self.width, self.height), resample=Image.Resampling.BILINEAR) + ) + else: + out_arr = _bound_longest_side(rgb, self.max_size) + out = sample.replace_field(self.output, ImageItem(out_arr, layout="HWC")) + return out.set_role(self.output, "input") + + __all__ = [ "Colormap", "COLORMAPS", + "ConvertToImage", "ConvertToImageOp", "NormalizeToUint8Op", "value_to_image", diff --git a/sampleflux/ops/numpy.py b/sampleflux/ops/numpy.py index c4a4626..a0c6e7c 100644 --- a/sampleflux/ops/numpy.py +++ b/sampleflux/ops/numpy.py @@ -7,6 +7,9 @@ from confluid import configurable from loggair import get_logger +from sampleflux.bag.items import Mask, NDArrayItem, Regions, item_data +from sampleflux.bag.sample import TypedSample +from sampleflux.bag.transform import Transform from sampleflux.sample import Sample from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType @@ -430,6 +433,85 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(input=mask) +@configurable(category="op", group="numpy") +class Threshold(Transform): + """Typed twin of :class:`ThresholdOp` — an array-bearing field → a boolean ``Mask`` item. + + The typed-bag counterpart of :class:`ThresholdOp`: it reads the array at ``field`` (blank = + the first array-bearing item in the bag) and thresholds it into a boolean mask with the SAME + bound / comparison / expression math — this twin REUSES the legacy op verbatim, so the booleans + are identical — writing a :class:`~sampleflux.Mask` item under ``output`` tagged ``aux`` (a + threshold mask is an intermediate that a later op — e.g. :class:`ConnectedComponents` — + consumes, not a model input or target). Any other field passes through untouched. + + Which mask is produced depends on which bounds are set, and the comparison for each is picked + by ``low_op`` / ``high_op`` (see :class:`ThresholdOp` for the full presence-driven rules and + the open-vs-closed interval semantics). At least one of ``low_level`` / ``high_level`` MUST be + set; passing neither raises ``ValueError`` when applied (the zero-arg default stays + constructible per the lazy-init convention). + + Each bound is a numeric literal or a ``resolve_expression`` string — ``5.5`` / ``"5.5"`` + (literal) or ``"$REF_SNR"`` (environment variable). NOTE: ``{meta_key}`` expressions have no + typed metadata source in the bag model (an item owns its own metadata; there is no shared + sample dict), so only literals and ``$ENV`` resolve here — a ``{key}`` bound raises ``KeyError``. + + Args: + low_level: Lower bound (numeric literal or ``$ENV`` expression) compared with ``low_op`` when set; + ``None`` disables the lower bound. + high_level: Upper bound (numeric literal or ``$ENV`` expression) compared with ``high_op`` when set; + ``None`` disables the upper bound. + low_op: Lower-bound comparison — ``">"`` (strict, default) or ``">="`` (inclusive). + high_op: Upper-bound comparison — ``"<"`` (strict, default) or ``"<="`` (inclusive). + field: Name of the array field to threshold; blank (default) picks the first array-bearing item. + output: Name of the field the boolean ``Mask`` item is written to (added if new; role ``aux``). + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (Mask,) + + def __init__( + self, + low_level: Optional[Union[float, int, str]] = None, + high_level: Optional[Union[float, int, str]] = None, + low_op: LowComparison = ">", + high_op: HighComparison = "<", + field: str = "", + output: str = "mask", + ) -> None: + super().__init__() + self.low_level = low_level + self.high_level = high_level + self.low_op = low_op + self.high_op = high_op + self.field = field + self.output = output + + def _find_array(self, sample: TypedSample) -> np.ndarray: + """Resolve the array to threshold (``self.field`` or the first array-bearing item).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError(f"Threshold: field {self.field!r} not in sample (fields: {list(sample.keys())})") + data = item_data(sample[self.field]) + if not isinstance(data, np.ndarray): + raise TypeError(f"Threshold: field {self.field!r} payload is {type(data).__name__}, expected an array") + return data + for _key, item in sample.items(): + data = item_data(item) + if isinstance(data, np.ndarray): + return data + raise ValueError(f"Threshold: no array-bearing field in sample (fields: {list(sample.keys())})") + + def __call__(self, sample: TypedSample) -> TypedSample: + arr = self._find_array(sample) + # Reuse the legacy op's threshold math VERBATIM on a shim Sample so the booleans are + # identical; the shim's empty metadata is why only literals / $ENV bounds resolve here. + legacy = ThresholdOp(self.low_level, self.high_level, self.low_op, self.high_op) + mask = legacy(Sample(input=arr, target=None, metadata={})).input + out = sample.replace_field(self.output, Mask(mask)) + return out.set_role(self.output, "aux") + + def connected_component_bboxes( mask: np.ndarray, min_area_bins: int = 1, connectivity: int = 4 ) -> List[Tuple[int, int, int, int]]: @@ -654,3 +736,83 @@ def __call__(self, sample: Sample) -> Sample: # validates min_area_bins / connectivity and raises the scipy ImportError. bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) return sample._replace(input=bboxes) + + +@configurable(category="op", group="numpy") +class ConnectedComponents(Transform): + """Typed twin of :class:`ConnectedComponentsOp` — a boolean ``Mask`` → a ``Regions`` item. + + The typed-bag counterpart of :class:`ConnectedComponentsOp`: it reads the + :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, else the first + array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into + ``(row_min, row_max, col_min, col_max)`` inclusive bin-box tuples via the SAME shared + :func:`connected_component_bboxes` helper the legacy op uses (so the numbers are identical), + writing them as a :class:`~sampleflux.Regions` item under ``output``. That field is tagged + ``aux``: these are RAW detections (thresholded blobs), NOT model predictions — the ``pred`` + role is reserved for a detector's output. Any other field passes through untouched. + + The ``Regions.boxes`` list holds ``(row_min, row_max, col_min, col_max)`` tuples — the exact + generic bin-box format (row bounds first, then column bounds; inclusive) a downstream + back-projection reads to map bins to a signal / world coordinate frame. Components smaller than + ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or 8-neighborhood. Requires + ``scipy`` (``pip install sampleflux[vision]``). + + Args: + min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). + connectivity: Pixel neighborhood — ``4`` (orthogonal only) or ``8`` (orthogonal + diagonal). + field: Name of the ``Mask`` field to label; blank (default) picks the first ``Mask`` (else first array). + output: Name of the field the ``Regions`` item is written to (added if new; role ``aux``). + """ + + handles = (Mask,) + consumes = (Mask,) + produces = (Regions,) + + def __init__( + self, + min_area_bins: int = 1, + connectivity: int = 4, + field: str = "", + output: str = "boxes", + ) -> None: + super().__init__() + self.min_area_bins = int(min_area_bins) + self.connectivity = int(connectivity) + self.field = field + self.output = output + + def _find_mask(self, sample: TypedSample) -> np.ndarray: + """Resolve the mask to label (``self.field``, else the first ``Mask``, else the first array).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError( + f"ConnectedComponents: field {self.field!r} not in sample (fields: {list(sample.keys())})" + ) + data = item_data(sample[self.field]) + else: + data = None + for _key, item in sample.items(): + if isinstance(item, Mask): + data = item_data(item) + break + if data is None: + for _key, item in sample.items(): + payload = item_data(item) + if isinstance(payload, np.ndarray): + data = payload + break + if data is None: + raise ValueError( + f"ConnectedComponents: no Mask or array-bearing field in sample (fields: {list(sample.keys())})" + ) + if not isinstance(data, np.ndarray): + raise TypeError(f"ConnectedComponents expects an np.ndarray mask, got {type(data).__name__}") + if data.ndim != 2: + raise ValueError(f"ConnectedComponents expects a 2-D mask; got shape {data.shape}") + return data + + def __call__(self, sample: TypedSample) -> TypedSample: + mask = self._find_mask(sample) + bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) + out = sample.replace_field(self.output, Regions(boxes=list(bboxes))) + return out.set_role(self.output, "aux") diff --git a/tests/test_typed_generic_ops.py b/tests/test_typed_generic_ops.py new file mode 100644 index 0000000..d63aa50 --- /dev/null +++ b/tests/test_typed_generic_ops.py @@ -0,0 +1,253 @@ +"""Typed-bag TWINS of the generic array→Image→Mask→Regions ops. + +Pins the three native typed transforms that let a ``TypedSample`` pipeline run the +detection/segmentation front-end without the legacy ``Sample`` path: + +* :class:`sampleflux.ops.image.ConvertToImage` — array-bearing field → ``Image`` item; +* :class:`sampleflux.ops.numpy.Threshold` — array field → boolean ``Mask`` item; +* :class:`sampleflux.ops.numpy.ConnectedComponents` — ``Mask`` → ``Regions`` item. + +Each twin REUSES its legacy op's math, so the twin's output is pinned to be byte-identical +to a legacy run on the equivalent ``Sample`` (parity). sampleflux-only — no waivefront import. +""" + +import numpy as np +import pytest +from confluid.registry import get_registry, resolve_class + +from sampleflux import Image, Mask, Regions, TypedSample +from sampleflux.ops.image import ConvertToImage, ConvertToImageOp +from sampleflux.ops.numpy import ConnectedComponents, ConnectedComponentsOp, Threshold, ThresholdOp +from sampleflux.sample import Sample + + +def _ramp_2d() -> np.ndarray: + return np.arange(8 * 10).reshape(8, 10).astype(np.float32) + + +def _blob_mask() -> np.ndarray: + m = np.zeros((6, 6), dtype=bool) + m[0:2, 0:2] = True # blob A (area 4) -> (0, 1, 0, 1) + m[4:6, 4:6] = True # blob B (area 4) -> (4, 5, 4, 5) + return m + + +# --------------------------------------------------------------------------- # +# ConvertToImage +# --------------------------------------------------------------------------- # +class TestConvertToImage: + def test_produces_image_item_shape_dtype_role(self) -> None: + out = ConvertToImage(colormap="gray")(TypedSample({"spec": Mask(_ramp_2d())})) + assert "image" in out + img = out["image"] + assert isinstance(img, Image) + assert np.asarray(img).shape == (8, 10, 3) + assert np.asarray(img).dtype == np.uint8 + assert img.layout == "HWC" + assert out.role_of("image") == "input" + # source field untouched + assert isinstance(out["spec"], Mask) + + def test_parity_with_legacy_default_sizing(self) -> None: + arr = _ramp_2d() + typed = ConvertToImage(colormap="viridis")(TypedSample({"spec": Mask(arr)})) + legacy = ConvertToImageOp(colormap="viridis")(Sample(input=arr, target=None, metadata={})) + assert np.array_equal(np.array(legacy.input), np.asarray(typed["image"])) + + def test_parity_with_legacy_exact_resize_and_flip(self) -> None: + arr = _ramp_2d() + typed = ConvertToImage(colormap="gray", width=20, height=16, flip_vertical=True)( + TypedSample({"spec": Mask(arr)}) + ) + legacy = ConvertToImageOp(colormap="gray", width=20, height=16, flip_vertical=True)( + Sample(input=arr, target=None, metadata={}) + ) + assert np.asarray(typed["image"]).shape == (16, 20, 3) + assert np.array_equal(np.array(legacy.input), np.asarray(typed["image"])) + + def test_explicit_field_and_custom_output(self) -> None: + s = TypedSample({"a": Mask(_ramp_2d()), "b": Mask(np.zeros((4, 4), dtype=np.float32))}) + out = ConvertToImage(field="b", output="preview")(s) + assert np.asarray(out["preview"]).shape == (4, 4, 3) + + def test_does_not_publish_image_dims_metadata(self) -> None: + # There is no shared metadata dict in the typed model; the Image SHAPE carries the dims. + out = ConvertToImage()(TypedSample({"spec": Mask(_ramp_2d())})) + assert set(out.keys()) == {"spec", "image"} # no image_width_px / image_height_px field + assert np.asarray(out["image"]).shape[:2] == (8, 10) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + ConvertToImage(field="nope")(TypedSample({"spec": Mask(_ramp_2d())})) + + def test_no_array_field_raises(self) -> None: + with pytest.raises(ValueError, match="no array-bearing field"): + ConvertToImage()(TypedSample({"lbl": Regions(boxes=[[0, 0, 1, 1]])})) + + +# --------------------------------------------------------------------------- # +# Threshold +# --------------------------------------------------------------------------- # +class TestThreshold: + def test_produces_mask_parity_role(self) -> None: + arr = _ramp_2d() + typed = Threshold(low_level=20.0)(TypedSample({"spec": Mask(arr)})) + assert isinstance(typed["mask"], Mask) + assert np.asarray(typed["mask"]).dtype == np.bool_ + assert typed.role_of("mask") == "aux" + legacy = ThresholdOp(low_level=20.0)(Sample(input=arr, target=None, metadata={})).input + assert np.array_equal(np.asarray(typed["mask"]), legacy) + + def test_string_literal_bound(self) -> None: + arr = _ramp_2d() + typed = Threshold(low_level="20")(TypedSample({"spec": Mask(arr)})) + legacy = ThresholdOp(low_level="20")(Sample(input=arr, target=None, metadata={})).input + assert np.array_equal(np.asarray(typed["mask"]), legacy) + + def test_env_var_expression_bound(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TEST_THRESH_LEVEL", "20") + arr = _ramp_2d() + typed = Threshold(low_level="$TEST_THRESH_LEVEL")(TypedSample({"spec": Mask(arr)})) + assert np.array_equal(np.asarray(typed["mask"]), arr > 20.0) + + def test_meta_key_expression_has_no_typed_source(self) -> None: + # {key} expressions have no typed metadata home -> loud KeyError (documented). + with pytest.raises(KeyError): + Threshold(low_level="{some_key}")(TypedSample({"spec": Mask(_ramp_2d())})) + + def test_band_pass_both_bounds_and_ops(self) -> None: + arr = _ramp_2d() + typed = Threshold(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")(TypedSample({"spec": Mask(arr)})) + legacy = ThresholdOp(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")( + Sample(input=arr, target=None, metadata={}) + ).input + assert np.array_equal(np.asarray(typed["mask"]), legacy) + assert np.array_equal(np.asarray(typed["mask"]), (arr >= 20.0) & (arr <= 60.0)) + + def test_no_bound_raises(self) -> None: + with pytest.raises(ValueError, match="at least one"): + Threshold()(TypedSample({"spec": Mask(_ramp_2d())})) + + def test_default_field_picks_first_array(self) -> None: + # No explicit field: first array-bearing item (insertion order). + s = TypedSample({"raw": Mask(_ramp_2d()), "other": Regions(boxes=[])}) + out = Threshold(low_level=20.0)(s) + assert np.array_equal(np.asarray(out["mask"]), _ramp_2d() > 20.0) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + Threshold(low_level=1.0, field="nope")(TypedSample({"spec": Mask(_ramp_2d())})) + + def test_non_array_field_raises(self) -> None: + with pytest.raises(TypeError, match="expected an array"): + Threshold(low_level=1.0, field="reg")(TypedSample({"reg": Regions(boxes=[])})) + + def test_no_array_field_default_raises(self) -> None: + with pytest.raises(ValueError, match="no array-bearing field"): + Threshold(low_level=1.0)(TypedSample({"reg": Regions(boxes=[])})) + + +# --------------------------------------------------------------------------- # +# ConnectedComponents +# --------------------------------------------------------------------------- # +class TestConnectedComponents: + def test_produces_regions_bin_box_contract_and_role(self) -> None: + out = ConnectedComponents()(TypedSample({"m": Mask(_blob_mask())})) + regions = out["boxes"] + assert isinstance(regions, Regions) + assert out.role_of("boxes") == "aux" + # The pinned generic contract: (row_min, row_max, col_min, col_max) inclusive tuples. + assert regions.boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] + + def test_parity_with_legacy(self) -> None: + mask = _blob_mask() + typed = ConnectedComponents()(TypedSample({"m": Mask(mask)})) + legacy = ConnectedComponentsOp()(Sample(input=mask, target=None, metadata={})).input + assert typed["boxes"].boxes == legacy + + def test_min_area_bins_filters_small_blobs(self) -> None: + m = np.zeros((6, 6), dtype=bool) + m[0:2, 0:2] = True # area 4 + m[5, 5] = True # area 1 -> dropped when min_area_bins=2 + out = ConnectedComponents(min_area_bins=2)(TypedSample({"m": Mask(m)})) + assert out["boxes"].boxes == [(0, 1, 0, 1)] + + def test_connectivity_parity(self) -> None: + # Diagonal touch: 4-connectivity keeps two blobs, 8 merges them. + m = np.zeros((4, 4), dtype=bool) + m[0, 0] = True + m[1, 1] = True + four = ConnectedComponents(connectivity=4)(TypedSample({"m": Mask(m)})) + eight = ConnectedComponents(connectivity=8)(TypedSample({"m": Mask(m)})) + assert len(four["boxes"].boxes) == 2 + assert len(eight["boxes"].boxes) == 1 + + def test_default_prefers_mask_over_other_array(self) -> None: + # An Image is inserted first, but a Mask is preferred by the default resolver. + s = TypedSample({"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())}) + out = ConnectedComponents()(s) + assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] + + def test_falls_back_to_first_array_when_no_mask(self) -> None: + # No Mask item — a 2-D array item is used. + out = ConnectedComponents()(TypedSample({"m": Image(_blob_mask())})) + assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] + + def test_non_2d_mask_raises(self) -> None: + with pytest.raises(ValueError, match="2-D mask"): + ConnectedComponents()(TypedSample({"m": Mask(np.zeros((2, 2, 2), dtype=bool))})) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + ConnectedComponents(field="nope")(TypedSample({"m": Mask(_blob_mask())})) + + def test_no_mask_or_array_raises(self) -> None: + with pytest.raises(ValueError, match="no Mask or array-bearing field"): + ConnectedComponents()(TypedSample({"reg": Regions(boxes=[])})) + + +# --------------------------------------------------------------------------- # +# End-to-end chain: array -> Image -> Mask -> Regions, all typed, sampleflux-only. +# --------------------------------------------------------------------------- # +def test_array_to_image_to_mask_to_regions_chain() -> None: + arr = _ramp_2d() + sample = TypedSample({"spec": Mask(arr)}) + out = ConnectedComponents(field="mask")(Threshold(field="spec", low_level=20.0)(ConvertToImage()(sample))) + # Every stage produced its typed field. + assert isinstance(out["image"], Image) + assert isinstance(out["mask"], Mask) + assert isinstance(out["boxes"], Regions) + # Regions carries (row_min, row_max, col_min, col_max) bin-box tuples. + assert out["boxes"].boxes + for box in out["boxes"].boxes: + assert len(box) == 4 + row_min, row_max, col_min, col_max = box + assert row_min <= row_max and col_min <= col_max + # The image field carries the pixel dims via its shape (no separate metadata). + assert np.asarray(out["image"]).shape[:2] == arr.shape + + +# --------------------------------------------------------------------------- # +# Discovery + zero-arg construction. +# --------------------------------------------------------------------------- # +def test_zero_arg_constructible() -> None: + assert ConvertToImage().output == "image" + assert Threshold().output == "mask" + assert ConnectedComponents().output == "boxes" + + +@pytest.mark.parametrize( + ("name", "cls", "group"), + [ + ("ConvertToImage", ConvertToImage, "image"), + ("Threshold", Threshold, "numpy"), + ("ConnectedComponents", ConnectedComponents, "numpy"), + ], +) +def test_discovery_tags(name: str, cls: type, group: str) -> None: + assert cls.__confluid_category__ == "op" # type: ignore[attr-defined] + assert cls.__confluid_group__ == group # type: ignore[attr-defined] + assert resolve_class(name) is cls + registry = get_registry() + assert name in registry.list_classes(category="op") + assert name in registry.list_classes(group=group) From 15fec48848df12965cd6d5750ac42ffd873a3e65 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 22 Jul 2026 21:20:05 +0200 Subject: [PATCH 029/102] =?UTF-8?q?feat(sampleflux):=20typed=20target/tens?= =?UTF-8?q?or=20ops=20=E2=80=94=20ToTensor=20+=20Encode/Decode/MetadataToT?= =?UTF-8?q?arget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive typed-bag twins for the classification input/target op path (legacy *Op untouched). 1199 passed (+39), mypy clean (109 files), flake8 clean. - ops/torch.py: ToTensor(Transform) — array item -> CHW float32 Image(layout="CHW") payload byte-identical to ToTensorOp; numpy->tensor happens at the collate/model boundary (a live-Tensor item is the flagged Tensor-subclass follow-up). - ops/target.py: EncodeTarget/DecodeTarget (delegate to the legacy pinned-map ops for byte-parity) + MetadataToTarget (largely redundant in the typed model — the source emits a Label target directly — kept for parity/config-compat). - docs/architecture.md: decision record; tests/test_typed_target_ops.py (39 tests). --- docs/architecture.md | 76 ++++++++++ sampleflux/ops/target.py | 209 ++++++++++++++++++++++++++ sampleflux/ops/torch.py | 78 ++++++++++ tests/test_typed_target_ops.py | 267 +++++++++++++++++++++++++++++++++ 4 files changed, 630 insertions(+) create mode 100644 tests/test_typed_target_ops.py diff --git a/docs/architecture.md b/docs/architecture.md index a5b8a4a..91ddcc3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -531,3 +531,79 @@ sample["boxes"].boxes # [(row_min, row_max, col_min, col_max), ...] — the pin - **Do not modify the legacy ops or reimplement their math in a twin** — a twin reuses the legacy math so parity is guaranteed; the twins are additive and the legacy `Sample`-path consumers must keep working. + +## A typed field cannot hold a live torch tensor — `ToTensor` stores CHW-float numpy (`ToTensor`/`EncodeTarget`/`DecodeTarget`/`MetadataToTarget`, 2026-07-22) + +### Context + +The typed detection twins above reach `Regions`; a typed CLASSIFICATION front-end needs the other +two shapes: turn the working image into the model's **input tensor**, and turn the class-name label +into the encoded **target id**. The legacy ops that do this (`ToTensorOp`, `MetadataToTargetOp`, +`EncodeTargetOp` / `DecodeTargetOp`) operate on the `Sample(input, target, metadata)` triple. Two +facts of the typed model shape the twins: (1) there is NO shared metadata dict — the label already +rides a `Label` field that owns its metadata; (2) an array item is an `np.ndarray` SUBCLASS whose +`__new__` runs `np.asarray(data)`, so **a field payload is coerced to numpy** — an `Image` cannot +hold a live `torch.Tensor` (verified: `item_data(Image(tensor))` is an `ndarray`), and a bare tensor +stored directly as a field value has no registered item type, so `typed_collate` / the storage codec +(`bag.io.encode_item`) cannot serialize it. + +### Decision + +Add native typed twins subclassing `Transform` and overriding `__call__` (the same shape as the +detection twins), each reusing its legacy op VERBATIM on a shim `Sample` for byte-parity: + +- **`ToTensor`** (`ops/torch.py`, `group="torch"`) resolves an array-bearing field (explicit `field` + or the first array/PIL item), runs `ToTensorOp` (HWC→CHW + `normalize`), and writes an `Image` + with `layout="CHW"`. Because `NDArrayItem` coerces the payload, the stored value is a CHW `float32` + **numpy** array whose values equal `ToTensorOp(...).input.numpy()` — NOT a live tensor. By default + it REPLACES the source field in place so the field's `input` role is preserved (`output` writes a + new field tagged `input` instead). `typed_collate` stacks these payloads with `np.stack`; the + numpy→tensor conversion is the collate / model boundary's job, exactly as for any numpy dataset. A + Tensor-subclass item that would let a field carry a live tensor is the documented follow-up + (`bag/items.py` PoC note + root TASKS.md). +- **`EncodeTarget` / `DecodeTarget`** (`ops/target.py`, `group="structure"`) resolve a `Label` field, + map its `.value` through the config-pinned `mapping` by delegating to `EncodeTargetOp` / + `DecodeTargetOp` (so the non-empty-mapping validation AND the shared `_lookup` are byte-identical), + and write a new `Label` (carrying the source label's `classes`) tagged `target`. In place by + default (`output` blank). +- **`MetadataToTarget`** is provided for PARITY / config-compat but is largely REDUNDANT in the typed + model: a source emits the label directly as a `Label` field already tagged `target`, so no + metadata→target move is needed. The twin reads a field's natural value (a `Label`'s `.value`, else + its array payload) or a named attribute (`key=`) and writes a target `Label` — the escape hatch for + a label that rode as another item's attribute. + +### Consequences + +- A `TypedSample` carrying an HWC `Image` (role input) + a name `Label` (role target) runs + `ToTensor → EncodeTarget` into a CHW-float input field + an int-id target field, with no legacy + `Sample` anywhere — the typed classification front-end is unblocked. +- The model-input payload is CHW-float **numpy**, not a live `torch.Tensor`; a consumer / trainer + tensorizes at the collate or forward boundary. This is a deliberate PoC limitation, not a bug — + it disappears when the Tensor-subclass item lands. +- The twins carry `category="op"` + the legacy `group`, so they are discoverable like the legacy ops + (their modules — `sampleflux-ops-torch` / `sampleflux-ops-target` — are already entry-pointed; a + class added to a registered module needs no new entry point). + +### Example + +```python +from sampleflux import TypedSample, Image, Label +from sampleflux.ops.torch import ToTensor +from sampleflux.ops.target import EncodeTarget + +sample = TypedSample( + {"image": Image(hwc_uint8), "class": Label("cat")}, + roles={"image": "input", "class": "target"}, +) +sample = ToTensor(field="image")(sample) # image -> CHW float32 Image (role input, in place) +sample = EncodeTarget(mapping={"cat": 0, "dog": 1}, field="class")(sample) # class -> Label(0) (role target) +``` + +### What you may change (and where it's documented) + +- **The Tensor-subclass item follow-up** — once a field can carry a live tensor, `ToTensor` should + store it directly; update this record and the `bag/items.py` PoC note together. +- **`ToTensor`'s replace-in-place default vs a new output field** — keep role preservation (in place) + as the default; a new `output` field is tagged `input`. +- **Do not modify the legacy ops or reimplement their math in a twin** — the twins delegate to the + legacy ops for byte-parity and are strictly additive. diff --git a/sampleflux/ops/target.py b/sampleflux/ops/target.py index 201ef9e..56505cd 100644 --- a/sampleflux/ops/target.py +++ b/sampleflux/ops/target.py @@ -26,6 +26,9 @@ from confluid import configurable +from sampleflux.bag.items import Label, item_data +from sampleflux.bag.sample import TypedSample +from sampleflux.bag.transform import Transform from sampleflux.sample import Sample #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo @@ -300,10 +303,216 @@ def __call__(self, sample: Sample) -> Sample: return sample._replace(target={"boxes": boxes_t, "labels": labels_t}) +@configurable(category="op", group="structure") +class MetadataToTarget(Transform): + """Typed twin of :class:`MetadataToTargetOp` — promote a field / attr value into a target ``Label``. + + The typed-bag counterpart of :class:`MetadataToTargetOp`. The legacy op copies + ``metadata[key]`` onto ``sample.target``, but the typed model has NO shared metadata dict — every + item OWNS its metadata, and the supervised label already rides a :class:`~sampleflux.Label` field. + So this twin reads a value from a SOURCE field (``field``; blank picks the first ``Label``, else + the first field) — either the field's natural value (a ``Label``'s ``.value``, otherwise the + item's array payload) or, when ``key`` is set, the named ATTRIBUTE of the source item — and writes + a fresh :class:`~sampleflux.Label` under ``output`` tagged ``target``. + + REDUNDANCY. In a typical typed classification pipeline the source emits the label directly as a + ``Label`` field already tagged ``target``, so this op is usually a NO-OP-ish re-home and is NOT + needed. It is provided for parity / config-compat with the legacy ``metadata → target`` step and + for the case where a label rode as another item's attribute (``key=``) and must become a + dedicated target ``Label``. + + Args: + field: Source field to read; blank (default) picks the first ``Label`` field, else the first field. + key: Optional attribute name to read off the source item (e.g. a carried label attr); blank + (default) reads the item's natural value (a ``Label``'s ``.value``, else its array payload). + output: Field the target ``Label`` is written to (added if new); its role is set to ``target``. + """ + + handles = (Label,) + consumes = (Label,) + produces = (Label,) + + def __init__(self, field: str = "", key: str = "", output: str = "target") -> None: + super().__init__() + self.field = str(field) + self.key = str(key) + self.output = str(output) + + def _find_source(self, sample: TypedSample) -> str: + """Resolve the KEY of the source field (``self.field``, else first ``Label``, else first field).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError( + f"MetadataToTarget: field {self.field!r} not in sample (fields: {list(sample.keys())})" + ) + return self.field + for key, _item in sample.items_of_type(Label): + return key + for key in sample.keys(): + return key + raise ValueError("MetadataToTarget: sample is empty — no source field to read") + + def __call__(self, sample: TypedSample) -> TypedSample: + key = self._find_source(sample) + item = sample[key] + if self.key: + if not hasattr(item, self.key): + raise AttributeError( + f"MetadataToTarget: field {key!r} ({type(item).__name__}) has no attribute {self.key!r}" + ) + value = getattr(item, self.key) + elif isinstance(item, Label): + value = item.value + else: + value = item_data(item) + out = sample.replace_field(self.output, Label(value)) + return out.set_role(self.output, "target") + + +@configurable(category="op", group="structure") +class EncodeTarget(Transform): + """Typed twin of :class:`EncodeTargetOp` — a class-NAME ``Label`` → a class-ID ``Label`` (role ``target``). + + The typed-bag counterpart of :class:`EncodeTargetOp`: it reads a :class:`~sampleflux.Label` field + (``field``; blank picks the first ``Label``) whose ``.value`` is a raw class name and maps it to + its class id through the config-pinned ``mapping`` — the declarative ``LabelEncoder`` analogue. + This twin REUSES the legacy ``EncodeTargetOp`` verbatim (its non-empty-mapping validation AND its + shared ``_lookup`` logic), so the encoded value is byte-identical. The result is a new + :class:`~sampleflux.Label` (carrying the source label's ``classes`` vocabulary) written under + ``output`` — blank (default) replaces the source field in place — tagged ``target``. + + Pinning the mapping (rather than fitting it) keeps train / eval / predict on one identical + label→id ordering. The non-empty-mapping requirement is validated LAZILY when the op runs (the + zero-arg default stays constructible per the lazy-init convention). + + Args: + mapping: Lookup from raw label name → class id, e.g. ``{"DJI AVATA2": 2, ...}``. Must be non-empty. + ignore_unknown: When ``False`` (default), raise on a label missing from ``mapping``; when + ``True``, substitute ``default``. + default: Value written for an unknown label when ``ignore_unknown=True`` (default ``0``). + field: ``Label`` field to encode; blank (default) picks the first ``Label`` field. + output: Field the encoded ``Label`` is written to; blank (default) replaces the source field + in place. Its role is set to ``target``. + """ + + handles = (Label,) + consumes = (Label,) + produces = (Label,) + + def __init__( + self, + mapping: Optional[Dict[Any, Any]] = None, + ignore_unknown: bool = False, + default: Any = 0, + field: str = "", + output: str = "", + ) -> None: + super().__init__() + # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + self.mapping = dict(mapping) if mapping else {} + self.ignore_unknown = bool(ignore_unknown) + self.default = default + self.field = str(field) + self.output = str(output) + + def _find_label(self, sample: TypedSample) -> str: + """Resolve the KEY of the ``Label`` field to encode (``self.field`` or the first ``Label``).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError(f"EncodeTarget: field {self.field!r} not in sample (fields: {list(sample.keys())})") + item = sample[self.field] + if not isinstance(item, Label): + raise TypeError(f"EncodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label") + return self.field + for key, _item in sample.items_of_type(Label): + return key + raise ValueError(f"EncodeTarget: no Label field in sample (fields: {list(sample.keys())})") + + def __call__(self, sample: TypedSample) -> TypedSample: + key = self._find_label(sample) + label = sample[key] + # Reuse the legacy op VERBATIM (non-empty validation + shared _lookup) for byte-parity. + encoded = EncodeTargetOp(self.mapping, self.ignore_unknown, self.default)( + Sample(input=None, target=label.value, metadata={}) + ).target + out_key = self.output or key + out = sample.replace_field(out_key, Label(encoded, classes=label.classes)) + return out.set_role(out_key, "target") + + +@configurable(category="op", group="structure") +class DecodeTarget(Transform): + """Typed twin of :class:`DecodeTargetOp` — a class-ID ``Label`` → a class-NAME ``Label`` (inverse of encode). + + The typed-bag counterpart of :class:`DecodeTargetOp`: it reads a :class:`~sampleflux.Label` field + (``field``; blank picks the first ``Label``) whose ``.value`` is an encoded class id and maps it + back to its label name through ``mapping`` — the readback half used in prediction / reporting. + This twin REUSES the legacy ``DecodeTargetOp`` verbatim, so the decoded value is byte-identical. + The result is a new :class:`~sampleflux.Label` (carrying the source label's ``classes``) written + under ``output`` — blank (default) replaces the source field in place — tagged ``target``. + + Args: + mapping: Lookup from class id → label name, e.g. ``{2: "DJI AVATA2", ...}``. Must be non-empty. + ignore_unknown: When ``False`` (default), raise on an id missing from ``mapping``; when + ``True``, substitute ``default``. + default: Value written for an unknown id when ``ignore_unknown=True`` (default ``None``). + field: ``Label`` field to decode; blank (default) picks the first ``Label`` field. + output: Field the decoded ``Label`` is written to; blank (default) replaces the source field + in place. Its role is set to ``target``. + """ + + handles = (Label,) + consumes = (Label,) + produces = (Label,) + + def __init__( + self, + mapping: Optional[Dict[Any, Any]] = None, + ignore_unknown: bool = False, + default: Any = None, + field: str = "", + output: str = "", + ) -> None: + super().__init__() + # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + self.mapping = dict(mapping) if mapping else {} + self.ignore_unknown = bool(ignore_unknown) + self.default = default + self.field = str(field) + self.output = str(output) + + def _find_label(self, sample: TypedSample) -> str: + """Resolve the KEY of the ``Label`` field to decode (``self.field`` or the first ``Label``).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError(f"DecodeTarget: field {self.field!r} not in sample (fields: {list(sample.keys())})") + item = sample[self.field] + if not isinstance(item, Label): + raise TypeError(f"DecodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label") + return self.field + for key, _item in sample.items_of_type(Label): + return key + raise ValueError(f"DecodeTarget: no Label field in sample (fields: {list(sample.keys())})") + + def __call__(self, sample: TypedSample) -> TypedSample: + key = self._find_label(sample) + label = sample[key] + # Reuse the legacy op VERBATIM (non-empty validation + shared _lookup) for byte-parity. + decoded = DecodeTargetOp(self.mapping, self.ignore_unknown, self.default)( + Sample(input=None, target=label.value, metadata={}) + ).target + out_key = self.output or key + out = sample.replace_field(out_key, Label(decoded, classes=label.classes)) + return out.set_role(out_key, "target") + + __all__ = [ "MetadataToTargetOp", "EncodeTargetOp", "DecodeTargetOp", "CocoToTorchVisionDetectionOp", "MasksToDetectionBoxesOp", + "MetadataToTarget", + "EncodeTarget", + "DecodeTarget", ] diff --git a/sampleflux/ops/torch.py b/sampleflux/ops/torch.py index e9e6e99..e059c5e 100644 --- a/sampleflux/ops/torch.py +++ b/sampleflux/ops/torch.py @@ -4,6 +4,10 @@ import torch from confluid import configurable +from sampleflux.bag.items import Image as ImageItem +from sampleflux.bag.items import NDArrayItem, item_data +from sampleflux.bag.sample import TypedSample +from sampleflux.bag.transform import Transform from sampleflux.sample import Sample from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType @@ -209,3 +213,77 @@ def __call__(self, sample: Sample) -> Sample: tensor = (tensor - mean_t) / std_t return sample._replace(input=tensor) + + +@configurable(category="op", group="torch") +class ToTensor(Transform): + """Typed twin of :class:`ToTensorOp` — an array-bearing field → a CHW-float ``Image`` item. + + The typed-bag counterpart of :class:`ToTensorOp`: it reads the payload of an array-bearing + field (blank ``field`` picks the first array/PIL-bearing item — typically the + :class:`~sampleflux.Image` a :class:`~sampleflux.ops.image.ConvertToImage` produced), runs the + SAME HWC→CHW transpose + ``normalize`` conversion (this twin REUSES the legacy op verbatim on a + shim ``Sample``, so the numbers are identical), and writes a CHW-layout :class:`~sampleflux.Image` + back. By default it REPLACES the resolved field in place (``output`` blank), so the field's role + is preserved — the model's working image tensor stays the ``input`` it already was; set + ``output`` to write a NEW field (tagged ``input``) instead. Any other field passes through + untouched. + + IMPORTANT — payload dtype. A :class:`~sampleflux.NDArrayItem` (which ``Image`` is) coerces its + payload through ``np.asarray`` on construction, so it CANNOT hold a live ``torch.Tensor``: the + stored payload is a CHW ``float32`` **numpy** array whose values are byte-identical to the legacy + ``ToTensorOp`` tensor (``legacy.input.numpy()``). The typed collate (``typed_collate``) stacks + these field payloads with ``np.stack`` into a batched CHW-float array; the numpy→``torch.Tensor`` + conversion happens at the collate / model boundary (exactly as for any numpy-backed dataset). A + torch-``Tensor``-subclass item that would let a field carry a live tensor is the documented + follow-up (see ``sampleflux.bag.items`` — "torch payloads ride in wrapper items in the PoC"). + + Args: + normalize: When ``True`` (default), scale integer pixel inputs into the ``[0, 1]`` float range. + mode: Optional PIL mode to convert a PIL payload to (e.g. ``"RGB"`` forces 3 channels); ``None`` = as-is. + field: Name of the source field to tensorize; blank (default) picks the first array/PIL-bearing item. + output: Field the CHW ``Image`` is written to; blank (default) replaces the source field in place + (role preserved). A non-blank name writes a new field tagged ``input``. + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (ImageItem,) + + def __init__( + self, + normalize: bool = True, + mode: Optional[str] = None, + field: str = "", + output: str = "", + ) -> None: + super().__init__() + self.normalize = bool(normalize) + self.mode = mode + self.field = field + self.output = output + + def _find_field(self, sample: TypedSample) -> str: + """Resolve the KEY of the field to tensorize (``self.field`` or the first array/PIL item).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError(f"ToTensor: field {self.field!r} not in sample (fields: {list(sample.keys())})") + return self.field + for key, item in sample.items(): + data = item_data(item) + if isinstance(data, np.ndarray) or hasattr(data, "convert"): + return key + raise ValueError(f"ToTensor: no array-bearing field in sample (fields: {list(sample.keys())})") + + def __call__(self, sample: TypedSample) -> TypedSample: + key = self._find_field(sample) + data = item_data(sample[key]) + # Reuse the legacy op's conversion VERBATIM on a shim Sample so the CHW / normalization + # values are identical; NDArrayItem then coerces the tensor to a CHW float32 numpy payload. + tensor = ToTensorOp(self.normalize, self.mode)(Sample(input=data, target=None, metadata={})).input + arr = tensor.detach().cpu().numpy() + out_key = self.output or key + out = sample.replace_field(out_key, ImageItem(arr, layout="CHW")) + if self.output: + out = out.set_role(out_key, "input") + return out diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py new file mode 100644 index 0000000..30006b7 --- /dev/null +++ b/tests/test_typed_target_ops.py @@ -0,0 +1,267 @@ +"""Typed-bag TWINS of the tensorization + target-shaping ops. + +Pins the native typed transforms that let a ``TypedSample`` classification pipeline build its +model INPUT tensor and its encoded TARGET ``Label`` without the legacy ``Sample`` path: + +* :class:`sampleflux.ops.torch.ToTensor` — array-bearing field → CHW-float ``Image`` item; +* :class:`sampleflux.ops.target.MetadataToTarget` — a field / attr value → a target ``Label``; +* :class:`sampleflux.ops.target.EncodeTarget` / ``DecodeTarget`` — class-name ↔ class-id ``Label``. + +Each twin REUSES its legacy op's math, so the twin's output is pinned byte-identical to a legacy +run on the equivalent ``Sample`` (parity). sampleflux-only — no waivefront import. +""" + +import numpy as np +import pytest +import torch +from confluid.registry import get_registry, resolve_class + +from sampleflux import Image, Label, Mask, TypedSample +from sampleflux.collate import typed_collate +from sampleflux.ops.image import ConvertToImage +from sampleflux.ops.target import DecodeTarget, DecodeTargetOp, EncodeTarget, EncodeTargetOp, MetadataToTarget +from sampleflux.ops.torch import ToTensor, ToTensorOp +from sampleflux.sample import Sample + +_MAP = {"cat": 0, "dog": 1, "fox": 2} +_INV = {0: "cat", 1: "dog", 2: "fox"} + + +def _hwc_uint8() -> np.ndarray: + return (np.arange(4 * 5 * 3).reshape(4, 5, 3) % 256).astype(np.uint8) + + +# --------------------------------------------------------------------------- # +# ToTensor +# --------------------------------------------------------------------------- # +class TestToTensor: + def test_produces_chw_float_image_role_preserved(self) -> None: + arr = _hwc_uint8() + out = ToTensor()(TypedSample({"image": Image(arr)}, roles={"image": "input"})) + img = out["image"] + assert isinstance(img, Image) + assert img.layout == "CHW" + payload = np.asarray(img) + assert payload.shape == (3, 4, 5) # HWC -> CHW + assert payload.dtype == np.float32 + assert payload.max() <= 1.0 # normalized + assert out.role_of("image") == "input" # replaced in place -> role preserved + + def test_parity_with_legacy_tensor(self) -> None: + arr = _hwc_uint8() + typed = ToTensor()(TypedSample({"image": Image(arr)})) + legacy = ToTensorOp()(Sample(input=arr, target=None, metadata={})).input + assert isinstance(legacy, torch.Tensor) + assert np.array_equal(np.asarray(typed["image"]), legacy.numpy()) + + def test_parity_no_normalize(self) -> None: + arr = _hwc_uint8() + typed = ToTensor(normalize=False)(TypedSample({"image": Image(arr)})) + legacy = ToTensorOp(normalize=False)(Sample(input=arr, target=None, metadata={})).input + assert np.array_equal(np.asarray(typed["image"]), legacy.numpy()) + + def test_payload_is_numpy_not_live_tensor(self) -> None: + # NDArrayItem coerces its payload via np.asarray, so an Image CANNOT hold a live tensor; + # the stored CHW-float payload is a numpy array (values identical to the legacy tensor). + from sampleflux.bag.items import item_data + + out = ToTensor()(TypedSample({"image": Image(_hwc_uint8())})) + assert isinstance(item_data(out["image"]), np.ndarray) + + def test_new_output_field_tagged_input(self) -> None: + arr = _hwc_uint8() + out = ToTensor(output="tensor")(TypedSample({"image": Image(arr)}, roles={"image": "input"})) + assert np.asarray(out["tensor"]).shape == (3, 4, 5) + assert out.role_of("tensor") == "input" + # original field left as-is (HWC uint8) + assert np.asarray(out["image"]).shape == (4, 5, 3) + + def test_explicit_field(self) -> None: + s = TypedSample({"a": Mask(np.zeros((2, 2), dtype=np.uint8)), "b": Image(_hwc_uint8())}) + out = ToTensor(field="b")(s) + assert np.asarray(out["b"]).shape == (3, 4, 5) + + def test_default_picks_first_array_field(self) -> None: + s = TypedSample({"lbl": Label("cat"), "image": Image(_hwc_uint8())}) + out = ToTensor()(s) + assert np.asarray(out["image"]).shape == (3, 4, 5) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + ToTensor(field="nope")(TypedSample({"image": Image(_hwc_uint8())})) + + def test_no_array_field_raises(self) -> None: + with pytest.raises(ValueError, match="no array-bearing field"): + ToTensor()(TypedSample({"lbl": Label("cat")})) + + def test_typed_collate_stacks_payloads(self) -> None: + # The typed collate stacks the CHW-float Image payloads into a batched array. + a = ToTensor()(TypedSample({"image": Image(_hwc_uint8())})) + b = ToTensor()(TypedSample({"image": Image(_hwc_uint8())})) + batch = typed_collate([a, b]) + assert np.asarray(batch["image"]).shape == (2, 3, 4, 5) + + +# --------------------------------------------------------------------------- # +# MetadataToTarget +# --------------------------------------------------------------------------- # +class TestMetadataToTarget: + def test_promotes_label_value_to_target(self) -> None: + s = TypedSample({"class": Label("cat")}, roles={"class": "aux"}) + out = MetadataToTarget(field="class", output="target")(s) + assert isinstance(out["target"], Label) + assert out["target"].value == "cat" + assert out.role_of("target") == "target" + + def test_default_picks_first_label(self) -> None: + s = TypedSample({"image": Image(_hwc_uint8()), "y": Label("dog")}) + out = MetadataToTarget()(s) + assert out["target"].value == "dog" + assert out.role_of("target") == "target" + + def test_read_named_attribute(self) -> None: + # Read a carried attribute off a field (a value that rode as item-scoped metadata). + s = TypedSample({"y": Label("cat", classes=["cat", "dog"])}) + out = MetadataToTarget(field="y", key="classes", output="vocab")(s) + assert out["vocab"].value == ["cat", "dog"] + + def test_missing_attribute_raises(self) -> None: + with pytest.raises(AttributeError, match="no attribute 'nope'"): + MetadataToTarget(field="y", key="nope")(TypedSample({"y": Label("cat")})) + + def test_missing_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + MetadataToTarget(field="nope")(TypedSample({"y": Label("cat")})) + + def test_empty_sample_raises(self) -> None: + with pytest.raises(ValueError, match="sample is empty"): + MetadataToTarget()(TypedSample({})) + + +# --------------------------------------------------------------------------- # +# EncodeTarget / DecodeTarget +# --------------------------------------------------------------------------- # +class TestEncodeDecodeTarget: + def test_encode_name_to_id_role_target(self) -> None: + out = EncodeTarget(mapping=_MAP)(TypedSample({"y": Label("cat")}, roles={"y": "target"})) + assert isinstance(out["y"], Label) + assert out["y"].value == 0 + assert out.role_of("y") == "target" + + def test_encode_parity_with_legacy(self) -> None: + for name in _MAP: + typed = EncodeTarget(mapping=_MAP)(TypedSample({"y": Label(name)})) + legacy = EncodeTargetOp(mapping=_MAP)(Sample(input=None, target=name, metadata={})).target + assert typed["y"].value == legacy + + def test_encode_preserves_classes_vocab(self) -> None: + out = EncodeTarget(mapping=_MAP)(TypedSample({"y": Label("dog", classes=list(_MAP))})) + assert out["y"].value == 1 + assert out["y"].classes == list(_MAP) + + def test_encode_new_output_field(self) -> None: + out = EncodeTarget(mapping=_MAP, output="target_id")(TypedSample({"y": Label("fox")})) + assert out["target_id"].value == 2 + assert out.role_of("target_id") == "target" + assert out["y"].value == "fox" # source left intact + + def test_encode_ignore_unknown(self) -> None: + out = EncodeTarget(mapping=_MAP, ignore_unknown=True, default=-1)(TypedSample({"y": Label("bird")})) + assert out["y"].value == -1 + + def test_encode_unknown_raises(self) -> None: + with pytest.raises(KeyError): + EncodeTarget(mapping=_MAP)(TypedSample({"y": Label("bird")})) + + def test_encode_empty_mapping_raises_lazily(self) -> None: + op = EncodeTarget() # constructible with no mapping (lazy) + with pytest.raises(ValueError, match="at least one entry"): + op(TypedSample({"y": Label("cat")})) + + def test_decode_id_to_name_parity(self) -> None: + for cid in _INV: + typed = DecodeTarget(mapping=_INV)(TypedSample({"y": Label(cid)})) + legacy = DecodeTargetOp(mapping=_INV)(Sample(input=None, target=cid, metadata={})).target + assert typed["y"].value == legacy + + def test_encode_then_decode_round_trip(self) -> None: + s = TypedSample({"y": Label("dog")}) + encoded = EncodeTarget(mapping=_MAP)(s) + assert encoded["y"].value == 1 + decoded = DecodeTarget(mapping=_INV)(encoded) + assert decoded["y"].value == "dog" + + def test_decode_empty_mapping_raises_lazily(self) -> None: + with pytest.raises(ValueError, match="at least one entry"): + DecodeTarget()(TypedSample({"y": Label(0)})) + + def test_encode_non_label_field_raises(self) -> None: + with pytest.raises(TypeError, match="expected a Label"): + EncodeTarget(mapping=_MAP, field="image")(TypedSample({"image": Image(_hwc_uint8())})) + + def test_encode_no_label_field_raises(self) -> None: + with pytest.raises(ValueError, match="no Label field"): + EncodeTarget(mapping=_MAP)(TypedSample({"image": Image(_hwc_uint8())})) + + +# --------------------------------------------------------------------------- # +# End-to-end typed classification input/target path (sampleflux-only). +# --------------------------------------------------------------------------- # +def test_typed_classification_input_and_target_chain() -> None: + # Source-shaped bag: an HWC image (role input) + a class-NAME label (role target). + sample = TypedSample( + {"image": Image(_hwc_uint8()), "class": Label("cat", classes=list(_MAP))}, + roles={"image": "input", "class": "target"}, + ) + # Build the model INPUT tensor (CHW float) and the encoded TARGET id — no legacy Sample. + out = EncodeTarget(mapping=_MAP, field="class")(ToTensor(field="image")(sample)) + + # Input field: a CHW-float Image tagged input. + assert isinstance(out["image"], Image) + assert out["image"].layout == "CHW" + assert np.asarray(out["image"]).shape == (3, 4, 5) + assert np.asarray(out["image"]).dtype == np.float32 + assert out.role_of("image") == "input" + assert out.inputs().keys() == {"image"} + + # Target field: an int-id Label tagged target. + assert isinstance(out["class"], Label) + assert out["class"].value == 0 + assert out.role_of("class") == "target" + assert out.targets().keys() == {"class"} + + +def test_convert_then_tensor_chain() -> None: + # A raw 2-D array field runs ConvertToImage -> ToTensor into a CHW-float input. + arr = np.arange(6 * 4).reshape(6, 4).astype(np.float32) + out = ToTensor(field="image")(ConvertToImage(colormap="gray")(TypedSample({"spec": Mask(arr)}))) + assert out["image"].layout == "CHW" + assert np.asarray(out["image"]).shape == (3, 6, 4) + + +# --------------------------------------------------------------------------- # +# Discovery + zero-arg construction. +# --------------------------------------------------------------------------- # +def test_zero_arg_constructible() -> None: + assert ToTensor().output == "" + assert MetadataToTarget().output == "target" + assert EncodeTarget().mapping == {} + assert DecodeTarget().mapping == {} + + +@pytest.mark.parametrize( + ("name", "cls", "group"), + [ + ("ToTensor", ToTensor, "torch"), + ("MetadataToTarget", MetadataToTarget, "structure"), + ("EncodeTarget", EncodeTarget, "structure"), + ("DecodeTarget", DecodeTarget, "structure"), + ], +) +def test_discovery_tags(name: str, cls: type, group: str) -> None: + assert cls.__confluid_category__ == "op" # type: ignore[attr-defined] + assert cls.__confluid_group__ == group # type: ignore[attr-defined] + assert resolve_class(name) is cls + registry = get_registry() + assert name in registry.list_classes(category="op") + assert name in registry.list_classes(group=group) From 25f4803ea0518ee619f5a21a6bc4d0a1937f871a Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 07:47:32 +0200 Subject: [PATCH 030/102] =?UTF-8?q?feat(sampleflux):=20typed=20detection?= =?UTF-8?q?=20target=20ops=20=E2=80=94=20CocoToTorchVisionDetection=20+=20?= =?UTF-8?q?MasksToDetectionBoxes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive typed-bag twins for the detection target path (legacy *Op untouched). 1286 passed (+22), mypy clean, flake8 clean. - ops/target.py: CocoToTorchVisionDetection (Label objects -> Regions target) and MasksToDetectionBoxes (Mask -> Regions target), both role "target". Target is a Regions item (boxes xyxy + labels) — typed_collate gathers it into per-sample lists (variable-N detection-target convention). Delegate to the legacy ops on a shim Sample for byte-parity (pinned via torch.equal). - docs/architecture.md: consequence note; tests/test_typed_detection_target_ops.py (+23). --- docs/architecture.md | 11 ++ sampleflux/ops/target.py | 178 ++++++++++++++++++- tests/test_typed_detection_target_ops.py | 209 +++++++++++++++++++++++ 3 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 tests/test_typed_detection_target_ops.py diff --git a/docs/architecture.md b/docs/architecture.md index 91ddcc3..2a73768 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -504,6 +504,17 @@ coordinate frame, so the tuple order is load-bearing, not incidental. - The twins carry `category="op"` + `group="image"`/`"numpy"`, so they are discoverable exactly like the legacy ops (their modules were already entry-pointed; a class added to a registered module needs no new entry point). +- The two DETECTION-TARGET twins `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` + (`sampleflux/ops/target.py`, `group="structure"`) are the SAME shape reaching one step further: + they read one source field (a `Label` carrying a COCO `objects` mapping, or a `Mask`) and write + the torchvision detection target as a `Regions` item — `boxes` = the `[N,4]` xyxy tensor, + `labels` = the class-id tensor — tagged **`target`** (not `aux`: this IS the supervised target a + loss consumes, whereas `ConnectedComponents`'s raw blobs are an intermediate). `Regions` is the + natural typed home for a bounding-box set and the batch-friendly one — `typed_collate` gathers + per-sample `Regions` into a list of targets (the variable-N detection batch convention, since + boxes can't be stacked), exactly as it gathers a classification target `Label`. Byte-parity is + again free (each delegates to its legacy `*Op` on a shim `Sample`). Pinned in + `tests/test_typed_detection_target_ops.py`. ### Example diff --git a/sampleflux/ops/target.py b/sampleflux/ops/target.py index 56505cd..df40ca4 100644 --- a/sampleflux/ops/target.py +++ b/sampleflux/ops/target.py @@ -20,13 +20,22 @@ HuggingFace / COCO ``objects`` annotation (``{bbox, category}``) into the torchvision detection target ``{"boxes": xyxy, "labels"}`` (torch tensors). It is the generic, image-detection counterpart of waivefront's signal-domain ``RegionsToDetectionBoxesOp``. + +The typed-bag TWINS (:class:`MetadataToTarget` / :class:`EncodeTarget` / :class:`DecodeTarget` +and the two detection twins :class:`CocoToTorchVisionDetection` / :class:`MasksToDetectionBoxes`) +are the ``TypedSample`` counterparts of the legacy ``*Op`` classes above — STRICTLY ADDITIVE, the +legacy ops untouched. Each detection twin reads one source field and writes the torchvision +detection target as a :class:`~sampleflux.Regions` item (``boxes`` = the xyxy tensor, ``labels`` = +the class-id tensor) tagged ``target``, reusing its legacy op's conversion math VERBATIM (via a +shim :class:`~sampleflux.sample.Sample`) so the numbers are byte-identical. """ from typing import Any, Dict, Literal, Optional +import numpy as np from confluid import configurable -from sampleflux.bag.items import Label, item_data +from sampleflux.bag.items import Label, Mask, Regions, item_data from sampleflux.bag.sample import TypedSample from sampleflux.bag.transform import Transform from sampleflux.sample import Sample @@ -506,6 +515,171 @@ def __call__(self, sample: TypedSample) -> TypedSample: return out.set_role(out_key, "target") +@configurable(category="op", group="structure") +class CocoToTorchVisionDetection(Transform): + """Typed twin of :class:`CocoToTorchVisionDetectionOp` — a COCO / HF ``objects`` annotation → a target ``Regions``. + + The typed-bag counterpart of :class:`CocoToTorchVisionDetectionOp`: it reads a source field + (``field``; blank picks the first :class:`~sampleflux.Label` field, else the first field) + carrying a HuggingFace / COCO ``objects`` mapping — ``{"bbox": [[...], ...], "category": [...]}``, + each box ``[x, y, w, h]`` in absolute pixels — either the field's natural value (a ``Label``'s + ``.value``, else the item's payload) and rewrites it to the torchvision detection target. This + twin REUSES the legacy ``CocoToTorchVisionDetectionOp`` VERBATIM (its objects-shape validation + AND its bbox/category conversion math on a shim :class:`~sampleflux.sample.Sample`), so the + ``boxes`` / ``labels`` tensors are byte-identical. + + The target rides as a :class:`~sampleflux.Regions` item under ``output`` (``boxes`` = the + ``[N, 4]`` float32 xyxy-pixel tensor, ``labels`` = the ``[N]`` int64 class-id tensor) tagged + ``target`` — the natural typed home for a bounding-box set, and the batch-friendly one (the + typed collate gathers per-sample ``Regions`` into a list of targets, the variable-N detection + batch convention, exactly as the classification :class:`EncodeTarget` twin gathers a target + ``Label``). An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example + contract torchvision detectors accept). + + Args: + bbox_key: Key in the objects mapping holding per-box coordinates (default ``"bbox"``). + category_key: Key holding the per-box integer class ids (default ``"category"``). + bbox_format: Box layout in pixels — ``xywh`` (COCO, default), ``xyxy``, or ``cxcywh``; output is xyxy. + label_offset: Added to each class id (default ``0``). Set ``1`` to reserve class ``0`` for background. + field: Source field with the objects mapping; blank (default) picks the first ``Label``, else the first field. + output: Field the target ``Regions`` is written to (added if new); its role is set to ``target``. + """ + + handles = (Label,) + consumes = (Label,) + produces = (Regions,) + + def __init__( + self, + bbox_key: str = "bbox", + category_key: str = "category", + bbox_format: BBoxFormat = "xywh", + label_offset: int = 0, + field: str = "", + output: str = "target", + ) -> None: + super().__init__() + self.bbox_key = str(bbox_key) + self.category_key = str(category_key) + self.bbox_format = bbox_format + self.label_offset = int(label_offset) + self.field = str(field) + self.output = str(output) + + def _find_source(self, sample: TypedSample) -> str: + """Resolve the KEY of the source field (``self.field``, else the first ``Label``, else the first field).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError( + f"CocoToTorchVisionDetection: field {self.field!r} not in sample (fields: {list(sample.keys())})" + ) + return self.field + for key, _item in sample.items_of_type(Label): + return key + for key in sample.keys(): + return key + raise ValueError("CocoToTorchVisionDetection: sample is empty — no source field to read") + + def __call__(self, sample: TypedSample) -> TypedSample: + key = self._find_source(sample) + item = sample[key] + objects = item.value if isinstance(item, Label) else item_data(item) + # Reuse the legacy op VERBATIM (objects-shape validation + bbox/category math) on a shim + # Sample so the boxes / labels tensors are byte-identical. + target = CocoToTorchVisionDetectionOp(self.bbox_key, self.category_key, self.bbox_format, self.label_offset)( + Sample(input=None, target=objects, metadata={}) + ).target + out = sample.replace_field(self.output, Regions(boxes=target["boxes"], labels=target["labels"])) + return out.set_role(self.output, "target") + + +@configurable(category="op", group="structure") +class MasksToDetectionBoxes(Transform): + """Typed twin of :class:`MasksToDetectionBoxesOp` — a segmentation ``Mask`` → a target ``Regions``. + + The typed-bag counterpart of :class:`MasksToDetectionBoxesOp`: it reads the + :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, else the first + array-bearing item) as a 2-D integer mask and derives one tight ``[x0,y0,x1,y1]`` box per object + — either one box per distinct non-zero pixel value (``connected=False``, an instance mask) or + one box per connected component of the binarized mask (``connected=True``, via the shared + :func:`sampleflux.ops.numpy.connected_component_bboxes` helper). This twin REUSES the legacy + ``MasksToDetectionBoxesOp`` VERBATIM on a shim :class:`~sampleflux.sample.Sample`, so the + ``boxes`` / ``labels`` tensors are byte-identical. + + The target rides as a :class:`~sampleflux.Regions` item under ``output`` (``boxes`` = the + ``[N, 4]`` float32 xyxy-pixel tensor, every box's ``labels`` id = ``label``) tagged ``target`` — + the same batch-friendly representation the sibling :class:`CocoToTorchVisionDetection` twin + writes. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract + torchvision detectors accept). + + Args: + label: Foreground class id assigned to every derived box (default ``1``; class 0 = background). + connected: True = connected-components on a binary mask; False (default) = each non-zero value is one instance. + min_area: Drop objects whose mask area (in pixels) is below this (default ``1``). + connectivity: Connected-components neighborhood when ``connected=True`` — ``4`` or ``8`` (default ``4``). + field: Name of the ``Mask`` field to read; blank (default) picks the first ``Mask`` (else the first array). + output: Field the target ``Regions`` is written to (added if new); its role is set to ``target``. + """ + + handles = (Mask,) + consumes = (Mask,) + produces = (Regions,) + + def __init__( + self, + label: int = 1, + connected: bool = False, + min_area: int = 1, + connectivity: int = 4, + field: str = "", + output: str = "target", + ) -> None: + super().__init__() + self.label = int(label) + self.connected = bool(connected) + self.min_area = int(min_area) + self.connectivity = int(connectivity) + self.field = str(field) + self.output = str(output) + + def _find_mask(self, sample: TypedSample) -> np.ndarray: + """Resolve the mask array (``self.field``, else the first ``Mask``, else the first array-bearing item).""" + if self.field: + if self.field not in sample.keys(): + raise ValueError( + f"MasksToDetectionBoxes: field {self.field!r} not in sample (fields: {list(sample.keys())})" + ) + data = item_data(sample[self.field]) + else: + data = None + for _key, item in sample.items_of_type(Mask): + data = item_data(item) + break + if data is None: + for _key, item in sample.items(): + payload = item_data(item) + if isinstance(payload, np.ndarray): + data = payload + break + if data is None: + raise ValueError( + f"MasksToDetectionBoxes: no Mask or array-bearing field in sample (fields: {list(sample.keys())})" + ) + if not isinstance(data, np.ndarray): + raise TypeError(f"MasksToDetectionBoxes: expected an np.ndarray mask, got {type(data).__name__}") + return data + + def __call__(self, sample: TypedSample) -> TypedSample: + mask = self._find_mask(sample) + # Reuse the legacy op VERBATIM (instance / connected-component derivation) on a shim Sample + # so the boxes / labels tensors are byte-identical. + target = MasksToDetectionBoxesOp(self.label, self.connected, self.min_area, self.connectivity)( + Sample(input=None, target=mask, metadata={}) + ).target + out = sample.replace_field(self.output, Regions(boxes=target["boxes"], labels=target["labels"])) + return out.set_role(self.output, "target") + + __all__ = [ "MetadataToTargetOp", "EncodeTargetOp", @@ -515,4 +689,6 @@ def __call__(self, sample: TypedSample) -> TypedSample: "MetadataToTarget", "EncodeTarget", "DecodeTarget", + "CocoToTorchVisionDetection", + "MasksToDetectionBoxes", ] diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py new file mode 100644 index 0000000..a0c7273 --- /dev/null +++ b/tests/test_typed_detection_target_ops.py @@ -0,0 +1,209 @@ +"""Typed-bag TWINS of the two detection target-shaping ops. + +Pins the native typed transforms that let a ``TypedSample`` detection pipeline build its +torchvision-style ``{boxes, labels}`` target as a :class:`~sampleflux.Regions` item without the +legacy ``Sample`` path: + +* :class:`sampleflux.ops.target.CocoToTorchVisionDetection` — a HuggingFace / COCO ``objects`` + annotation → a target ``Regions``; +* :class:`sampleflux.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Regions``. + +Each twin REUSES its legacy op's conversion math, so the twin's ``boxes`` / ``labels`` tensors are +pinned byte-identical to a legacy run on the equivalent ``Sample`` (parity). sampleflux-only — no +waivefront import. +""" + +import numpy as np +import pytest +import torch +from confluid.registry import get_registry, resolve_class + +from sampleflux import Image, Label, Mask, Regions, TypedSample +from sampleflux.collate import typed_collate +from sampleflux.ops.target import ( + CocoToTorchVisionDetection, + CocoToTorchVisionDetectionOp, + MasksToDetectionBoxes, + MasksToDetectionBoxesOp, +) +from sampleflux.sample import Sample + +# A COCO / HF objects annotation: two boxes in [x, y, w, h] pixels + integer categories. +_OBJECTS = {"bbox": [[10.0, 20.0, 30.0, 40.0], [5.0, 6.0, 7.0, 8.0]], "category": [1, 3]} + + +def _instance_mask() -> np.ndarray: + """A 2-D instance mask with three known objects (pixel values 1/2/3), one per instance.""" + mask = np.zeros((10, 12), dtype=np.uint8) + mask[1:4, 2:5] = 1 # object 1 + mask[6:9, 7:10] = 2 # object 2 + mask[0:2, 9:12] = 3 # object 3 + return mask + + +# --------------------------------------------------------------------------- # +# CocoToTorchVisionDetection +# --------------------------------------------------------------------------- # +class TestCocoToTorchVisionDetection: + def test_produces_target_regions(self) -> None: + s = TypedSample({"objects": Label(_OBJECTS)}, roles={"objects": "aux"}) + out = CocoToTorchVisionDetection(field="objects")(s) + regions = out["target"] + assert isinstance(regions, Regions) + assert out.role_of("target") == "target" + assert isinstance(regions.boxes, torch.Tensor) + assert isinstance(regions.labels, torch.Tensor) + assert regions.boxes.shape == (2, 4) + assert regions.labels.shape == (2,) + + def test_parity_with_legacy(self) -> None: + typed = CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label(_OBJECTS)})) + legacy = CocoToTorchVisionDetectionOp()(Sample(input=None, target=_OBJECTS, metadata={})).target + assert torch.equal(typed["target"].boxes, legacy["boxes"]) + assert torch.equal(typed["target"].labels, legacy["labels"]) + + def test_parity_xyxy_and_label_offset(self) -> None: + objects = {"bbox": [[10.0, 20.0, 40.0, 60.0]], "category": [2]} + typed = CocoToTorchVisionDetection(field="objects", bbox_format="xyxy", label_offset=1)( + TypedSample({"objects": Label(objects)}) + ) + legacy = CocoToTorchVisionDetectionOp(bbox_format="xyxy", label_offset=1)( + Sample(input=None, target=objects, metadata={}) + ).target + assert torch.equal(typed["target"].boxes, legacy["boxes"]) + assert torch.equal(typed["target"].labels, legacy["labels"]) + + def test_empty_annotation_yields_empty_tensors(self) -> None: + out = CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label({"bbox": [], "category": []})})) + assert out["target"].boxes.shape == (0, 4) + assert out["target"].labels.shape == (0,) + + def test_default_picks_first_label(self) -> None: + s = TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "objects": Label(_OBJECTS)}) + out = CocoToTorchVisionDetection()(s) + assert out["target"].boxes.shape == (2, 4) + + def test_new_output_field_keeps_source(self) -> None: + s = TypedSample({"objects": Label(_OBJECTS)}) + out = CocoToTorchVisionDetection(field="objects", output="det")(s) + assert isinstance(out["det"], Regions) + assert out.role_of("det") == "target" + assert out["objects"].value == _OBJECTS # source left intact + + def test_missing_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + CocoToTorchVisionDetection(field="nope")(TypedSample({"objects": Label(_OBJECTS)})) + + def test_empty_sample_raises(self) -> None: + with pytest.raises(ValueError, match="sample is empty"): + CocoToTorchVisionDetection()(TypedSample({})) + + def test_non_dict_source_raises(self) -> None: + # The reused legacy op rejects a non-objects-shaped value loudly. + with pytest.raises(TypeError, match="objects mapping"): + CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label("not a dict")})) + + +# --------------------------------------------------------------------------- # +# MasksToDetectionBoxes +# --------------------------------------------------------------------------- # +class TestMasksToDetectionBoxes: + def test_instance_mask_produces_target_regions(self) -> None: + s = TypedSample({"mask": Mask(_instance_mask())}, roles={"mask": "aux"}) + out = MasksToDetectionBoxes(field="mask")(s) + regions = out["target"] + assert isinstance(regions, Regions) + assert isinstance(regions.boxes, torch.Tensor) + assert isinstance(regions.labels, torch.Tensor) + assert out.role_of("target") == "target" + assert regions.boxes.shape == (3, 4) # three instances + assert regions.labels.tolist() == [1, 1, 1] # every box → foreground class 1 + + def test_instance_parity_with_legacy(self) -> None: + mask = _instance_mask() + typed = MasksToDetectionBoxes(field="mask")(TypedSample({"mask": Mask(mask)})) + legacy = MasksToDetectionBoxesOp()(Sample(input=None, target=mask, metadata={})).target + assert torch.equal(typed["target"].boxes, legacy["boxes"]) + assert torch.equal(typed["target"].labels, legacy["labels"]) + + def test_connected_components_parity(self) -> None: + # A binary/semantic mask (all objects share value 1): connected=True splits into blobs. + binary = (_instance_mask() != 0).astype(np.uint8) + typed = MasksToDetectionBoxes(field="mask", connected=True, label=2)(TypedSample({"mask": Mask(binary)})) + legacy = MasksToDetectionBoxesOp(connected=True, label=2)(Sample(input=None, target=binary, metadata={})).target + assert typed["target"].boxes.shape[0] == 3 # three connected blobs + assert torch.equal(typed["target"].boxes, legacy["boxes"]) + assert torch.equal(typed["target"].labels, legacy["labels"]) + + def test_min_area_drops_small_instances(self) -> None: + mask = _instance_mask() + typed = MasksToDetectionBoxes(field="mask", min_area=10)(TypedSample({"mask": Mask(mask)})) + legacy = MasksToDetectionBoxesOp(min_area=10)(Sample(input=None, target=mask, metadata={})).target + assert torch.equal(typed["target"].boxes, legacy["boxes"]) + + def test_empty_mask_yields_empty_tensors(self) -> None: + out = MasksToDetectionBoxes(field="mask")(TypedSample({"mask": Mask(np.zeros((4, 4), dtype=np.uint8))})) + assert out["target"].boxes.shape == (0, 4) + assert out["target"].labels.shape == (0,) + + def test_default_picks_first_mask(self) -> None: + s = TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "seg": Mask(_instance_mask())}) + out = MasksToDetectionBoxes()(s) + assert out["target"].boxes.shape == (3, 4) + + def test_new_output_field_keeps_source(self) -> None: + s = TypedSample({"mask": Mask(_instance_mask())}) + out = MasksToDetectionBoxes(field="mask", output="det")(s) + assert isinstance(out["det"], Regions) + assert out.role_of("det") == "target" + assert isinstance(out["mask"], Mask) # source left intact + + def test_missing_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in sample"): + MasksToDetectionBoxes(field="nope")(TypedSample({"mask": Mask(_instance_mask())})) + + def test_no_mask_or_array_field_raises(self) -> None: + with pytest.raises(ValueError, match="no Mask or array-bearing field"): + MasksToDetectionBoxes()(TypedSample({"lbl": Label("x")})) + + +# --------------------------------------------------------------------------- # +# Collate — per-sample Regions gather into a list of detection targets. +# --------------------------------------------------------------------------- # +def test_typed_collate_gathers_regions_as_list() -> None: + a = CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label(_OBJECTS)})) + c = CocoToTorchVisionDetection(field="objects")( + TypedSample({"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [5]})}) + ) + batch = typed_collate([a, c]) + # Variable-N boxes can't be stacked → the collate gathers them as a per-sample list of tensors. + assert isinstance(batch["target"], Regions) + assert isinstance(batch["target"].boxes, list) and len(batch["target"].boxes) == 2 + assert batch["target"].boxes[0].shape == (2, 4) + assert batch["target"].boxes[1].shape == (1, 4) + + +# --------------------------------------------------------------------------- # +# Discovery + zero-arg construction. +# --------------------------------------------------------------------------- # +def test_zero_arg_constructible() -> None: + assert CocoToTorchVisionDetection().output == "target" + assert CocoToTorchVisionDetection().bbox_format == "xywh" + assert MasksToDetectionBoxes().output == "target" + assert MasksToDetectionBoxes().connected is False + + +@pytest.mark.parametrize( + ("name", "cls"), + [ + ("CocoToTorchVisionDetection", CocoToTorchVisionDetection), + ("MasksToDetectionBoxes", MasksToDetectionBoxes), + ], +) +def test_discovery_tags(name: str, cls: type) -> None: + assert cls.__confluid_category__ == "op" # type: ignore[attr-defined] + assert cls.__confluid_group__ == "structure" # type: ignore[attr-defined] + assert resolve_class(name) is cls + registry = get_registry() + assert name in registry.list_classes(category="op") + assert name in registry.list_classes(group="structure") From ae15b70c0014500e7c29f503f63f94c6e8ad429d Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 08:09:11 +0200 Subject: [PATCH 031/102] feat(sampleflux)!: HuggingFaceSource yields TypedSample (non-additive) HuggingFaceSource.__iter__/__getitem__ now mint a TypedSample: - input_feature -> Image field "image" (role input) - target_feature -> Label field "class" (role target) - each resolved metadata_features column -> its own Label field (role aux); METADATA_ALL_FEATURES "*" sentinel semantics unchanged - hf_path/hf_split provenance -> Label aux fields (traceability preserved) Kept lazy, zero-arg, random-access __getitem__, __len__. Added the typed SupportsProjection.project() efficient path (role-restricted, skips image decode). projection.py made carrier-aware (_carrier_field reads .input/.target from a Sample or primary()/Label.value from a TypedSample) so num_classes/iter_targets work for both; legacy behavior byte-identical. DatasetSplit/RangeSource/ConcatSource pass a TypedSample through verbatim (_pass_through). sampleflux 1286 passed, mypy clean, flake8 clean. Consumers that run a model on this source (sonair etc.) still need their typed collate flip, pending the separate marainer training-removal refactoring reaching them. --- sampleflux/projection.py | 51 +++++++++++--- sampleflux/sources.py | 148 +++++++++++++++++++++++++++------------ tests/test_sources.py | 27 +++++-- 3 files changed, 166 insertions(+), 60 deletions(-) diff --git a/sampleflux/projection.py b/sampleflux/projection.py index 06e5631..41612e8 100644 --- a/sampleflux/projection.py +++ b/sampleflux/projection.py @@ -25,6 +25,8 @@ from typing import Any, Collection, Iterator, Literal, Protocol, Tuple, get_args, runtime_checkable +from sampleflux.bag.items import Label, item_data +from sampleflux.bag.sample import Role, TypedSample, primary from sampleflux.sample import Sample #: The projectable :class:`~sampleflux.sample.Sample` fields, as a *closed* @@ -56,14 +58,38 @@ class SupportsProjection(Protocol): def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: ... -def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample]: - """Yield :class:`Sample` records from ``source`` carrying only ``fields``. +def _carrier_field(carrier: Any, field: ProjectionField) -> Any: + """Read one field's VALUE from either a legacy :class:`Sample` or a typed :class:`TypedSample`. - Uses the source's own ``project`` when it implements - :class:`SupportsProjection` (the efficient path that skips building - unrequested fields); otherwise falls back to a full iteration that builds - every field and nulls the unrequested ones — always correct, just not faster. - Lazy: a generator that never materializes the source. + For a ``TypedSample`` (the typed-bag carrier the migrated sources yield) the value of the + ``input`` / ``target`` role is the FIRST field of that role — a ``Label``'s ``.value`` (the class + id / scalar), else the item's raw payload (:func:`item_data`). A missing role yields ``None``, so + a target-only walk over a typed source feeds :func:`num_classes` exactly as the legacy carrier did. + """ + if isinstance(carrier, TypedSample): + role: Role = "input" if field == INPUT else "target" + try: + _key, item = primary(carrier, role) + except KeyError: + return None + return item.value if isinstance(item, Label) else item_data(item) + s = Sample.from_any(carrier) + if field == INPUT: + return s.input + if field == TARGET: + return s.target + return s.meta + + +def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Any]: + """Yield partial records from ``source`` carrying only ``fields``. + + Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the efficient + path that skips building unrequested fields); otherwise falls back to a full iteration that builds + every field and nulls the unrequested ones — always correct, just not faster. A typed-bag + :class:`TypedSample` is passed through VERBATIM (never coerced into a legacy ``Sample``); the walk + helpers below extract the requested field from whichever carrier flows. Lazy: a generator that + never materializes the source. """ want = frozenset(fields) unknown = want - frozenset(_FIELDS) @@ -73,6 +99,9 @@ def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample yield from source.project(want) return for raw in source: + if isinstance(raw, TypedSample): + yield raw + continue s = Sample.from_any(raw) yield Sample( input=s.input if INPUT in want else None, @@ -82,15 +111,15 @@ def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample def iter_inputs(source: Any) -> Iterator[Any]: - """Lazily yield each sample's ``input`` (skipping target construction when supported).""" + """Lazily yield each sample's ``input`` value (skipping target construction when supported).""" for s in project(source, (INPUT,)): - yield s.input + yield _carrier_field(s, INPUT) def iter_targets(source: Any) -> Iterator[Any]: - """Lazily yield each sample's ``target`` (skipping input construction when supported).""" + """Lazily yield each sample's ``target`` value (skipping input construction when supported).""" for s in project(source, (TARGET,)): - yield s.target + yield _carrier_field(s, TARGET) def _to_int(value: Any) -> int: diff --git a/sampleflux/sources.py b/sampleflux/sources.py index 57e37e1..edb6460 100644 --- a/sampleflux/sources.py +++ b/sampleflux/sources.py @@ -1,14 +1,30 @@ import bisect import random -from typing import Any, Dict, Iterator, List, Literal, Optional, get_args +from typing import Any, Collection, Dict, Iterator, List, Literal, Optional, get_args from confluid import configurable from loggair import get_logger +from sampleflux.bag import Image, Label, TypedSample +from sampleflux.projection import ProjectionField from sampleflux.sample import Sample logger = get_logger(__name__) + +def _pass_through(item: Any) -> Any: + """Coerce a wrapped source's item to a carrier the engine accepts. + + A typed-bag :class:`~sampleflux.TypedSample` is passed through VERBATIM — the view sources + (:class:`DatasetSplit` / :class:`RangeSource` / :class:`ConcatSource`) only slice/index, they + never inspect payloads, so a typed source flows through them unchanged. Any legacy carrier is + normalized to a :class:`~sampleflux.sample.Sample` via ``Sample.from_any``. + """ + if isinstance(item, TypedSample): + return item + return Sample.from_any(item) + + # Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer # closed Literals over bare strings — self-documenting + machine-introspectable by UIs / # navigaitor form-spec / MCP schemas via ``typing.get_args``). The runtime-validation tuple @@ -52,8 +68,16 @@ def _resolve_metadata_features( @configurable(category="source") class HuggingFaceSource: """ - SampleFlux Source for Hugging Face Datasets. - Configurable mapping of dataset features to SampleFlux Sample triplets. + SampleFlux Source for Hugging Face Datasets, yielding typed-bag :class:`~sampleflux.TypedSample`\\ s. + + Field mapping (the typed-bag layout that replaces the ``Sample(input, target, metadata)`` triple): + + * the ``input_feature`` value (image / array) -> an :class:`~sampleflux.Image` field named + ``"image"`` (role ``input``); + * the ``target_feature`` value (label) -> a :class:`~sampleflux.Label` field named ``"class"`` + (role ``target``); + * each ``metadata_features`` column -> its own :class:`~sampleflux.Label` field keyed by the column + name (role ``aux``), plus the source-provenance ``hf_path`` / ``hf_split`` aux Labels. Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and @@ -64,9 +88,9 @@ class HuggingFaceSource: Args: path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. split: HF split name (``train`` / ``validation`` / ``test`` / etc.). - input_feature: Dataset feature column to map onto ``Sample.input``. - target_feature: Dataset feature column to map onto ``Sample.target``. - metadata_features: Columns onto ``Sample.metadata``; ``None``=none, ``"*"``=all but input/target, else a list. + input_feature: Dataset feature column mapped onto the ``"image"`` input field (an ``Image`` item). + target_feature: Dataset feature column mapped onto the ``"class"`` target field (a ``Label`` item). + metadata_features: Columns -> per-column aux ``Label`` fields; ``None``=none, ``"*"``=all-but-i/o, else a list. count: Optional cap on the number of samples yielded (useful for fast smoke runs). name: Optional HF subset/config name (e.g. for multi-config datasets). """ @@ -130,40 +154,78 @@ def resolved_metadata_features(self) -> List[str]: self.metadata_features, getattr(self.dataset, "column_names", None), self.input_feature, self.target_feature ) - def __iter__(self) -> Iterator[Sample]: - counter = 0 + def _to_typed_sample( + self, + item: Any, + metadata_features: List[str], + *, + want_input: bool = True, + want_target: bool = True, + want_meta: bool = True, + ) -> TypedSample: + """Assemble one :class:`~sampleflux.TypedSample` from a raw HF row dict (see the class docstring + for the field mapping). + + ``want_input`` / ``want_target`` / ``want_meta`` gate which roles are built — the projection + path (:meth:`project`) passes only the requested ones, so an unwanted image is never decoded. + """ + fields: Dict[str, Any] = {} + roles: Dict[str, Any] = {} + if want_input: + # The input value (image/array) becomes an ``Image`` item; a PIL image / list is coerced + # to an ndarray by ``Image.__new__`` (np.asarray), preserving the default HWC layout. + fields["image"] = Image(item.get(self.input_feature)) + roles["image"] = "input" + if want_target: + fields["class"] = Label(item.get(self.target_feature)) + roles["class"] = "target" + if want_meta: + # Each requested metadata column rides its OWN aux Label field (typed-bag: metadata belongs + # to the item it describes), keyed by the column name. Source provenance follows the same shape. + for feature in metadata_features: + fields[feature] = Label(item.get(feature)) + roles[feature] = "aux" + fields["hf_path"] = Label(self.path) + fields["hf_split"] = Label(self.split) + roles["hf_path"] = "aux" + roles["hf_split"] = "aux" + return TypedSample(fields, roles) + + def __iter__(self) -> Iterator[TypedSample]: dataset = self.dataset metadata_features = self.resolved_metadata_features limit = self.count or len(dataset) - for item in dataset: + for counter, item in enumerate(dataset): if counter >= limit: break + yield self._to_typed_sample(item, metadata_features) - # 1. Extract Input - input_val = item.get(self.input_feature) - - # 2. Extract Target - target_val = item.get(self.target_feature) - - # 3. Build Metadata - metadata = {f: item.get(f) for f in metadata_features} - metadata["hf_path"] = self.path - metadata["hf_split"] = self.split - - yield Sample(input=input_val, target=target_val, metadata=metadata) - counter += 1 - - def __getitem__(self, index: int) -> Sample: - item = self.dataset[index] - metadata = {f: item.get(f) for f in self.resolved_metadata_features} - metadata["hf_path"] = self.path - metadata["hf_split"] = self.split - return Sample( - input=item.get(self.input_feature), - target=item.get(self.target_feature), - metadata=metadata, - ) + def __getitem__(self, index: int) -> TypedSample: + return self._to_typed_sample(self.dataset[index], self.resolved_metadata_features) + + def project(self, fields: Collection[ProjectionField]) -> Iterator[TypedSample]: + """Yield role-restricted ``TypedSample``\\ s — the ``SupportsProjection`` efficient path. + + Only the requested roles are built, so a target-only walk (e.g. :func:`~sampleflux.num_classes`) + skips decoding the image entirely: ``"input"`` -> the ``"image"`` field, ``"target"`` -> the + ``"class"`` Label, ``"metadata"`` -> the aux metadata-feature / provenance Labels. + """ + want = frozenset(fields) + dataset = self.dataset + want_meta = "metadata" in want + metadata_features = self.resolved_metadata_features if want_meta else [] + limit = self.count or len(dataset) + for counter, item in enumerate(dataset): + if counter >= limit: + break + yield self._to_typed_sample( + item, + metadata_features, + want_input="input" in want, + want_target="target" in want, + want_meta=want_meta, + ) def __len__(self) -> int: # A ``count`` of 0 (or None) means "all samples", matching __iter__'s @@ -315,7 +377,7 @@ def test(self) -> "_SplitView": def __iter__(self) -> Iterator[Sample]: return iter(self._view(self.split or "train")) - def __getitem__(self, index: int) -> Sample: + def __getitem__(self, index: int) -> Any: return self._view(self.split or "train")[index] def __len__(self) -> int: @@ -337,10 +399,10 @@ def __init__(self, source: Any, indices: List[int]) -> None: def __iter__(self) -> Iterator[Sample]: for idx in self.indices: - yield Sample.from_any(self.source[idx]) + yield _pass_through(self.source[idx]) - def __getitem__(self, index: int) -> Sample: - return Sample.from_any(self.source[self.indices[index]]) + def __getitem__(self, index: int) -> Any: + return _pass_through(self.source[self.indices[index]]) def __len__(self) -> int: return len(self.indices) @@ -395,10 +457,10 @@ def indices(self) -> List[int]: def __iter__(self) -> Iterator[Sample]: for idx in self.indices: - yield Sample.from_any(self.source[idx]) + yield _pass_through(self.source[idx]) - def __getitem__(self, index: int) -> Sample: - return Sample.from_any(self.source[self.indices[index]]) + def __getitem__(self, index: int) -> Any: + return _pass_through(self.source[self.indices[index]]) def __len__(self) -> int: return len(self.indices) @@ -450,7 +512,7 @@ def offsets(self) -> List[int]: def __len__(self) -> int: return self.offsets[-1] if self.offsets else 0 - def __getitem__(self, index: int) -> Sample: + def __getitem__(self, index: int) -> Any: n = len(self) if index < 0: index += n @@ -458,9 +520,9 @@ def __getitem__(self, index: int) -> Sample: raise IndexError(index) j = bisect.bisect_right(self.offsets, index) start = self.offsets[j - 1] if j > 0 else 0 - return Sample.from_any(self.sources[j][index - start]) + return _pass_through(self.sources[j][index - start]) def __iter__(self) -> Iterator[Sample]: for src in self.sources: for item in src: - yield Sample.from_any(item) + yield _pass_through(item) diff --git a/tests/test_sources.py b/tests/test_sources.py index f5a1381..1569641 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -532,8 +532,9 @@ def __init__(self, rows: List[Any], column_names: List[str]) -> None: def test_hf_source_iter_metadata_features_star_expands_on_real_dataset() -> None: # End-to-end through __iter__: a dataset with extra columns + metadata_features="*" carries - # every non-input/target column onto Sample.metadata (plus the synthetic hf_path/hf_split). + # every non-input/target column onto its OWN aux Label field (plus the synthetic hf_path/hf_split). # The "*" expansion is now lazy (resolved_metadata_features reads dataset.column_names). + from sampleflux import Image, Label, TypedSample from sampleflux.sources import HuggingFaceSource rows = [{"image": i, "label": i % 2, "id": f"r{i}", "src": "a"} for i in range(3)] @@ -541,8 +542,22 @@ def test_hf_source_iter_metadata_features_star_expands_on_real_dataset() -> None src._dataset = _StubHFDataset(rows, ["image", "label", "id", "src"]) # pre-seed: no network samples = list(src) - assert [s.input for s in samples] == [0, 1, 2] - md = samples[0].meta - assert md["id"] == "r0" and md["src"] == "a" - assert "image" not in md and "label" not in md # input/target excluded from metadata - assert md["hf_path"] == "fake/ds" and md["hf_split"] == "train" + assert all(isinstance(s, TypedSample) for s in samples) + + # input_feature -> "image" Image (role input); target_feature -> "class" Label (role target). + assert [int(s["image"]) for s in samples] == [0, 1, 2] + assert all(isinstance(s["image"], Image) and s.role_of("image") == "input" for s in samples) + assert [s["class"].value for s in samples] == [0, 1, 0] # label = i % 2 + assert all(isinstance(s["class"], Label) and s.role_of("class") == "target" for s in samples) + + # Each metadata column rides its own aux Label field, keyed by the column name. + s0 = samples[0] + assert s0["id"].value == "r0" and s0["src"].value == "a" + assert s0.role_of("id") == "aux" and s0.role_of("src") == "aux" + assert isinstance(s0["id"], Label) and isinstance(s0["src"], Label) + # input/target features are NOT duplicated as aux metadata fields. + assert s0.role_of("image") == "input" and s0.role_of("class") == "target" + + # Source provenance rides aux Label fields too. + assert s0["hf_path"].value == "fake/ds" and s0["hf_split"].value == "train" + assert s0.role_of("hf_path") == "aux" and s0.role_of("hf_split") == "aux" From d22eddbe53ad188303ef10ae5c077129da5a1910 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 08:36:17 +0200 Subject: [PATCH 032/102] feat: runnable protocol + workflow/processing + run CLI + entrypoint markers Add the carrier-agnostic runnable/orchestration layer (moved from marainer): - runnable.py: TorchRunner + ProgressReporting marker mixins, plus the entrypoint(task, role, primary) method-annotation decorator + runnable_entrypoints/entrypoint_tasks introspectors so one merged train+eval class can declare its per-capability entry points. - workflow.py: Sequence/Conditional/Switch combinators + PathExists/Not/AllOf/AnyOf. - processing.py: DatasetProcessor (zero-arg constructible; flux validated in run()). - cli.py: 'sampleflux run ' liquifai app running any Confluid-wired runnable. Exported at the package top level; pyproject gains liquifai dep, the sampleflux console script, liquifai.apps + sampleflux-processing/-workflow entry points. --- pyproject.toml | 22 +++- sampleflux/__init__.py | 25 ++++ sampleflux/cli.py | 63 +++++++++ sampleflux/processing.py | 153 ++++++++++++++++++++++ sampleflux/runnable.py | 156 ++++++++++++++++++++++ sampleflux/workflow.py | 270 +++++++++++++++++++++++++++++++++++++++ tests/test_cli_run.py | 44 +++++++ tests/test_entrypoint.py | 56 ++++++++ tests/test_processing.py | 172 +++++++++++++++++++++++++ tests/test_runnable.py | 71 ++++++++++ tests/test_workflow.py | 243 +++++++++++++++++++++++++++++++++++ 11 files changed, 1274 insertions(+), 1 deletion(-) create mode 100644 sampleflux/cli.py create mode 100644 sampleflux/processing.py create mode 100644 sampleflux/runnable.py create mode 100644 sampleflux/workflow.py create mode 100644 tests/test_cli_run.py create mode 100644 tests/test_entrypoint.py create mode 100644 tests/test_processing.py create mode 100644 tests/test_runnable.py create mode 100644 tests/test_workflow.py diff --git a/pyproject.toml b/pyproject.toml index 33bfd50..8dfa412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,11 @@ dependencies = [ "orjson", "fsspec", "cloudpathlib", - "scikit-learn" # LabelMap.fit() uses sklearn.preprocessing.LabelEncoder (lazy-imported) + "scikit-learn", # LabelMap.fit() uses sklearn.preprocessing.LabelEncoder (lazy-imported) + # liquifai powers the generic `sampleflux run ` CLI (sampleflux.cli) that + # runs any Confluid-wired runnable; it also brings `rich` (used by DatasetProcessor's + # optional console progress bar). liquifai depends only on confluid/loggair/rich — no cycle. + "liquifai>=0.1.0", ] requires-python = ">=3.12" @@ -112,6 +116,22 @@ sampleflux-bag-transform = "sampleflux.bag.transform" # Typed-bag structure ops (SetRole/RenameField/DropField/CopyField/SelectFields) — reshape a # TypedSample's named fields; the typed replacement for the classic triple-slot plumbing. sampleflux-ops-structure = "sampleflux.ops.structure" +# The runnable orchestration layer: DatasetProcessor (generic source→sink runner) and the +# workflow combinators (Sequence/Conditional/Switch + PathExists/Not/AllOf/AnyOf predicates). +# Discovered as @configurable runnables; run via `sampleflux run`. +sampleflux-processing = "sampleflux.processing" +sampleflux-workflow = "sampleflux.workflow" + +# The `sampleflux` console script — `sampleflux run ` runs any Confluid-wired +# runnable (a trainer, an evaluator, a DatasetProcessor, a workflow). +[project.scripts] +sampleflux = "sampleflux.cli:main" + +# Liquifai app registration: lets `liquifai-install-completions` discover this CLI via an +# instant entry-point metadata read instead of a subprocess probe. Name = the binary name; +# value = the LiquifyApp instance. +[project.entry-points."liquifai.apps"] +sampleflux = "sampleflux.cli:app" [tool.setuptools.packages.find] where = ["."] diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py index a75db7b..7cd8042 100644 --- a/sampleflux/__init__.py +++ b/sampleflux/__init__.py @@ -52,7 +52,16 @@ from sampleflux.kinds import INPUT, TARGET, Input, OpContract, SampleKind, Target, classify_carrier, op_contract from sampleflux.labels import LabelMap from sampleflux.ops import RescaleOp, StandardizeOp, ToTensorOp +from sampleflux.processing import DatasetProcessor from sampleflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project +from sampleflux.runnable import ( + ProgressCallback, + ProgressReporting, + TorchRunner, + entrypoint, + entrypoint_tasks, + runnable_entrypoints, +) from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName from sampleflux.typespec import ( @@ -73,6 +82,7 @@ infer_type, typed, ) +from sampleflux.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch __all__ = [ # ---- typed-bag surface (THE data model) ---- @@ -119,6 +129,21 @@ "get_collate", "register_collate", "LabelMap", + # ---- runnable protocol + orchestration (carrier-agnostic) ---- + "TorchRunner", + "ProgressReporting", + "ProgressCallback", + "entrypoint", + "entrypoint_tasks", + "runnable_entrypoints", + "DatasetProcessor", + "Sequence", + "Conditional", + "Switch", + "PathExists", + "Not", + "AllOf", + "AnyOf", # ---- legacy surface (dies with the purge stage) ---- "AnyType", "ArrayType", diff --git a/sampleflux/cli.py b/sampleflux/cli.py new file mode 100644 index 0000000..442b25a --- /dev/null +++ b/sampleflux/cli.py @@ -0,0 +1,63 @@ +"""The ``sampleflux`` CLI — a generic runner for any Confluid-wired runnable. + +``sampleflux run `` loads a Confluid YAML that binds a *runnable* +object (anything exposing a no-arg ``run()``) under the top-level ``runnable:`` +key, flows it under the active context (so nested ``!ref:`` markers resolve), and +calls ``run()``. This is the single entry point that replaces bespoke per-verb +CLIs: a training run, an evaluation, a dataset conversion, or a whole +:mod:`~sampleflux.workflow` are all just runnables — the ``!class:`` the YAML roots +on decides what happens. + +Example:: + + # convert.yaml + runnable: !class:sampleflux.processing.DatasetProcessor + flux: !class:sampleflux.Flux { source: !class:my.Source(), ops: [...] } + sink: !class:sampleflux.storage.HDF5Sink { path: out.h5 } + + sampleflux run convert.yaml +""" + +from typing import Any + +from liquifai import LiquifyApp +from loggair import get_logger + +logger = get_logger(__name__) + +app = LiquifyApp(name="sampleflux") + + +@app.script_command(flow_mode="auto") +def run(runnable: Any) -> None: + """Run any Confluid-instantiated object that exposes ``.run()``. + + The YAML config binds the object under the top-level ``runnable:`` key. + ``flow_mode="auto"`` deep-flows it under Confluid's active context so nested + ``!ref:`` markers resolve against the loaded YAML's top-level keys. + """ + if runnable is None: + logger.error("'runnable' was not bound — provide one under 'runnable:' in your YAML.") + return + + from confluid import flow + from confluid.fluid import Fluid + + if isinstance(runnable, Fluid): + runnable = flow(runnable) + + label = runnable.__class__.__name__ + run_method = getattr(runnable, "run", None) + if not callable(run_method): + logger.error(f"The injected runnable ({label}) does not implement 'run()'.") + return + logger.info(f"sampleflux running: {label}") + run_method() + + +def main() -> None: + app.run() + + +if __name__ == "__main__": + main() diff --git a/sampleflux/processing.py b/sampleflux/processing.py new file mode 100644 index 0000000..68cb2da --- /dev/null +++ b/sampleflux/processing.py @@ -0,0 +1,153 @@ +"""Generic source→sink pipeline runner. + +:class:`DatasetProcessor` orchestrates a :class:`~sampleflux.core.Flux` from source +to sink — a runnable that drives whole-dataset processing (windowing, format +conversion, data acquisition) with an optional console progress bar. It is the +generic, modality-neutral data-pipeline runner: it iterates the flux and writes +each item to the sink, carrier-agnostic (it never inspects item internals), so it +works for any ``Flux`` regardless of what flows through it. + +Wired as the ``runnable:`` object of a config and run via ``sampleflux run``, or +docked into a visual-editor canvas as a runnable node. +""" + +from contextlib import nullcontext +from typing import Any, Iterable, Iterator, Optional + +from confluid import configurable +from loggair import get_logger +from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeRemainingColumn + +from sampleflux.core import Flux +from sampleflux.runnable import ProgressReporting +from sampleflux.storage.base import Storage + +logger = get_logger(__name__) + + +def _flux_total(flux: Flux) -> Optional[int]: + """``len(flux.source)`` when the source is sized, else ``None`` (a glob-based / streaming source).""" + try: + return len(flux.source) # type: ignore[arg-type] + except (TypeError, AttributeError): + return None + + +@configurable +class DatasetProcessor(ProgressReporting): + """Orchestrate a SampleFlux pipeline from source to sink. + + Args: + flux: The :class:`~sampleflux.core.Flux` to execute. Required to run; + defaulted to ``None`` for zero-arg construction (validated in + :meth:`run`, the workspace lazy-construction rule). + sink: Optional sink; when absent, samples are materialized to a list. + show_progress: If ``True``, wrap iteration with a ``rich.progress`` bar. + The total is derived from ``len(flux.source)`` when available; an + unsized source falls back to a count-only bar. Default ``False``. + progress_desc: Optional label for the progress bar (defaults to + ``"DatasetProcessor"``). Ignored when ``show_progress`` is ``False``. + """ + + def __init__( + self, + flux: Optional[Flux] = None, + sink: Optional[Any] = None, + show_progress: bool = False, + progress_desc: Optional[str] = None, + ) -> None: + self.flux = flux + self.sink = sink + self.show_progress = show_progress + self.progress_desc = progress_desc + + def run(self) -> None: + logger.info("Starting DatasetProcessor...") + if self.flux is None: + raise ValueError("DatasetProcessor.run() requires a 'flux' — none was configured.") + # Confluid keeps Class kwargs deferred (post-construction paradigm) so + # when the processor was loaded from YAML, ``self.flux``, its source, + # its ops, and ``self.sink`` may all be Fluid stubs. Materialize them + # here so callers don't need to know. + from confluid import flow + from confluid.fluid import Fluid + + flux = flow(self.flux) if isinstance(self.flux, Fluid) else self.flux + if isinstance(flux.source, Fluid): + flux.source = flow(flux.source) + flux.ops = [flow(op) if isinstance(op, Fluid) else op for op in flux.ops] + self.flux = flux + sink = flow(self.sink) if isinstance(self.sink, Fluid) else self.sink + + iterator = self._wrap_progress(flux) + # Drive an executor's progress bar (a GUI canvas) per item — independent of the console + # ``show_progress`` rich bar; a no-op when no progress callback was injected. + total = _flux_total(flux) + desc = self.progress_desc or "DatasetProcessor" + + if sink: + logger.info(f"Streaming data to sink: {sink.__class__.__name__}") + # Replicates sampleflux.core.Flux.to_sink so we can iterate through + # our progress wrapper while preserving the Storage context + flush. + sink_ctx: Any = sink if isinstance(sink, Storage) else nullcontext() + count = 0 + with sink_ctx: + for sample in iterator: + sink.write(sample) + count += 1 + self._report_progress(count, total, desc) + sink.flush() + logger.info(f"Streamed {count} sample(s) to sink.") + else: + logger.info("No sink provided. Materializing data in-memory.") + results = [] + for count, sample in enumerate(iterator, start=1): + results.append(sample) + self._report_progress(count, total, desc) + logger.info(f"Processed {len(results)} samples.") + + logger.info("Processing complete.") + + def _wrap_progress(self, flux: Flux) -> Iterable[Any]: + """Wrap the flux iterator in rich.progress when ``show_progress`` is enabled. + + Source length is probed defensively — not every DataSource implements + ``__len__`` (e.g. glob-based streaming sources). Missing lengths + degrade to a count-only bar instead of breaking the run. + """ + if not self.show_progress: + return flux + desc = self.progress_desc or "DatasetProcessor" + return _ProgressIter(flux, total=_flux_total(flux), desc=desc) + + +class _ProgressIter: + """Iterable wrapper that opens a ``rich.progress`` bar at ``__iter__`` time. + + Kept as a class (not a generator) so ``DatasetProcessor._wrap_progress`` + can return a value that is truthy to ``bool()`` even when empty, matching + the contract of raw ``Flux`` which behaves like a ``Sized`` (``Flux`` + subclasses ``torch.utils.data.Dataset``). + """ + + def __init__(self, flux: Flux, total: Optional[int], desc: str) -> None: + self._flux = flux + self._total = total + self._desc = desc + + def __iter__(self) -> Iterator[Any]: + with Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TextColumn("•"), + TimeRemainingColumn(), + transient=True, + ) as progress: + task = progress.add_task(self._desc, total=self._total) + for sample in self._flux: + yield sample + progress.update(task, advance=1) + + +__all__ = ["DatasetProcessor"] diff --git a/sampleflux/runnable.py b/sampleflux/runnable.py new file mode 100644 index 0000000..b333518 --- /dev/null +++ b/sampleflux/runnable.py @@ -0,0 +1,156 @@ +"""The runnable protocol: gradient + progress marker mixins for long-running objects. + +A *runnable* is any object exposing a no-arg ``run(self)`` — a trainer, an +evaluator, a dataset processor, a workflow. This module carries the two stateless +mixins a runnable inherits so a GUI executor (one that runs the object as a graph +node) can cooperate with it WITHOUT this package importing the GUI framework: + +* :class:`TorchRunner` — marks a runnable whose ``run()`` needs autograd (it + performs gradient-based optimization). A GUI executor that evaluates nodes under + ``torch.inference_mode()`` reads the duck-typed ``__torch_runner__`` flag and + re-enables autograd for the duration of ``run()``. +* :class:`ProgressReporting` — gives a runnable a framework-free progress callback. + The executor injects a ``(value, total, desc) -> None`` sink via + :meth:`~ProgressReporting.set_progress_callback`; the runnable drains it from its + loop via :meth:`~ProgressReporting._report_progress` (a silent no-op when no sink + was injected, so a plain CLI run is unaffected). + +Both are pure marker/utility mixins (no required state) so they compose cleanly +with any base class (a ``torch.nn.Module``, a ``LightningModule``, a plain object). +This package declares only *that* a runnable needs autograd / can report progress; +the framework-specific bridges (e.g. a Lightning ``Callback`` that maps per-batch +hooks onto :meth:`~ProgressReporting._report_progress`) live in the consuming +framework package, not here. + +It also carries the :func:`entrypoint` method marker: a runnable that exposes SEVERAL +capabilities from one class — the merged train+eval classes drive ``fit`` / +``evaluate`` / ``test`` / ``predict`` off a single ``task`` knob — annotates each such +method with the ``task`` value it runs and a ``role`` label (``"trainer"`` / +``"evaluator"`` / ``"predictor"``). A discovery consumer (a config generator, a visual +editor) reads these via :func:`runnable_entrypoints` to learn that one class both +trains and evaluates, instead of assuming a separate class per role. +""" + +from typing import Callable, Dict, List, Optional + +from loggair import get_logger + +logger = get_logger(__name__) + +#: Attribute stamped on a method by :func:`entrypoint`. +_ENTRYPOINT_ATTR = "__runnable_entrypoint__" + +#: A progress sink: ``(value, total, description) -> None``. ``value`` / ``total`` are +#: floats in the same unit (optimizer steps, samples); ``description`` is a short stage label. +ProgressCallback = Callable[[float, float, str], None] + + +class TorchRunner: + """Mixin marking a runnable whose ``run()`` performs gradient-based optimization. + + Pure marker — no state, no ``__init__`` — so it composes cleanly with any base + (``torch.nn.Module``, ``pytorch_lightning.LightningModule``, …). It lets a GUI + executor distinguish "this run needs autograd" from "this run is inference-only". + + Why it exists: a GUI executor may call ``run()`` from inside a graph evaluation + wrapped in ``torch.inference_mode()`` for cheap, grad-free node evaluation. Under + inference mode every tensor created — model parameters, forward activations, the + loss — is an inference tensor with no autograd graph, so ``loss.backward()`` dies + with *"element 0 of tensors does not require grad and does not have a grad_fn"*. + The executor reads the duck-typed ``__torch_runner__`` flag (no hard import on the + GUI side) and re-enables normal autograd for the duration of ``run()``. + + Inference-only runnables (a pure evaluator, a dataset processor) deliberately do + NOT inherit this — they run as-is under the executor's inference mode. + """ + + __torch_runner__: bool = True + + +class ProgressReporting: + """Mixin giving a runnable a framework-free progress callback. + + Pure mixin — no ``__init__``, no required state. The executor injects a sink via + :meth:`set_progress_callback`; the runnable drains it from its loop via + :meth:`_report_progress`. When no sink is injected (a plain CLI run, or any + non-GUI caller) every call is a silent no-op, so behaviour is unchanged. + """ + + #: Set by the executor via :meth:`set_progress_callback`; ``None`` ⇒ progress reporting is a no-op. + _progress_callback: Optional[ProgressCallback] = None + + def set_progress_callback(self, callback: Optional[ProgressCallback]) -> None: + """Inject (or clear, with ``None``) the progress sink the executor drains. Idempotent.""" + self._progress_callback = callback + + def _report_progress(self, value: float, total: Optional[float], desc: str = "") -> None: + """Report ``value`` of ``total`` to the injected callback; no-op when unset or invalid. + + Never raises — a broken progress sink must never abort a run (it is cosmetic). A ``None`` or + non-positive ``total`` (an unbounded / unknown length, e.g. a streaming source) is skipped so + the executor's bar stays in its indeterminate state rather than dividing by zero. + """ + callback = self._progress_callback + if callback is None or not total or total <= 0: + return + try: + callback(float(value), float(total), desc) + except Exception: # noqa: BLE001 - progress reporting must never break the run + logger.debug("progress callback raised; ignoring", exc_info=True) + + +def entrypoint(task: str, role: str = "runnable", primary: bool = False) -> Callable: + """Mark a runnable method as a named capability entry point. + + A class that drives several capabilities off one ``task`` knob (the merged train+eval + runnables: ``fit`` / ``evaluate`` / ``test`` / ``predict``) annotates each entry-point + method so a discovery consumer (a config generator, a visual editor) can learn — from + ONE class — which capabilities it exposes, instead of assuming a separate class per role. + + Args: + task: The ``task`` value the runnable's ``run()`` dispatches to for this method + (e.g. ``"fit"`` / ``"test"``). + role: A capability label — conventionally ``"trainer"`` (fits/trains), + ``"evaluator"`` (computes metrics over a held-out set), or ``"predictor"`` + (streams predictions). Free-form so new capabilities need no change here. + primary: When several methods share a ``role``, marks the default one (e.g. ``test`` + is the primary ``"evaluator"`` over ``evaluate``/validate). + """ + + def deco(fn: Callable) -> Callable: + setattr(fn, _ENTRYPOINT_ATTR, {"task": task, "role": role, "primary": primary}) + return fn + + return deco + + +def runnable_entrypoints(cls: type) -> Dict[str, Dict[str, object]]: + """Return ``{method_name: {"task", "role", "primary"}}`` for every :func:`entrypoint` method. + + Walks the MRO so an inherited entry point is found, and reads the marker off the raw + function object (never triggering a property getter). + """ + out: Dict[str, Dict[str, object]] = {} + for klass in reversed(cls.__mro__): + for name, attr in vars(klass).items(): + meta = getattr(attr, _ENTRYPOINT_ATTR, None) + if meta is not None: + out[name] = dict(meta) + return out + + +def entrypoint_tasks(cls: type, role: str) -> List[str]: + """Return the ``task`` values of ``cls``'s entry points with the given ``role``, primary first.""" + matches = [(name, meta) for name, meta in runnable_entrypoints(cls).items() if meta.get("role") == role] + matches.sort(key=lambda nm: not nm[1].get("primary", False)) # primary (True) sorts first + return [str(meta["task"]) for _, meta in matches] + + +__all__ = [ + "ProgressCallback", + "ProgressReporting", + "TorchRunner", + "entrypoint", + "entrypoint_tasks", + "runnable_entrypoints", +] diff --git a/sampleflux/workflow.py b/sampleflux/workflow.py new file mode 100644 index 0000000..16df38d --- /dev/null +++ b/sampleflux/workflow.py @@ -0,0 +1,270 @@ +"""Higher-order runnables — compose other runnables into a workflow. + +A *runnable* is any object exposing a no-arg ``run(self)`` (a trainer, an +evaluator, a :class:`~sampleflux.processing.DatasetProcessor`). This module adds +Confluid-``@configurable`` *combinators* that HOLD other runnables and orchestrate +them — the runnable-level analogue of the higher-order ops (``Parallel`` / +``TransformChain`` / ``Enable``): + +* :class:`Sequence` — run a list of runnables in order (the workflow itself). +* :class:`Conditional` — run one of two runnables depending on a condition. +* :class:`Switch` — run one of several runnables keyed by a selector value. + +Conditions are themselves Confluid-``@configurable`` *predicates* — a no-arg +``__call__(self) -> bool`` (:class:`PathExists` / :class:`Not` / :class:`AllOf` +/ :class:`AnyOf`) — so a whole workflow (steps, branches, AND the conditions +that pick them) serialises to ONE Confluid YAML document, runs via the generic +``sampleflux run workflow.yaml`` runner, and — being plain ``@configurable`` +classes — is surfaced by discovery / a visual editor with no bespoke glue. + +Branches are held as INSTANCES and only ``run()`` when selected. Example:: + + !class:sampleflux.workflow.Sequence + steps: + - !lazy:DownloadData + - !class:sampleflux.workflow.Conditional + condition: !class:sampleflux.workflow.PathExists { path: $MODEL_ROOT/model.ckpt } + if_false: !lazy:TrainModel # cache miss -> train + if_true: null # cache hit -> skip, fall through + - !lazy:Evaluate + +The guarantee: **only the selected branch's ``run()`` is ever called** — an +unchosen branch is never run (and, wired ``!lazy:``, never even built, so no +model / dataset is materialised). This is the *memoise-and-continue* answer to +"don't recompute, move on": the next ``steps:`` entry runs regardless, because +``Sequence`` drives them in order — no execution-blocking, no dead branches. + +All combinators are zero-arg constructible and do NO functional work in +``__init__`` (the workspace lazy-construction convention); branches and +conditions are flowed lazily inside ``run()`` (a Confluid ``!class:`` / ``!lazy:`` +member arrives as a deferred ``Fluid`` stub and is materialised on demand). + +The combinators inherit :class:`~sampleflux.runnable.TorchRunner` and +:class:`~sampleflux.runnable.ProgressReporting` so a workflow runs correctly on a +GUI canvas: a combinator may wrap a *trainer*, so it declares ``__torch_runner__`` +(the executor re-enables autograd for the whole run — otherwise an inner +``loss.backward()`` dies under the executor's inference mode; restoring autograd is +harmless for an inner evaluator), and it FORWARDS the executor-injected progress +callback to whichever branch is running (:func:`_run`). +""" + +from pathlib import Path +from typing import Any, Dict, List, Optional + +from confluid import configurable, flow +from confluid.fluid import Fluid +from loggair import get_logger + +from sampleflux.runnable import ProgressReporting, TorchRunner + +logger = get_logger(__name__) + + +def _resolve(value: Any) -> Any: + """Flow a possibly-deferred Confluid value to a live object (idempotent on live ones).""" + return flow(value) if isinstance(value, Fluid) else value + + +def _evaluate_condition(condition: Any) -> bool: + """Evaluate a workflow condition to a ``bool``. + + Accepts a ``@configurable`` predicate (no-arg ``__call__ -> bool``), any + zero-arg callable, a plain ``bool``, or a deferred ``Fluid`` resolving to one + of those. ``None`` is falsy. + """ + condition = _resolve(condition) + if condition is None: + return False + if callable(condition): + return bool(condition()) + return bool(condition) + + +def _run(runnable: Any, progress_callback: Any = None) -> None: + """Flow ``runnable`` (if deferred) and call its ``run()``; ``None`` is a no-op. + + When ``progress_callback`` is supplied (the combinator's OWN callback, injected by a GUI + executor via :meth:`ProgressReporting.set_progress_callback`) it is FORWARDED to the branch + first, so a canvas progress bar tracks whichever runnable is executing inside the workflow + (the combinator itself has no loop to report from). + """ + runnable = _resolve(runnable) + if runnable is None: + return + run = getattr(runnable, "run", None) + if not callable(run): + raise TypeError(f"workflow branch {type(runnable).__name__!r} has no callable run() method") + if progress_callback is not None: + setter = getattr(runnable, "set_progress_callback", None) + if callable(setter): + setter(progress_callback) + run() + + +@configurable +class Sequence(TorchRunner, ProgressReporting): + """Run a list of runnables in order — the workflow itself. + + Args: + steps: Runnables (or deferred ``!lazy:`` / ``!class:`` markers) to run in + order. A ``None`` entry is skipped. An empty list is a no-op. + """ + + def __init__(self, steps: Optional[List[Any]] = None) -> None: + # Lazy / zero-arg: store config only; branches are flowed in run(). + self.steps: List[Any] = list(steps) if steps else [] + + def run(self) -> None: + total = len(self.steps) + for i, step in enumerate(self.steps): + resolved = _resolve(step) + self.steps[i] = resolved # cache the flowed step so we only flow once + if resolved is None: + continue + logger.info(f"Sequence step {i + 1}/{total}: {type(resolved).__name__}") + _run(resolved, self._progress_callback) + + +@configurable +class Conditional(TorchRunner, ProgressReporting): + """Run one of two runnables depending on a condition. + + The runnable-level ``if``/``else``: a runnable that, on a condition, triggers + another runnable held as an instance. + + Args: + condition: A predicate (no-arg ``__call__ -> bool``), zero-arg callable, + ``bool``, or deferred ``Fluid`` resolving to one of those. ``None`` is + falsy. + if_true: Runnable to run when the condition holds. ``None`` = do nothing. + if_false: Runnable to run otherwise. ``None`` = do nothing (skip and let + an enclosing :class:`Sequence` continue to the next step). + """ + + def __init__(self, condition: Any = None, if_true: Any = None, if_false: Any = None) -> None: + self.condition = condition + self.if_true = if_true + self.if_false = if_false + + def run(self) -> None: + chosen = self.if_true if _evaluate_condition(self.condition) else self.if_false + branch = _resolve(chosen) + if branch is None: + logger.debug("Conditional: selected branch is None — nothing to run.") + return + logger.info(f"Conditional -> {type(branch).__name__}") + _run(branch, self._progress_callback) + + +@configurable +class Switch(TorchRunner, ProgressReporting): + """Run one of several runnables keyed by a selector's value. + + Args: + selector: A no-arg callable / predicate / deferred value producing the + case KEY (coerced to ``str``). ``None`` (or a ``None`` result) selects + ``default``. + cases: Mapping of key -> runnable. The runnable whose key matches the + selector runs; an unmatched key falls back to ``default``. + default: Runnable to run when no case matches. ``None`` = no-op. + """ + + def __init__( + self, + selector: Any = None, + cases: Optional[Dict[str, Any]] = None, + default: Any = None, + ) -> None: + self.selector = selector + self.cases: Dict[str, Any] = dict(cases) if cases else {} + self.default = default + + def run(self) -> None: + key = self._select() + chosen = self.cases.get(key, self.default) if key is not None else self.default + branch = _resolve(chosen) + if branch is None: + logger.debug(f"Switch: no branch for key {key!r} and no default — nothing to run.") + return + logger.info(f"Switch[{key!r}] -> {type(branch).__name__}") + _run(branch, self._progress_callback) + + def _select(self) -> Optional[str]: + selector = _resolve(self.selector) + if selector is None: + return None + value = selector() if callable(selector) else selector + return None if value is None else str(value) + + +@configurable +class PathExists: + """Predicate: ``True`` iff ``path`` exists on disk. + + The canonical cache check — pair with :class:`Conditional` to skip a step + whose output artifact (a checkpoint, a converted dataset) is already present. + + Args: + path: Filesystem path to test. Empty / ``None`` -> ``False``. + """ + + def __init__(self, path: str = "") -> None: + self.path = path + + def __call__(self) -> bool: + return bool(self.path) and Path(self.path).exists() + + +@configurable +class Not: + """Predicate: the negation of another condition. + + Args: + condition: The condition to negate (predicate / callable / ``bool`` / + ``Fluid``). ``None`` is falsy, so ``Not(None)`` is ``True``. + """ + + def __init__(self, condition: Any = None) -> None: + self.condition = condition + + def __call__(self) -> bool: + return not _evaluate_condition(self.condition) + + +@configurable +class AllOf: + """Predicate: ``True`` iff EVERY sub-condition is truthy (logical AND). + + An empty list is ``True`` (vacuous truth). + + Args: + conditions: Conditions to AND together (each a predicate / callable / + ``bool`` / ``Fluid``). + """ + + def __init__(self, conditions: Optional[List[Any]] = None) -> None: + self.conditions: List[Any] = list(conditions) if conditions else [] + + def __call__(self) -> bool: + return all(_evaluate_condition(c) for c in self.conditions) + + +@configurable +class AnyOf: + """Predicate: ``True`` iff at least ONE sub-condition is truthy (logical OR). + + An empty list is ``False``. + + Args: + conditions: Conditions to OR together (each a predicate / callable / + ``bool`` / ``Fluid``). + """ + + def __init__(self, conditions: Optional[List[Any]] = None) -> None: + self.conditions: List[Any] = list(conditions) if conditions else [] + + def __call__(self) -> bool: + return any(_evaluate_condition(c) for c in self.conditions) + + +__all__ = ["Sequence", "Conditional", "Switch", "PathExists", "Not", "AllOf", "AnyOf"] diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py new file mode 100644 index 0000000..9ca2e33 --- /dev/null +++ b/tests/test_cli_run.py @@ -0,0 +1,44 @@ +"""Tests for the `sampleflux run` CLI dispatch (sampleflux.cli.run).""" + +from typing import List + +from confluid import Class + +from sampleflux.cli import run + + +def test_run_calls_runnable_run() -> None: + calls: List[str] = [] + + class R: + def run(self) -> None: + calls.append("ran") + + run(R()) + assert calls == ["ran"] + + +def test_run_flows_deferred_marker() -> None: + from confluid import configurable + + log: List[str] = [] + + @configurable + class R: + def __init__(self, tag: str = "") -> None: + self.tag = tag + + def run(self) -> None: + log.append(self.tag) + + # A deferred Confluid marker is flowed before run() is called. + run(Class(R, tag="x")) + assert log == ["x"] + + +def test_run_none_is_noop() -> None: + run(None) # must not raise + + +def test_run_non_runnable_is_handled() -> None: + run(object()) # no .run() -> logged + returns, never raises diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py new file mode 100644 index 0000000..1a8da78 --- /dev/null +++ b/tests/test_entrypoint.py @@ -0,0 +1,56 @@ +"""Tests for the runnable entry-point marker (entrypoint / runnable_entrypoints).""" + +from sampleflux.runnable import entrypoint, entrypoint_tasks, runnable_entrypoints + + +class _Runnable: + @entrypoint("fit", role="trainer") + def fit(self) -> None: ... + + @entrypoint("evaluate", role="evaluator") + def evaluate(self) -> None: ... + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: ... + + @entrypoint("predict", role="predictor") + def predict(self) -> None: ... + + def not_an_entrypoint(self) -> None: ... + + +def test_runnable_entrypoints_finds_all_marked_methods() -> None: + eps = runnable_entrypoints(_Runnable) + assert set(eps) == {"fit", "evaluate", "test", "predict"} + assert eps["fit"] == {"task": "fit", "role": "trainer", "primary": False} + assert eps["test"]["primary"] is True + assert "not_an_entrypoint" not in eps + + +def test_entrypoint_tasks_by_role_primary_first() -> None: + assert entrypoint_tasks(_Runnable, "trainer") == ["fit"] + # test is primary → sorts first among evaluators. + assert entrypoint_tasks(_Runnable, "evaluator") == ["test", "evaluate"] + assert entrypoint_tasks(_Runnable, "predictor") == ["predict"] + assert entrypoint_tasks(_Runnable, "nonexistent") == [] + + +def test_entrypoints_are_inherited_via_mro() -> None: + class _Sub(_Runnable): + @entrypoint("fit", role="trainer") + def fit(self) -> None: ... # override keeps the marker + + eps = runnable_entrypoints(_Sub) + assert set(eps) == {"fit", "evaluate", "test", "predict"} + + +def test_marker_does_not_break_calling_the_method() -> None: + calls = [] + + class _R: + @entrypoint("fit", role="trainer") + def fit(self) -> None: + calls.append("fit") + + _R().fit() + assert calls == ["fit"] diff --git a/tests/test_processing.py b/tests/test_processing.py new file mode 100644 index 0000000..a3bf8c0 --- /dev/null +++ b/tests/test_processing.py @@ -0,0 +1,172 @@ +"""Tests for :class:`sampleflux.processing.DatasetProcessor` — progress-bar toggle.""" + +from typing import Any, Iterator, List +from unittest.mock import patch + +import confluid +import pytest +from confluid import configurable + +from sampleflux.core import Flux +from sampleflux.processing import DatasetProcessor +from sampleflux.sample import Sample + + +@configurable +class _SizedSource: + """Minimal Sized source yielding ``n`` trivial Samples.""" + + def __init__(self, n: int) -> None: + self.n = n + + def __len__(self) -> int: + return self.n + + def __iter__(self) -> Iterator[Sample]: + for i in range(self.n): + yield Sample(input=i, target=None, metadata={}) + + +@configurable +class _UnsizedSource: + """Source that supports __iter__ but not __len__ — e.g. a streaming glob.""" + + def __init__(self, n: int) -> None: + self.n = n + + def __iter__(self) -> Iterator[Sample]: + for i in range(self.n): + yield Sample(input=i, target=None, metadata={}) + + +class _RecordingSink: + """DataSink stand-in: records writes and flush, so we can assert order.""" + + def __init__(self) -> None: + self.written: List[Sample] = [] + self.flushed = False + + def write(self, sample: Sample) -> None: + self.written.append(sample) + + def flush(self) -> None: + self.flushed = True + + +def test_run_without_progress_uses_raw_flux() -> None: + flux = Flux(source=_SizedSource(3)) + sink = _RecordingSink() + with patch("sampleflux.processing.Progress") as mock_progress: + DatasetProcessor(flux=flux, sink=sink).run() + mock_progress.assert_not_called() + assert len(sink.written) == 3 + assert sink.flushed + + +def test_run_with_progress_sized_source_sets_total() -> None: + flux = Flux(source=_SizedSource(5)) + sink = _RecordingSink() + with patch("sampleflux.processing.Progress") as mock_progress: + DatasetProcessor(flux=flux, sink=sink, show_progress=True).run() + mock_progress.assert_called_once() + progress = mock_progress.return_value.__enter__.return_value + progress.add_task.assert_called_once() + call = progress.add_task.call_args + assert call.args[0] == "DatasetProcessor" + assert call.kwargs["total"] == 5 + assert len(sink.written) == 5 + + +def test_run_with_progress_unsized_source_falls_back_to_none() -> None: + flux = Flux(source=_UnsizedSource(4)) + sink = _RecordingSink() + with patch("sampleflux.processing.Progress") as mock_progress: + DatasetProcessor(flux=flux, sink=sink, show_progress=True).run() + progress = mock_progress.return_value.__enter__.return_value + progress.add_task.assert_called_once() + assert progress.add_task.call_args.kwargs["total"] is None + + +def test_progress_desc_overrides_default() -> None: + flux = Flux(source=_SizedSource(1)) + sink = _RecordingSink() + with patch("sampleflux.processing.Progress") as mock_progress: + DatasetProcessor(flux=flux, sink=sink, show_progress=True, progress_desc="my-run").run() + progress = mock_progress.return_value.__enter__.return_value + assert progress.add_task.call_args.args[0] == "my-run" + + +def test_no_sink_materializes_in_memory() -> None: + """Progress wrapper also exercised on the sinkless path.""" + flux = Flux(source=_SizedSource(2)) + with patch("sampleflux.processing.Progress") as mock_progress: + DatasetProcessor(flux=flux, show_progress=True).run() + progress = mock_progress.return_value.__enter__.return_value + assert progress.add_task.call_args.kwargs["total"] == 2 + + +def test_progress_bar_updates_per_sample() -> None: + """Asserts progress.update() fires exactly once per emitted sample.""" + flux = Flux(source=_SizedSource(3)) + sink = _RecordingSink() + with patch("sampleflux.processing.Progress") as mock_progress: + progress = mock_progress.return_value.__enter__.return_value + DatasetProcessor(flux=flux, sink=sink, show_progress=True).run() + assert progress.update.call_count == 3 + + +def test_yaml_roundtrip_preserves_progress_flags() -> None: + flux = Flux(source=_SizedSource(1)) + proc = DatasetProcessor(flux=flux, show_progress=True, progress_desc="from-yaml") + state = confluid.dump(proc) + restored: Any = confluid.load(state) + assert restored.show_progress is True + assert restored.progress_desc == "from-yaml" + + +def test_yaml_roundtrip_default_is_off() -> None: + proc = DatasetProcessor(flux=Flux(source=_SizedSource(1))) + restored: Any = confluid.load(confluid.dump(proc)) + assert restored.show_progress is False + assert restored.progress_desc is None + + +def test_progress_callback_fires_per_sample_without_console_bar() -> None: + """The executor's (FluxStudio) progress callback fires per sample even when show_progress is OFF. + + The native ComfyUI bar is independent of the rich console bar — set_progress_callback drives it + regardless of ``show_progress``. + """ + flux = Flux(source=_SizedSource(3)) + sink = _RecordingSink() + reports: List[tuple] = [] + proc = DatasetProcessor(flux=flux, sink=sink) # show_progress defaults to False + proc.set_progress_callback(lambda value, total, desc: reports.append((value, total, desc))) + proc.run() + # One report per emitted sample, with a monotonically increasing value and the sized total. + assert [v for v, _, _ in reports] == [1.0, 2.0, 3.0] + assert all(total == 3.0 for _, total, _ in reports) + assert all(desc == "DatasetProcessor" for *_, desc in reports) + + +def test_progress_callback_uses_progress_desc() -> None: + flux = Flux(source=_SizedSource(1)) + reports: List[tuple] = [] + proc = DatasetProcessor(flux=flux, progress_desc="my-run") # sinkless path + proc.set_progress_callback(lambda value, total, desc: reports.append((value, total, desc))) + proc.run() + assert reports == [(1.0, 1.0, "my-run")] + + +def test_progress_callback_noop_for_unsized_source() -> None: + """An unsized source has no total — the executor bar stays indeterminate (callback never fires).""" + flux = Flux(source=_UnsizedSource(4)) + reports: List[tuple] = [] + proc = DatasetProcessor(flux=flux) + proc.set_progress_callback(lambda value, total, desc: reports.append((value, total, desc))) + proc.run() + assert reports == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_runnable.py b/tests/test_runnable.py new file mode 100644 index 0000000..42b96ea --- /dev/null +++ b/tests/test_runnable.py @@ -0,0 +1,71 @@ +"""Tests for the runnable protocol markers (TorchRunner + ProgressReporting).""" + +from typing import List + +from sampleflux.runnable import ProgressReporting, TorchRunner + + +def test_torch_runner_flag() -> None: + assert TorchRunner.__torch_runner__ is True + + class Trainer(TorchRunner): + pass + + assert Trainer().__torch_runner__ is True # inherited by subclasses + + +def test_progress_reporting_noop_without_callback() -> None: + class R(ProgressReporting): + pass + + r = R() + # No callback set -> _report_progress is a silent no-op (never raises). + r._report_progress(1, 10, "x") + assert r._progress_callback is None + + +def test_progress_reporting_fires_when_set() -> None: + reports: List[tuple] = [] + + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(lambda v, t, d: reports.append((v, t, d))) + r._report_progress(3, 10, "step") + assert reports == [(3.0, 10.0, "step")] + + +def test_progress_reporting_skips_nonpositive_total() -> None: + reports: List[tuple] = [] + + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(lambda v, t, d: reports.append((v, t, d))) + r._report_progress(1, None, "x") # unknown total -> skipped + r._report_progress(1, 0, "x") # non-positive -> skipped + assert reports == [] + + +def test_progress_reporting_swallows_callback_errors() -> None: + def boom(v: float, t: float, d: str) -> None: + raise RuntimeError("sink broke") + + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(boom) + r._report_progress(1, 10, "x") # must not raise — progress is cosmetic + + +def test_set_progress_callback_clears_with_none() -> None: + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(lambda v, t, d: None) + r.set_progress_callback(None) + assert r._progress_callback is None diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..cd15162 --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,243 @@ +"""Tests for sampleflux.workflow — higher-order runnable combinators. + +Covers Sequence / Conditional / Switch orchestration, the PathExists / Not / +AllOf / AnyOf predicates, zero-arg construction (the lazy-construction mandate), +the only-the-selected-branch-runs guarantee, error paths, and a Confluid +dump/load round-trip that proves a whole workflow serialises and runs (the +deferred branches are flowed inside run()). +""" + +from typing import Any, Iterator, List + +import confluid +import pytest +from confluid import configurable + +from sampleflux.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch + +# A module-global run log so the @configurable runnables below survive a Confluid +# round-trip: their only config is a tag, and run() appends it here. +_RUN_LOG: List[str] = [] + + +@configurable +class _RunStep: + """A minimal runnable: ``run()`` records its tag in the module log.""" + + def __init__(self, tag: str = "") -> None: + self.tag = tag + + def run(self) -> None: + _RUN_LOG.append(self.tag) + + +@configurable +class _FixedPredicate: + """A predicate whose truth value is pinned in config (round-trippable).""" + + def __init__(self, value: bool = False) -> None: + self.value = value + + def __call__(self) -> bool: + return self.value + + +@pytest.fixture(autouse=True) +def _clear_log() -> Iterator[None]: + _RUN_LOG.clear() + yield + _RUN_LOG.clear() + + +# --------------------------------------------------------------------------- # +# Sequence +# --------------------------------------------------------------------------- # +def test_sequence_runs_steps_in_order() -> None: + Sequence([_RunStep("a"), _RunStep("b"), _RunStep("c")]).run() + assert _RUN_LOG == ["a", "b", "c"] + + +def test_sequence_empty_is_noop() -> None: + Sequence().run() # zero-arg + empty: must not raise + assert _RUN_LOG == [] + + +def test_sequence_skips_none_entries() -> None: + Sequence([_RunStep("a"), None, _RunStep("b")]).run() + assert _RUN_LOG == ["a", "b"] + + +def test_sequence_caches_flowed_step() -> None: + seq = Sequence([confluid.Class(_RunStep, tag="x")]) + seq.run() + # After run, the deferred marker has been replaced by the live, flowed object. + assert isinstance(seq.steps[0], _RunStep) + assert _RUN_LOG == ["x"] + + +# --------------------------------------------------------------------------- # +# Conditional +# --------------------------------------------------------------------------- # +def test_conditional_true_runs_if_true_only() -> None: + Conditional(condition=True, if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["t"] + + +def test_conditional_false_runs_if_false_only() -> None: + Conditional(condition=False, if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["f"] + + +def test_conditional_none_branch_is_noop() -> None: + Conditional(condition=True, if_true=None, if_false=_RunStep("f")).run() + assert _RUN_LOG == [] + + +def test_conditional_none_condition_is_falsy() -> None: + Conditional(if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["f"] + + +def test_conditional_accepts_callable_condition() -> None: + Conditional(condition=lambda: True, if_true=_RunStep("t")).run() + assert _RUN_LOG == ["t"] + + +def test_conditional_accepts_predicate_condition() -> None: + Conditional(condition=_FixedPredicate(True), if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["t"] + + +# --------------------------------------------------------------------------- # +# Switch +# --------------------------------------------------------------------------- # +def test_switch_selects_matching_case() -> None: + Switch(selector=lambda: "b", cases={"a": _RunStep("a"), "b": _RunStep("b")}).run() + assert _RUN_LOG == ["b"] + + +def test_switch_falls_back_to_default_on_miss() -> None: + Switch(selector=lambda: "z", cases={"a": _RunStep("a")}, default=_RunStep("d")).run() + assert _RUN_LOG == ["d"] + + +def test_switch_none_selector_uses_default() -> None: + Switch(default=_RunStep("d")).run() + assert _RUN_LOG == ["d"] + + +def test_switch_no_match_no_default_is_noop() -> None: + Switch(selector=lambda: "z", cases={"a": _RunStep("a")}).run() + assert _RUN_LOG == [] + + +def test_switch_coerces_non_string_key() -> None: + Switch(selector=lambda: 2, cases={"2": _RunStep("two")}).run() + assert _RUN_LOG == ["two"] + + +# --------------------------------------------------------------------------- # +# Predicates +# --------------------------------------------------------------------------- # +def test_path_exists(tmp_path: Any) -> None: + present = tmp_path / "model.ckpt" + present.write_text("x") + assert PathExists(str(present))() is True + assert PathExists(str(tmp_path / "missing.ckpt"))() is False + assert PathExists("")() is False # empty path never True (no bogus Path('.') hit) + + +def test_not_negates() -> None: + assert Not(True)() is False + assert Not(False)() is True + assert Not(None)() is True # None is falsy -> Not(None) is True + + +def test_allof_is_and_with_vacuous_truth() -> None: + assert AllOf([True, True])() is True + assert AllOf([True, False])() is False + assert AllOf()() is True # empty AND is True + + +def test_anyof_is_or() -> None: + assert AnyOf([False, True])() is True + assert AnyOf([False, False])() is False + assert AnyOf()() is False # empty OR is False + + +def test_predicate_combinators_compose() -> None: + # Not(AllOf(True, AnyOf(False, True))) == not (True and True) == False + assert Not(AllOf([True, AnyOf([False, True])]))() is False + + +# --------------------------------------------------------------------------- # +# Error paths + zero-arg construction +# --------------------------------------------------------------------------- # +def test_run_non_runnable_branch_raises() -> None: + with pytest.raises(TypeError, match="has no callable run"): + Conditional(condition=True, if_true=object()).run() + + +@pytest.mark.parametrize("cls", [Sequence, Conditional, Switch]) +def test_combinator_zero_arg_construct_and_run(cls: Any) -> None: + cls().run() # must construct AND run with no args (no-op), never raise + assert _RUN_LOG == [] + + +@pytest.mark.parametrize("cls", [PathExists, Not, AllOf, AnyOf]) +def test_predicate_zero_arg_construct_and_call(cls: Any) -> None: + assert cls()() in (True, False) # zero-arg construct + call yields a bool + + +# --------------------------------------------------------------------------- # +# FluxStudio canvas integration — TorchRunner + ProgressReporting forwarding +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("cls", [Sequence, Conditional, Switch]) +def test_combinators_declare_torch_runner(cls: Any) -> None: + # A combinator may wrap a trainer, so it declares __torch_runner__ — FluxStudio's executor + # re-enables autograd for the whole run (otherwise the inner loss.backward() dies under + # ComfyUI's inference_mode). + assert cls().__torch_runner__ is True + + +def test_progress_callback_forwarded_to_running_branch() -> None: + from sampleflux.runnable import ProgressReporting + + received: List[str] = [] + + @configurable + class _ProgressStep(ProgressReporting): + def run(self) -> None: + self._report_progress(1, 1, "step") # fires through whatever callback was forwarded + + seq = Sequence([_ProgressStep()]) + seq.set_progress_callback(lambda v, t, d: received.append(d)) # executor injects on the combinator + seq.run() + assert received == ["step"] # forwarded to the branch, which reported through it + + +def test_progress_forward_is_safe_when_branch_has_no_callback() -> None: + seq = Sequence([_RunStep("a")]) + seq.set_progress_callback(lambda v, t, d: None) + seq.run() # _RunStep has no set_progress_callback -> _run must not raise + assert _RUN_LOG == ["a"] + + +# --------------------------------------------------------------------------- # +# Confluid round-trip — a whole workflow serialises and runs +# --------------------------------------------------------------------------- # +def test_confluid_roundtrip_runs_nested_workflow() -> None: + workflow = Sequence( + [ + _RunStep("download"), + Conditional( + condition=_FixedPredicate(False), # cache miss -> train branch + if_true=_RunStep("skip"), + if_false=_RunStep("train"), + ), + _RunStep("evaluate"), + ] + ) + restored: Any = confluid.load(confluid.dump(workflow)) + restored.run() + assert _RUN_LOG == ["download", "train", "evaluate"] From cdbe0f40fa6e1efec2d776d69e2cdc271200c748 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 09:06:57 +0200 Subject: [PATCH 033/102] docs: add runnable-protocol / entrypoint / sampleflux-run mandate --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 92b9c25..da710b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,6 @@ # SampleFlux Mandates +- **The Runnable Protocol Lives Here (`sampleflux.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** sampleflux owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `sampleflux.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `sampleflux.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `sampleflux.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `flux` validated in `run()`). `sampleflux.cli`: the `sampleflux run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `sampleflux-processing`/`sampleflux-workflow` + the `sampleflux` console script + `liquifai.apps`. - **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases). `Tee` threaded the sample through its branches sequentially, making it executionally identical to `TransformChain(ops=[...])` — use `TransformChain` for grouping and the context ops (`Save`/`Use`/`Mix`) for real, isolated fan-out. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom `ConfigureOp(ops=[UnstashInputOp(key)])` is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-sample side-branch (`ops` chain → `metadata[key]` + setattr) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters (fluxstudio export.py AND graphio.py) emit ONLY context ops for wiring; graphio's legacy `__taidal_stash_*` import replay was removed (pre-2026-07 stash-format ops-docs no longer import — re-export from the canvas). Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. **Scope (2026-07-21):** this mandate governs the CLASSIC engine (`sampleflux.core` / `sampleflux.ops` / `sampleflux.sample`). The experimental typed-bag redesign `sampleflux.bag` (a coexisting proof of concept — see [[typed-bag-model]] below and `docs/architecture.md`) DELIBERATELY introduces a `Transform` base + typed item classes; it does not relax this rule for the classic engine, whose ops stay plain callables. From 41be4bcf0fc7198aa55d9782e156e0bc944298b9 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 10:52:59 +0200 Subject: [PATCH 034/102] =?UTF-8?q?refactor(sampleflux)!:=20Stage=207=20?= =?UTF-8?q?=E2=80=94=20TypedSample=20becomes=20THE=20Sample;=20purge=20leg?= =?UTF-8?q?acy=20triple=20engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The typed-bag carrier (formerly TypedSample) is now `sampleflux.Sample`; the legacy Sample(input,target,metadata) NamedTuple, the triple views (Pair/InputMeta/TargetMeta), kinds.py, typespec.py (ACCEPTS/PRODUCES), bag/interop.py, and the legacy *Op twins are DELETED. The classic engine (core/collate/flow/context/projection/storage/ops) is rewritten typed-only; each deleted legacy op's math is extracted to a module-level helper the typed Transform twin now calls directly. 561 passed, mypy clean, flake8 clean. BREAKING: `from sampleflux import Sample` is the typed bag; `from sampleflux.sample import Sample` and TypedSample are gone. Downstream packages are renamed in the following waves. --- examples/advanced_storage_demo.py | 45 - examples/augmentation_ops.py | 140 --- examples/augmentation_training.py | 129 -- examples/basic_pipeline.py | 59 - examples/discovery_demo.py | 24 +- examples/flow_graph.py | 84 -- examples/hdf5_pipeline.py | 51 - examples/parallel_hdf5_stream.py | 50 - examples/parallel_pipeline.py | 45 - examples/storage_roundtrip.py | 64 - examples/typed_pipeline.py | 20 +- pyproject.toml | 13 +- sampleflux/__init__.py | 109 +- sampleflux/bag/__init__.py | 8 +- sampleflux/bag/adapters/albumentations.py | 6 +- sampleflux/bag/adapters/torchvision.py | 6 +- sampleflux/bag/interop.py | 63 - sampleflux/bag/io.py | 10 +- sampleflux/bag/sample.py | 48 +- sampleflux/bag/transform.py | 12 +- sampleflux/collate.py | 94 +- sampleflux/core.py | 528 ++------- sampleflux/flow.py | 196 +--- sampleflux/kinds.py | 333 ------ sampleflux/labels.py | 14 +- sampleflux/ops/__init__.py | 95 +- sampleflux/ops/albumentations.py | 51 +- sampleflux/ops/configure.py | 7 +- sampleflux/ops/context.py | 103 +- sampleflux/ops/copy.py | 55 - sampleflux/ops/debug.py | 15 +- sampleflux/ops/enable.py | 2 +- sampleflux/ops/formula.py | 8 +- sampleflux/ops/image.py | 187 +-- sampleflux/ops/metadata.py | 51 - sampleflux/ops/numpy.py | 710 ++--------- sampleflux/ops/parallel.py | 2 +- sampleflux/ops/random_apply.py | 2 +- sampleflux/ops/sink.py | 2 +- sampleflux/ops/stash.py | 140 --- sampleflux/ops/structure.py | 18 +- sampleflux/ops/swap.py | 17 - sampleflux/ops/target.py | 537 +++------ sampleflux/ops/torch.py | 265 +---- sampleflux/ops/torchvision.py | 39 +- sampleflux/ops/transform_chain.py | 2 +- sampleflux/projection.py | 91 +- sampleflux/sample.py | 175 --- sampleflux/sources.py | 32 +- sampleflux/storage/base.py | 7 +- sampleflux/storage/directory.py | 43 +- sampleflux/storage/hdf5.py | 94 +- sampleflux/storage/query.py | 11 +- sampleflux/storage/zarr.py | 127 +- sampleflux/typespec.py | 937 --------------- tests/_bag_fixtures.py | 6 +- tests/test_augment_ops.py | 312 ----- tests/test_bag_interop.py | 77 -- tests/test_bag_io.py | 4 +- tests/test_bag_pipeline.py | 20 +- tests/test_bag_sample.py | 30 +- tests/test_bag_transform.py | 20 +- tests/test_categories.py | 170 ++- tests/test_context.py | 442 ------- tests/test_coverage_gap.py | 189 --- tests/test_enable.py | 265 ----- tests/test_expanding_ops.py | 182 --- tests/test_flow.py | 396 ------- tests/test_flux.py | 263 ----- tests/test_from_ops_yaml.py | 46 - tests/test_image_ops.py | 499 -------- tests/test_joint.py | 107 -- tests/test_kinds.py | 580 --------- tests/test_labels.py | 20 +- tests/test_node_docs.py | 39 +- tests/test_ops.py | 1292 --------------------- tests/test_parallel.py | 16 +- tests/test_parallel_op.py | 124 -- tests/test_processing.py | 172 --- tests/test_projection.py | 218 ---- tests/test_query.py | 78 -- tests/test_random_apply.py | 121 -- tests/test_sample.py | 87 -- tests/test_sources.py | 563 --------- tests/test_storage.py | 263 ----- tests/test_structure_ops.py | 16 +- tests/test_target_ops.py | 216 ---- tests/test_transform_chain.py | 157 --- tests/test_typed_collate.py | 23 +- tests/test_typed_detection_target_ops.py | 83 +- tests/test_typed_flow.py | 36 +- tests/test_typed_generic_ops.py | 97 +- tests/test_typed_storage.py | 51 +- tests/test_typed_target_ops.py | 93 +- tests/test_typespec.py | 658 ----------- 95 files changed, 1162 insertions(+), 12845 deletions(-) delete mode 100644 examples/advanced_storage_demo.py delete mode 100644 examples/augmentation_ops.py delete mode 100644 examples/augmentation_training.py delete mode 100644 examples/basic_pipeline.py delete mode 100644 examples/flow_graph.py delete mode 100644 examples/hdf5_pipeline.py delete mode 100644 examples/parallel_hdf5_stream.py delete mode 100644 examples/parallel_pipeline.py delete mode 100644 examples/storage_roundtrip.py delete mode 100644 sampleflux/bag/interop.py delete mode 100644 sampleflux/kinds.py delete mode 100644 sampleflux/ops/copy.py delete mode 100644 sampleflux/ops/metadata.py delete mode 100644 sampleflux/ops/stash.py delete mode 100644 sampleflux/ops/swap.py delete mode 100644 sampleflux/sample.py delete mode 100644 sampleflux/typespec.py delete mode 100644 tests/test_augment_ops.py delete mode 100644 tests/test_bag_interop.py delete mode 100644 tests/test_context.py delete mode 100644 tests/test_coverage_gap.py delete mode 100644 tests/test_enable.py delete mode 100644 tests/test_expanding_ops.py delete mode 100644 tests/test_flow.py delete mode 100644 tests/test_flux.py delete mode 100644 tests/test_from_ops_yaml.py delete mode 100644 tests/test_image_ops.py delete mode 100644 tests/test_joint.py delete mode 100644 tests/test_kinds.py delete mode 100644 tests/test_ops.py delete mode 100644 tests/test_parallel_op.py delete mode 100644 tests/test_processing.py delete mode 100644 tests/test_projection.py delete mode 100644 tests/test_query.py delete mode 100644 tests/test_random_apply.py delete mode 100644 tests/test_sample.py delete mode 100644 tests/test_sources.py delete mode 100644 tests/test_storage.py delete mode 100644 tests/test_target_ops.py delete mode 100644 tests/test_transform_chain.py delete mode 100644 tests/test_typespec.py diff --git a/examples/advanced_storage_demo.py b/examples/advanced_storage_demo.py deleted file mode 100644 index e0df7fa..0000000 --- a/examples/advanced_storage_demo.py +++ /dev/null @@ -1,45 +0,0 @@ -from pathlib import Path - -import numpy as np - -from sampleflux.core import Flux -from sampleflux.sample import Sample -from sampleflux.storage.directory import DirectorySink -from sampleflux.storage.zarr import ZarrBatchSink, ZarrGroupSink - - -def main() -> None: - # --- 1. DirectorySink (Irregular Lengths) --- - print("--- Testing DirectorySink ---") - dir_path = Path("examples/dir_store") - samples = [ - Sample(input=np.random.randn(5), metadata={"id": "tiny"}), - Sample(input=np.random.randn(50), metadata={"id": "large"}), - ] - - sink_dir = DirectorySink(dir_path, overwrite=True) - Flux(samples).to_sink(sink_dir) - print(f"Created {len(list(dir_path.glob('*')))} sample folders.") - - # --- 2. ZarrGroupSink (Irregular Lengths, Single Bundle) --- - print("\n--- Testing ZarrGroupSink ---") - zarr_group_path = "examples/group_store.zarr" - sink_group = ZarrGroupSink(zarr_group_path, overwrite=True) - Flux(samples).to_sink(sink_group) - print("Zarr group write complete.") - - # --- 3. ZarrBatchSink (Uniform Lengths, Optimized) --- - print("\n--- Testing ZarrBatchSink ---") - zarr_batch_path = "examples/batch_store.zarr" - # Uniform samples: 10 elements each - uniform_samples = [Sample(input=np.random.randn(10).astype(np.float32)) for _ in range(10)] - - sink_batch = ZarrBatchSink(zarr_batch_path, shape=[10], overwrite=True) - Flux(uniform_samples).to_sink(sink_batch) - print("Zarr batch append complete.") - - print("\nAll advanced storage sinks verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/augmentation_ops.py b/examples/augmentation_ops.py deleted file mode 100644 index 2cbd80b..0000000 --- a/examples/augmentation_ops.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Augmentation adapters: well-known libraries as SampleFlux ops, for input AND target. - -Demonstrates the library-augmentation surface: -1. input-only augmentation — ``AlbumentationsOp`` flips the image, target untouched; -2. joint input+target — ``target="mask"`` flips image AND segmentation mask consistently - (one library draw, metadata preserved); -3. the same joint flip through torchvision ``transforms.v2`` — cross-library parity; -4. detection boxes — ``MasksToDetectionBoxesOp`` derives xyxy boxes from the mask, then - both adapters transform image AND boxes jointly (``target="boxes"``, bbox_params - added automatically); -5. target-side augmentation/encoding — ``MetadataToTargetOp`` + ``EncodeTargetOp`` turn a - raw metadata label into the supervised class id; -6. the GENERATED per-transform ops — every library transform is its own op - (``AlbHorizontalFlip``, ``TvRandomHorizontalFlip``, …) chaining like any other op; -7. stochastic composition — ``TransformChain(RandomApply(AlbRandomBrightnessContrast))`` - gated per sample; -8. Confluid-NATIVE YAML — nested ``!class:albumentations.HorizontalFlip`` nodes and the - registered short names (``!class:AlbHorizontalFlip``), with dump→load parity. - -Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). -""" - -import albumentations as A -import confluid # type: ignore[import-not-found] -import numpy as np -import torch -from torchvision.transforms import v2 - -from sampleflux import Flux, Sample -from sampleflux.ops.albumentations import AlbumentationsOp -from sampleflux.ops.albumentations_transforms import AlbHorizontalFlip, AlbRandomBrightnessContrast -from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.target import EncodeTargetOp, MasksToDetectionBoxesOp, MetadataToTargetOp -from sampleflux.ops.torch import ToTensorOp -from sampleflux.ops.torchvision import TorchvisionTransformOp -from sampleflux.ops.torchvision_transforms import TvRandomHorizontalFlip -from sampleflux.ops.transform_chain import TransformChain - - -def make_sample() -> Sample: - """A deterministic 48x48 RGB gradient with a bright square and its binary mask.""" - height = width = 48 - image = np.linspace(0, 200, height * width * 3, dtype=np.float64).reshape(height, width, 3) - image = image.astype(np.uint8) - mask = np.zeros((height, width), dtype=np.uint8) - image[8:20, 4:16] = 255 # bright square, deliberately OFF-center so a flip moves it - mask[8:20, 4:16] = 1 - return Sample(image, mask, {"label": "square", "idx": 0}) - - -def main() -> None: - sample = make_sample() - image, mask = sample.input, sample.target - - # 1. Input-only augmentation: the target and metadata pass through untouched. - out = list(Flux(source=[sample], ops=[AlbumentationsOp(A.HorizontalFlip(p=1.0))]))[0] - assert np.array_equal(out.input, image[:, ::-1]) - assert np.array_equal(out.target, mask) - assert out.meta == sample.meta - print("1. albumentations input-only: image flipped, mask + metadata untouched") - - # 2. Joint input+target: ONE random draw moves image AND mask together; metadata - # survives verbatim. - out_alb = list(Flux(source=[sample], ops=[AlbumentationsOp(A.HorizontalFlip(p=1.0), target="mask")]))[0] - assert np.array_equal(out_alb.input, image[:, ::-1]) - assert np.array_equal(out_alb.target, mask[:, ::-1]) - assert out_alb.meta == sample.meta - print("2. albumentations target='mask': image AND mask flipped consistently") - - # 3. Same augmentation via torchvision transforms.v2 — identical pixels, different - # layout contract (torchvision emits CHW tensors; albumentations stays HWC numpy). - tv_op = TorchvisionTransformOp(v2.RandomHorizontalFlip(p=1.0), target="mask") - out_tv = list(Flux(source=[sample], ops=[tv_op]))[0] - assert np.array_equal(out_tv.input.permute(1, 2, 0).numpy(), out_alb.input) - assert np.array_equal(out_tv.target.numpy(), out_alb.target) - print("3. torchvision target='mask': cross-library parity (CHW tensor out)") - - # 4. Detection boxes: derive {"boxes" xyxy, "labels"} from the mask, then flip image - # AND boxes jointly. The adapter adds the required bbox_params automatically. - det = MasksToDetectionBoxesOp()(sample) - (x0, y0, x1, y1) = det.target["boxes"][0].tolist() - mirrored = [det.input.shape[1] - x1, y0, det.input.shape[1] - x0, y1] - out_tvb = list(Flux(source=[det], ops=[TorchvisionTransformOp(v2.RandomHorizontalFlip(p=1.0), target="boxes")]))[0] - assert out_tvb.target["boxes"][0].tolist() == mirrored - out_albb = list(Flux(source=[det], ops=[AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="boxes")]))[0] - assert np.allclose(out_albb.target["boxes"][0].tolist(), mirrored, atol=1e-4) - print(f"4. target='boxes': {[x0, y0, x1, y1]} -> {mirrored} (both libraries agree)") - - # 5. Target-side augmentation/encoding: raw label from metadata -> supervised class id. - encode = [MetadataToTargetOp(key="label"), EncodeTargetOp(mapping={"square": 0, "disc": 1})] - out_enc = list(Flux(source=[sample], ops=encode))[0] - assert out_enc.target == 0 - print("5. MetadataToTargetOp + EncodeTargetOp: metadata['label'] -> class id 0") - - # 6. Generated per-transform ops: every library transform is its OWN op — no wrapper - # boilerplate, the transform's params are the op's params, and it chains anywhere. - out_gen = list(Flux(source=[sample], ops=[AlbHorizontalFlip(p=1.0, target="mask")]))[0] - assert np.array_equal(out_gen.target, mask[:, ::-1]) - out_gen_tv = list(Flux(source=[sample], ops=[TvRandomHorizontalFlip(p=1.0, target="mask")]))[0] - assert np.array_equal(out_gen_tv.target.numpy(), mask[:, ::-1]) - print("6. generated ops: AlbHorizontalFlip / TvRandomHorizontalFlip chain like any op") - - # 7. Stochastic composition: gate a generated photometric op per sample, tensorize. - chain = TransformChain( - ops=[ - RandomApply(op=AlbHorizontalFlip(p=1.0, target="mask"), probability=0.5, random_state=0), - AlbRandomBrightnessContrast(p=1.0, seed=0), - ToTensorOp(), - ] - ) - source = [Sample(image.copy(), mask.copy(), {"idx": i}) for i in range(8)] - outputs = list(Flux(source=source, ops=[chain])) - flipped = sum(1 for s in outputs if np.array_equal(s.target, mask[:, ::-1])) - assert all(isinstance(s.input, torch.Tensor) and s.input.shape == (3, 48, 48) for s in outputs) - assert 0 < flipped < len(outputs) # the gate fired for some samples, not all - print(f"7. TransformChain(RandomApply(flip), brightness, ToTensorOp): {flipped}/{len(outputs)} flipped") - - # 8. Confluid-NATIVE YAML: transforms are nested !class: nodes (or registered short - # names) — dump and load round-trip with identical behavior. - yaml_text = ( - "!class:sampleflux.ops.albumentations.AlbumentationsOp\n" - "target: mask\n" - "seed: 0\n" - "transforms:\n" - " - !class:albumentations.HorizontalFlip\n" - " p: 1.0\n" - ) - op = confluid.load(yaml_text) - short = confluid.load("!class:AlbHorizontalFlip\np: 1.0\ntarget: mask\n") - out_yaml = op(sample) - out_short = short(sample) - assert np.array_equal(out_yaml.target, out_short.target) - reloaded = confluid.load(confluid.dump(op)) - assert np.array_equal(reloaded(sample).input, out_yaml.input) - print("8. Confluid-native YAML (nested !class: + short names), dump->load parity:") - print(" " + "\n ".join(confluid.dump(op).strip().splitlines())) - - -if __name__ == "__main__": - main() diff --git a/examples/augmentation_training.py b/examples/augmentation_training.py deleted file mode 100644 index 8bc9420..0000000 --- a/examples/augmentation_training.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Train a tiny segmentation CNN on an augmented SampleFlux pipeline (end to end). - -Demonstrates the full "augment to train" story: -1. a synthetic, network-free dataset — 64 RGB images with a bright square or disc and - its binary segmentation mask (input AND target); -2. joint geometric augmentation — ``AlbumentationsOp(target="mask")`` flips/translates - image AND mask with one library draw per sample; -3. gated photometric augmentation — ``RandomApply`` fires brightness/contrast on the - input only, for half the samples; -4. tensorization — ``ToTensorOp`` for the image, a raw-callable ``.map(select="target")`` - for the mask (``WrappedOp`` under the hood); -5. ``Flux`` is a ``torch.utils.data.Dataset`` — it plugs straight into a ``DataLoader`` - with the registry collate (``get_collate("sample")``: stacked tensors + list-form - batched metadata), and augmentations re-draw every epoch via random access; -6. a 3-epoch training loop of a tiny CNN (BCE on the mask) — losses must be finite and - improve, proving gradients flow through the augmented pipeline. - -Standalone, zero-arg, exit 0, seconds on CPU (CI runs every ``examples/*.py``). -""" - -import math -from typing import List - -import albumentations as A -import numpy as np -import torch -from torch import nn -from torch.utils.data import DataLoader - -from sampleflux import Flux, Sample -from sampleflux.collate import get_collate -from sampleflux.ops.albumentations import AlbumentationsOp -from sampleflux.ops.albumentations_transforms import AlbRandomBrightnessContrast -from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.torch import ToTensorOp - -SIZE = 32 # image edge in pixels - - -def make_dataset(count: int = 64, seed: int = 0) -> List[Sample]: - """Synthetic segmentation set: noisy background + one bright square or disc + its mask.""" - rng = np.random.default_rng(seed) - samples = [] - for idx in range(count): - image = rng.integers(0, 60, size=(SIZE, SIZE, 3), dtype=np.uint8) - mask = np.zeros((SIZE, SIZE), dtype=np.uint8) - cy, cx = rng.integers(8, SIZE - 8, size=2) - r = int(rng.integers(3, 7)) - shape = "square" if idx % 2 == 0 else "disc" - if shape == "square": - region = np.zeros((SIZE, SIZE), dtype=bool) - region[cy - r : cy + r, cx - r : cx + r] = True - else: - yy, xx = np.ogrid[:SIZE, :SIZE] - region = (yy - cy) ** 2 + (xx - cx) ** 2 <= r**2 - image[region] = rng.integers(180, 255, size=3, dtype=np.uint8) - mask[region] = 1 - samples.append(Sample(image, mask, {"idx": idx, "shape": shape})) - return samples - - -def mask_to_float(mask: np.ndarray) -> torch.Tensor: - """Binary HxW mask -> float32 (1, H, W) tensor, the shape BCEWithLogitsLoss expects.""" - return torch.from_numpy(np.ascontiguousarray(mask)).float().unsqueeze(0) - - -def build_pipeline(samples: List[Sample]) -> Flux: - """Source -> joint geometric aug -> gated photometric aug -> tensors, all seeded.""" - geometric = AlbumentationsOp( # image AND mask move together (one draw per sample) - transforms=[A.HorizontalFlip(p=0.5), A.Affine(translate_percent=0.1, p=1.0)], - target="mask", - seed=0, - ) - photometric = AlbRandomBrightnessContrast(p=1.0, seed=1) # generated per-transform op - flux = Flux( - source=samples, - ops=[ - geometric, - RandomApply(op=photometric, probability=0.5, random_state=0), - ToTensorOp(), # image -> float CHW in [0, 1] - ], - ) - return flux.map(mask_to_float, select="target") # raw-callable target map (WrappedOp) - - -def main() -> None: - torch.manual_seed(0) - samples = make_dataset() - flux = build_pipeline(samples) - - # Flux implements the torch Dataset protocol; the registry collate stacks - # input/target and keeps per-sample metadata as a list (Sample.is_batched). - loader = DataLoader(flux, batch_size=8, shuffle=True, collate_fn=get_collate("sample")) - - batch = next(iter(loader)) - assert batch.input.shape == (8, 3, SIZE, SIZE) and batch.input.dtype == torch.float32 - assert batch.target.shape == (8, 1, SIZE, SIZE) and batch.target.dtype == torch.float32 - assert batch.is_batched and len(batch.batch_meta) == 8 - print(f"batch: input {tuple(batch.input.shape)}, target {tuple(batch.target.shape)}") - print(f" metadata (first 3): {batch.batch_meta[:3]}") - - model = nn.Sequential( - nn.Conv2d(3, 8, kernel_size=3, padding=1), - nn.ReLU(), - nn.Conv2d(8, 1, kernel_size=3, padding=1), - ) - optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) - criterion = nn.BCEWithLogitsLoss() - - epoch_means = [] - for epoch in range(3): - losses = [] - for batch in loader: # augmentations re-draw here: each epoch sees new variants - optimizer.zero_grad() - loss = criterion(model(batch.input), batch.target) - loss.backward() - optimizer.step() - losses.append(float(loss.detach())) - mean = sum(losses) / len(losses) - epoch_means.append(mean) - print(f"epoch {epoch}: mean loss {mean:.4f}") - - assert all(math.isfinite(v) for v in epoch_means) - assert epoch_means[-1] < epoch_means[0], f"loss did not improve: {epoch_means}" - print(f"loss improved {epoch_means[0]:.4f} -> {epoch_means[-1]:.4f} on augmented data") - - -if __name__ == "__main__": - main() diff --git a/examples/basic_pipeline.py b/examples/basic_pipeline.py deleted file mode 100644 index 41c3d21..0000000 --- a/examples/basic_pipeline.py +++ /dev/null @@ -1,59 +0,0 @@ -import confluid # type: ignore[import-not-found] -import numpy as np - -from sampleflux.core import Flux - - -# 1. Define simple functional transformations -def add_noise(data: np.ndarray, std: float = 0.1) -> np.ndarray: - noise = np.random.normal(0, std, data.shape) - return np.asarray(data + noise) - - -def multiply(data: np.ndarray, factor: float = 2.0) -> np.ndarray: - return data * factor - - -def main() -> None: - # 2. Create raw data source - raw_data = [(np.array([1.0, 2.0]), 1), (np.array([3.0, 4.0]), 0)] - - # 3. Build the Flux pipeline - # For robust serialization, we should use strings for function references - # or ensure the classes/functions are part of the registry. - pipeline = Flux(raw_data).map(multiply, factor=10.0).map(add_noise, std=0.01) - - # 4. Serialize the Pipeline - # We set source=None before serialization to only serialize the "recipe" - # and avoid serializing raw numpy data which is not YAML-safe. - print("\n--- Serialized SampleFlux Pipeline ---") - yaml_state = "" - try: - pipeline.source = None - yaml_state = confluid.dump(pipeline) - print(yaml_state) - except Exception as e: - print(f"Serialization failed: {e}") - - # 5. Reconstruct and Execute - print("\n--- Reconstructing Pipeline from YAML ---") - try: - if yaml_state: - # Explicitly pass as YAML string (containing \n ensures - # load treats it as YAML) or ensure it's handled by path vs yaml logic. - new_pipeline = confluid.load(yaml_state) - - new_pipeline.source = raw_data - for sample in new_pipeline: - print(f"Reconstructed Output: {sample.input}") - else: - print("No YAML state to reconstruct.") - except Exception as e: - print(f"Reconstruction failed: {e}") - import traceback - - traceback.print_exc() - - -if __name__ == "__main__": - main() diff --git a/examples/discovery_demo.py b/examples/discovery_demo.py index 7541e46..81ea845 100644 --- a/examples/discovery_demo.py +++ b/examples/discovery_demo.py @@ -1,25 +1,23 @@ +"""Passive introspection: scan a module and get a JSON schema per callable (no manual tool defs). + +Standalone, zero-arg, exit 0. +""" + import json -from pathlib import Path from sampleflux.discovery import scan_module def main() -> None: - # 1. Path to our basic pipeline script - pipeline_script = Path(__file__).parent / "basic_pipeline.py" - - # 2. Scan the module for callables - print(f"--- Scanning Module: {pipeline_script.name} ---") - schemas = scan_module(pipeline_script) + module = "sampleflux.ops.numpy" + print(f"--- Scanning Module: {module} ---") + schemas = scan_module(module) - # 3. Print the results as formatted JSON - # This is exactly what FluxStudio will see print(json.dumps(schemas, indent=2)) - # 4. Verification - found_ops = [s["name"] for s in schemas] - assert "multiply" in found_ops - assert "add_noise" in found_ops + found = [s["name"] for s in schemas] + assert "Threshold" in found + assert "ConnectedComponents" in found print("\nDiscovery Engine Verified!") diff --git a/examples/flow_graph.py b/examples/flow_graph.py deleted file mode 100644 index 3fdeb0b..0000000 --- a/examples/flow_graph.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Flow-document graphs: author a named-step graph, run it on BOTH engines, convert both ways. - -Demonstrates the graph execution model: -- a ``flow`` mapping (step-name -> op) with fan-out (two readers of one step), fan-in - (``target_from``), and a per-sample parameter bind; -- native execution on :class:`sampleflux.FlowGraph`; -- lowering to a flat context-ops list (:func:`sampleflux.to_ops`) executed by the plain - serial :class:`sampleflux.Flux` engine — with identical results; -- lifting a flat list back into a flow (:func:`sampleflux.from_ops`). - -Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). -""" - -from confluid import configurable - -from sampleflux import FlowGraph, Flux, Sample, from_ops, to_ops -from sampleflux.ops.swap import SwapInputTargetOp - - -@configurable -class AddOp: - """Add a constant to the input. - - Args: - amount: Value added to ``sample.input``. - """ - - def __init__(self, amount: float = 1.0) -> None: - self.amount = amount - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + self.amount) - - -@configurable -class ScaleOp: - """Multiply the input by a factor. - - Args: - factor: Multiplier applied to ``sample.input``. - """ - - def __init__(self, factor: float = 2.0) -> None: - self.factor = factor - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input * self.factor) - - -def build_flow() -> dict: - """The graph: fork `a`; branch A swaps v+1 into target; branch B computes (v+1)*2 + bind.""" - return { - "a": AddOp(amount=1.0), # input: the source sample - "branch_a": {"op": SwapInputTargetOp(), "from": "a"}, # A: park v+1 in target - "branch_b": {"op": ScaleOp(factor=2.0), "from": "a"}, # B: (v+1)*2 (2nd read of a = fan-out) - "shifted": {"op": AddOp(), "from": "branch_b", "bind": {"amount": "a"}}, # per-sample param - "out": {"from": "shifted", "target_from": "branch_a"}, # fan-in - } - - -def main() -> None: - source = [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(4)] - - # 1. Native FlowGraph execution - native = [(s.input, s.target) for s in FlowGraph(source=source, flow=build_flow())] - print(f"FlowGraph (native): {native}") - - # 2. Lower to the flat context-ops list -> serial Flux engine - ops = to_ops(build_flow()) - print(f"Lowered ops: {[type(o).__name__ for o in ops]}") - serial = [(s.input, s.target) for s in Flux(source=source, ops=ops)] - print(f"Flux (lowered): {serial}") - assert native == serial, "engine parity is a hard contract" - - # 3. Lift the flat list back into a flow document - lifted, outputs = from_ops(to_ops(build_flow())) - relifted = [(s.input, s.target) for s in FlowGraph(source=source, flow=lifted, outputs=outputs)] - assert relifted == native - print(f"Lifted flow steps: {list(lifted)} (outputs={outputs!r})") - print("flow -> ops -> flow round-trip: parity holds ✓") - - -if __name__ == "__main__": - main() diff --git a/examples/hdf5_pipeline.py b/examples/hdf5_pipeline.py deleted file mode 100644 index 7a90b3a..0000000 --- a/examples/hdf5_pipeline.py +++ /dev/null @@ -1,51 +0,0 @@ -from pathlib import Path - -import confluid # type: ignore[import-not-found] -import numpy as np - -from sampleflux.core import Flux -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source - - -# 1. Define a simple transform -def rescale(data: np.ndarray, scale: float = 1.0) -> np.ndarray: - return data * scale - - -def main() -> None: - # 2. Setup paths - h5_path = Path("examples/test_data.h5") - - # 3. Create synthetic data and write to HDF5 - print(f"--- Writing synthetic data to {h5_path} ---") - raw_data = [np.random.randn(10) for _ in range(5)] - - # We use overwrite=True to ensure a fresh file - sink = HDF5Sink(h5_path, overwrite=True) - Flux(raw_data).to_sink(sink) - - # Force close if not already closed by context - sink.close() - - # 4. Serialize the Pipeline - pipeline = Flux().map(rescale, scale=100.0) - print("\n--- Serialized SampleFlux Pipeline ---") - yaml_state = confluid.dump(pipeline) - print(yaml_state) - - # 5. Read back and process using reconstructed pipeline - print("\n--- Reading back through Flux + HDF5Source ---") - source = HDF5Source(h5_path) - - # Reconstruct the processing logic from YAML - new_pipeline = confluid.load(yaml_state) - new_pipeline.source = source # Assign the live source - - for i, sample in enumerate(new_pipeline): - print(f"Sample {i}: mean={sample.input.mean():.2f}") - - print("\nHDF5 end-to-end flow verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/parallel_hdf5_stream.py b/examples/parallel_hdf5_stream.py deleted file mode 100644 index 999bce0..0000000 --- a/examples/parallel_hdf5_stream.py +++ /dev/null @@ -1,50 +0,0 @@ -import time -from pathlib import Path - -import numpy as np - -from sampleflux.core import Flux -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source - - -def heavy_rescale(data: np.ndarray, factor: float = 1.0) -> np.ndarray: - """A simulated heavy transformation.""" - time.sleep(0.05) # Simulate CPU work - return data * factor - - -def main() -> None: - h5_path = Path("examples/stream_results.h5") - num_samples = 50 - - print(f"--- Starting Parallel Stream to {h5_path} ---") - print(f"Generating {num_samples} samples with 4 parallel workers...") - - # 1. Setup Source (Synthetic) - raw_data = [np.random.randn(1000) for _ in range(num_samples)] - - # 2. Setup Sink - sink = HDF5Sink(h5_path, overwrite=True) - - # 3. Build and Execute Pipeline - # Processing happens in parallel, Writing happens sequentially in the main process - start = time.time() - Flux(raw_data).parallel(workers=4).map(heavy_rescale, factor=255.0).to_sink(sink) - - duration = time.time() - start - print(f"Streaming completed in {duration:.2f}s") - - # 4. Verify the results - print("\n--- Verifying HDF5 Sink Content ---") - source = HDF5Source(h5_path) - with source: - print(f"Total samples in file: {len(source)}") - sample = next(iter(source)) - print(f"First sample shape: {sample.input.shape}") - print(f"First sample max value: {sample.input.max():.2f}") - - print("\nParallel HDF5 Streaming Verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/parallel_pipeline.py b/examples/parallel_pipeline.py deleted file mode 100644 index 546f4ae..0000000 --- a/examples/parallel_pipeline.py +++ /dev/null @@ -1,45 +0,0 @@ -import time - -import numpy as np - -from sampleflux.core import Flux - - -def heavy_computation(data: np.ndarray, intensity: int = 10) -> np.ndarray: - """Simulate a heavy CPU-bound transformation.""" - # Artificial delay to simulate processing time - time.sleep(0.1) - return data * intensity - - -def main() -> None: - # 1. Create a large synthetic dataset - print("--- Creating data source (20 items) ---") - raw_data = [np.random.randn(100) for _ in range(20)] - - # 2. Sequential Execution - print("\n--- Running Sequentially ---") - start = time.time() - sequential_pipeline = Flux(raw_data).map(heavy_computation, intensity=2) - results_seq = sequential_pipeline.collect() - duration_seq = time.time() - start - print(f"Sequential duration: {duration_seq:.2f}s") - - # 3. Parallel Execution (4 workers) - print("\n--- Running in Parallel (4 workers) ---") - start = time.time() - # .parallel() enables the multiprocess engine - parallel_pipeline = Flux(raw_data).map(heavy_computation, intensity=2).parallel(workers=4) - results_par = parallel_pipeline.collect() - duration_par = time.time() - start - print(f"Parallel duration: {duration_par:.2f}s") - - # 4. Verification - assert len(results_seq) == len(results_par) - speedup = duration_seq / duration_par - print(f"\nSpeedup: {speedup:.1f}x") - print("Multiprocessing Verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py deleted file mode 100644 index fc9e1aa..0000000 --- a/examples/storage_roundtrip.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Storage round-trips: HDF5 array-valued metadata + the Zarr sources. - -Demonstrates two storage features end-to-end on synthetic data (no external -deps or services): - -1. ``HDF5Sink`` / ``HDF5Source`` round-trips a ``Sample`` whose ``metadata`` - carries a 2-D array (a stand-in for a segmentation mask). Array metadata is - stored as a dataset under a per-sample ``{prefix}_meta/`` group — it would - otherwise overflow HDF5's attribute-size limit and be silently truncated. -2. ``ZarrGroupSink`` / ``ZarrGroupSource`` round-trips input + target + metadata, - and ``ZarrBatchSink`` / ``ZarrBatchSource`` round-trips a stacked uniform array. -""" - -import tempfile -from pathlib import Path - -import numpy as np - -from sampleflux.core import Flux -from sampleflux.sample import Sample -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource - - -def main() -> None: - with tempfile.TemporaryDirectory(prefix="sampleflux-storage-demo-") as tmp: - root = Path(tmp) - - # 1. HDF5 with an array in metadata (the segmentation-mask case). - print("--- HDF5: Sample with a 2-D mask in metadata ---") - mask = np.random.randint(0, 2, size=(64, 64), dtype=np.uint8) - h5 = root / "ds.h5" - Flux([Sample(input=np.random.randn(10), target=np.array([1]), metadata={"mask": mask, "snr": 12.0})]).to_sink( - HDF5Sink(h5, overwrite=True) - ) - loaded = next(iter(HDF5Source(h5))) - print(f" mask round-trips exact : {np.array_equal(loaded.meta['mask'], mask)}") - print(f" scalar metadata kept : snr={loaded.meta['snr']}") - - # 2. Zarr group source — full input/target/metadata round-trip. - print("\n--- Zarr group: ZarrGroupSink -> ZarrGroupSource ---") - zg = root / "group.zarr" - samples = [ - Sample(input=np.arange(5, dtype="float32"), target=np.array([1]), metadata={"id": "a"}), - Sample(input=np.arange(3, dtype="float32"), metadata={"id": "b"}), - ] - Flux(samples).to_sink(ZarrGroupSink(zg, overwrite=True)) - for s in ZarrGroupSource(zg): - tgt = None if s.target is None else s.target.tolist() - print(f" input={s.input.tolist()} target={tgt} id={s.meta['id']!r}") - - # 3. Zarr batch source — stacked uniform array, input only. - print("\n--- Zarr batch: ZarrBatchSink -> ZarrBatchSource ---") - zb = root / "batch.zarr" - Flux([Sample(input=np.full((4,), i, dtype=np.float32)) for i in range(3)]).to_sink( - ZarrBatchSink(zb, shape=[4], overwrite=True) - ) - print(f" rows read back: {[int(s.input[0]) for s in ZarrBatchSource(zb)]}") - - print("\nStorage round-trips verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/typed_pipeline.py b/examples/typed_pipeline.py index 71c3693..0eead45 100644 --- a/examples/typed_pipeline.py +++ b/examples/typed_pipeline.py @@ -3,7 +3,7 @@ Demonstrates the modality-neutral core of the redesign that steps away from ``Sample(input, target, metadata)``: -1. a ``TypedSample`` is a NAMED BAG of TYPED ITEMS, each owning its metadata — an ``Image`` +1. a ``Sample`` is a NAMED BAG of TYPED ITEMS, each owning its metadata — an ``Image`` carries its layout, a ``Regions`` its canvas, a ``Label`` its classes; ``input`` / ``target`` are ROLE TAGS, not fixed positions; 2. the HEADLINE — ONE pipeline of BARE library transforms (each wrapped by its registered @@ -12,8 +12,7 @@ augmentation transforms — the libraries cover that through adapter coercion; 3. cross-field consistency — ONE library flip draw moves Image, Mask and Regions together, the Label untouched; -4. a custom transform from a plain function (``as_transform``), no library, no core edit; -5. interop — lower to a legacy ``Sample`` and lift back losslessly. +4. a custom transform from a plain function (``as_transform``), no library, no core edit. Signal-domain items (``Signal`` / ``Spectrogram``) and the ``Fourier`` transform are NOT here — sampleflux is modality-neutral. They live in ``waivefront.bag`` and register into the SAME @@ -26,13 +25,12 @@ import numpy as np from torchvision.transforms import v2 -from sampleflux import Image, Label, Mask, Pipeline, Regions, TypedSample, as_transform -from sampleflux.bag.interop import to_legacy, to_typed +from sampleflux import Image, Label, Mask, Pipeline, Regions, Sample, as_transform -def make_sample(rng: np.random.Generator) -> TypedSample: +def make_sample(rng: np.random.Generator) -> Sample: """A detection sample: an image, its mask, its boxes (targets), and a class label (target).""" - return TypedSample( + return Sample( { "image": Image(rng.random((16, 20, 3)).astype(np.float32)), "mask": Mask(rng.random((16, 20)) > 0.5), @@ -76,14 +74,6 @@ def main() -> None: print("\n--- custom function transform ---") print("image brightened:", np.allclose(np.asarray(brightened["image"]), np.asarray(sample["image"]) + 0.1)) - # 4. Interop — lossless round-trip through the legacy Sample. - legacy = to_legacy(sample) - back = to_typed(legacy) - print("\n--- legacy interop ---") - print("legacy input:", np.asarray(legacy.input).shape, " metadata keys:", list(legacy.meta)) - print("round-trip equal:", back == sample) - assert back == sample - print("\nOK") diff --git a/pyproject.toml b/pyproject.toml index 8dfa412..4bf435a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,18 +70,15 @@ sampleflux-ops-random-apply = "sampleflux.ops.random_apply" sampleflux-ops-configure = "sampleflux.ops.configure" sampleflux-ops-formula = "sampleflux.ops.formula" sampleflux-ops-transform-chain = "sampleflux.ops.transform_chain" -# Context ops (Save/Use/Drop/Apply/Capture/Mix) — the graph-plane building blocks lowered from flow: docs +# Context ops (Save/Use/Drop/Apply/Capture/MergeFields) — the graph-plane building blocks lowered from flow: docs sampleflux-ops-context = "sampleflux.ops.context" # The FlowGraph engine (flow: named-step documents + the flow<->ops converters) sampleflux-flow = "sampleflux.flow" # The queryable-metadata scan protocol + MetadataFilterSource view source sampleflux-storage-query = "sampleflux.storage.query" sampleflux-ops-sink = "sampleflux.ops.sink" -sampleflux-ops-stash = "sampleflux.ops.stash" sampleflux-ops-numpy = "sampleflux.ops.numpy" sampleflux-ops-torch = "sampleflux.ops.torch" -sampleflux-ops-copy = "sampleflux.ops.copy" -sampleflux-ops-swap = "sampleflux.ops.swap" sampleflux-ops-target = "sampleflux.ops.target" sampleflux-ops-image = "sampleflux.ops.image" # Augmentation adapters over well-known libraries (pair-scoped input+target ops). @@ -94,10 +91,8 @@ sampleflux-ops-torchvision = "sampleflux.ops.torchvision" # library transform, generated at import time by sampleflux.ops._augment_bridge. sampleflux-ops-albumentations-transforms = "sampleflux.ops.albumentations_transforms" sampleflux-ops-torchvision-transforms = "sampleflux.ops.torchvision_transforms" -# DropMetadataOp (strip metadata keys — e.g. the __taidal_stash_* snapshots) + PrintSampleOp -# (log/print a per-sample summary). Entry-point changes need an editable reinstall before -# FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). -sampleflux-ops-metadata = "sampleflux.ops.metadata" +# PrintSampleOp (log/print a per-sample summary). Entry-point changes need an editable reinstall +# before FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). sampleflux-ops-debug = "sampleflux.ops.debug" # Storage SINKS (HDF5Sink / ZarrGroupSink / ZarrBatchSink / DirectorySink) carry # category="sink" so FluxStudio surfaces them as DatasetProcessor sink nodes. They live under @@ -114,7 +109,7 @@ sampleflux-storage-directory = "sampleflux.storage.directory" # discovery sees the module (`aisland setup`, never --reinstall). sampleflux-bag-transform = "sampleflux.bag.transform" # Typed-bag structure ops (SetRole/RenameField/DropField/CopyField/SelectFields) — reshape a -# TypedSample's named fields; the typed replacement for the classic triple-slot plumbing. +# Sample's named fields; the typed replacement for the classic triple-slot plumbing. sampleflux-ops-structure = "sampleflux.ops.structure" # The runnable orchestration layer: DatasetProcessor (generic source→sink runner) and the # workflow combinators (Sequence/Conditional/Switch + PathExists/Not/AllOf/AnyOf predicates). diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py index 7cd8042..8d70896 100644 --- a/sampleflux/__init__.py +++ b/sampleflux/__init__.py @@ -1,13 +1,14 @@ """ SampleFlux: Modular, functional data pipelines. -The TYPED-BAG model (``TypedSample`` + typed items + type-dispatched ``Transform``\\ s) is THE -data model — import its surface from here (``from sampleflux import TypedSample, Image, ...``); -the internal module layout is transitional. The legacy ``Sample`` triple surface below it is -being migrated out and will be deleted once every consumer has flipped. +The data model is the TYPED BAG: a :class:`Sample` is a named bag of typed items (each +owning its metadata), ``input`` / ``target`` are ROLE TAGS on fields, and transforms +dispatch on item TYPE. Import the whole surface from the package top level +(``from sampleflux import Sample, Image, Transform, primary, ...``); the internal module +layout (``sampleflux.bag.*``) is transitional and may be promoted to the package root. """ -# --- the typed-bag surface (THE data model; frozen — consumers import ONLY from here) ----- +# --- the typed-bag data model + transforms + item codec ------------------------------------ from sampleflux.bag import ( ROLES, EncodedField, @@ -20,8 +21,8 @@ Pipeline, Regions, Role, + Sample, Transform, - TypedSample, as_transform, coerce_transform, decode_item, @@ -42,16 +43,12 @@ with_data, ) -# --- shared infrastructure (carrier-agnostic) ---------------------------------------------- -from sampleflux.collate import collate, get_collate, register_collate +# --- shared infrastructure ----------------------------------------------------------------- +from sampleflux.collate import collate, get_collate, register_collate, registered_collates, typed_collate from sampleflux.context import Context -from sampleflux.core import Flux, JointFlux, WrappedOp +from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp from sampleflux.flow import FlowGraph, from_ops, to_ops - -# --- LEGACY surface (the Sample triple era — dies with the purge stage) -------------------- -from sampleflux.kinds import INPUT, TARGET, Input, OpContract, SampleKind, Target, classify_carrier, op_contract from sampleflux.labels import LabelMap -from sampleflux.ops import RescaleOp, StandardizeOp, ToTensorOp from sampleflux.processing import DatasetProcessor from sampleflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project from sampleflux.runnable import ( @@ -62,31 +59,12 @@ entrypoint_tasks, runnable_entrypoints, ) -from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName -from sampleflux.typespec import ( - AnyType, - ArrayType, - Dim, - Dtype, - DtypeFamily, - DtypeSpec, - Framework, - ListType, - MappingType, - PythonType, - SampleType, - UnionType, - infer_field_types, - infer_sample_type, - infer_type, - typed, -) from sampleflux.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch __all__ = [ - # ---- typed-bag surface (THE data model) ---- - "TypedSample", + # ---- typed-bag data model ---- + "Sample", "Role", "ROLES", "primary", @@ -117,19 +95,35 @@ "decode_item", "encode_sample", "decode_sample", - "infer_field_types", # ---- shared infrastructure ---- "Context", "Flux", "JointFlux", + "FilterOp", + "WrappedOp", "FlowGraph", "from_ops", "to_ops", "collate", "get_collate", "register_collate", + "registered_collates", + "typed_collate", "LabelMap", - # ---- runnable protocol + orchestration (carrier-agnostic) ---- + # ---- sources ---- + "HuggingFaceSource", + "DatasetSplit", + "RangeSource", + "ConcatSource", + "SplitName", + # ---- projection ---- + "ProjectionField", + "SupportsProjection", + "iter_inputs", + "iter_targets", + "num_classes", + "project", + # ---- runnable protocol + orchestration ---- "TorchRunner", "ProgressReporting", "ProgressCallback", @@ -144,47 +138,4 @@ "Not", "AllOf", "AnyOf", - # ---- legacy surface (dies with the purge stage) ---- - "AnyType", - "ArrayType", - "ConcatSource", - "DatasetSplit", - "Dim", - "INPUT", - "Input", - "InputMeta", - "OpContract", - "Pair", - "TARGET", - "Target", - "TargetMeta", - "SampleKind", - "classify_carrier", - "op_contract", - "Dtype", - "DtypeFamily", - "DtypeSpec", - "Framework", - "HuggingFaceSource", - "ListType", - "MappingType", - "ProjectionField", - "PythonType", - "RangeSource", - "RescaleOp", - "Sample", - "SampleType", - "SplitName", - "StandardizeOp", - "SupportsProjection", - "ToTensorOp", - "UnionType", - "WrappedOp", - "infer_sample_type", - "infer_type", - "iter_inputs", - "iter_targets", - "num_classes", - "project", - "typed", ] diff --git a/sampleflux/bag/__init__.py b/sampleflux/bag/__init__.py index aa64805..08e98e9 100644 --- a/sampleflux/bag/__init__.py +++ b/sampleflux/bag/__init__.py @@ -1,6 +1,6 @@ """``sampleflux.bag`` — the typed-bag data model with type-dispatched transforms. -A sample is a NAMED BAG of TYPED ITEMS (:class:`TypedSample`), each item owning its own +A sample is a NAMED BAG of TYPED ITEMS (:class:`Sample`), each item owning its own metadata; ``input``/``target`` are ROLE TAGS on fields, not tuple positions. Transforms dispatch on item TYPE via a kernel registry, sampling their parameters once per sample so multi-field consistency (flip image + mask + boxes together) is automatic. External libraries @@ -11,7 +11,7 @@ This is THE sampleflux data model (the legacy ``Sample`` triple is being migrated out; it survives only until every consumer has flipped). Import the public surface from the PACKAGE -TOP LEVEL (``from sampleflux import TypedSample, Image, Transform, ...``) — the ``bag`` +TOP LEVEL (``from sampleflux import Sample, Image, Transform, ...``) — the ``bag`` module path is a transitional home. See ``docs/typed-model.md`` (usage) and ``docs/architecture.md`` (rationale). """ @@ -45,7 +45,7 @@ register_item, with_data, ) -from sampleflux.bag.sample import ROLES, Role, TypedSample, primary +from sampleflux.bag.sample import ROLES, Role, Sample, primary from sampleflux.bag.transform import ( FunctionTransform, Pipeline, @@ -57,7 +57,7 @@ __all__ = [ # data model - "TypedSample", + "Sample", "Role", "ROLES", "primary", diff --git a/sampleflux/bag/adapters/albumentations.py b/sampleflux/bag/adapters/albumentations.py index 2881c26..1abcb61 100644 --- a/sampleflux/bag/adapters/albumentations.py +++ b/sampleflux/bag/adapters/albumentations.py @@ -14,7 +14,7 @@ import numpy as np from sampleflux.bag.items import Image, Mask, Regions, item_data, with_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.bag.transform import Transform, register_adapter @@ -35,7 +35,7 @@ def __init__(self, transform: Optional[Any] = None, only: Optional[List[str]] = super().__init__(only=only) self.transform = transform - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if self.transform is None: raise ValueError("AlbumentationsAdapter: 'transform' must be set before calling.") @@ -71,7 +71,7 @@ def __call__(self, sample: TypedSample) -> TypedSample: ) return result - def _pick(self, sample: TypedSample, item_type: type) -> Optional[str]: + def _pick(self, sample: Sample, item_type: type) -> Optional[str]: """The first field of ``item_type`` (honoring ``only``), or ``None``.""" for key, item in sample.items(): if self.only is not None and key not in self.only: diff --git a/sampleflux/bag/adapters/torchvision.py b/sampleflux/bag/adapters/torchvision.py index c5fe467..a67e13f 100644 --- a/sampleflux/bag/adapters/torchvision.py +++ b/sampleflux/bag/adapters/torchvision.py @@ -18,7 +18,7 @@ import numpy as np from sampleflux.bag.items import Image, Mask, Regions, item_data, with_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.bag.transform import Transform, register_adapter @@ -50,7 +50,7 @@ def __init__(self, transform: Optional[Any] = None, only: Optional[List[str]] = super().__init__(only=only) self.transform = transform - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: import torch from torchvision import tv_tensors @@ -76,7 +76,7 @@ def __call__(self, sample: TypedSample) -> TypedSample: return out -def _canvas_size(sample: TypedSample) -> Optional[Tuple[int, int]]: +def _canvas_size(sample: Sample) -> Optional[Tuple[int, int]]: """``(H, W)`` from the first Image/Mask field — the reference frame for bounding boxes.""" for _, item in sample.items(): if isinstance(item, (Image, Mask)): diff --git a/sampleflux/bag/interop.py b/sampleflux/bag/interop.py deleted file mode 100644 index a7572bf..0000000 --- a/sampleflux/bag/interop.py +++ /dev/null @@ -1,63 +0,0 @@ -"""TEMPORARY bridge between the legacy ``Sample`` triple and :class:`TypedSample`. - -MIGRATION NOTE: this module dies in the purge stage (when legacy ``Sample`` is deleted). -The structural item codec it used to own moved to :mod:`sampleflux.bag.io` (the storage -serializer registry); this file keeps only the legacy-carrier bridge so typed pipelines can -run against not-yet-migrated Sample sources/sinks during the transition. - -The lowering is LOSSLESS — :func:`to_legacy` embeds the encoded typed bag in the legacy -metadata (under :data:`ENCODE_KEY`) while ALSO exposing the primary input / target payloads -on ``Sample.input`` / ``Sample.target`` so a legacy consumer still sees them; -:func:`to_typed` reconstructs the exact bag (``to_typed(to_legacy(x)) == x``). -""" - -from typing import Any, Callable, Dict, List, Optional, Tuple - -from sampleflux.bag.io import EncodedField, EncodedItem, decode_sample, encode_sample -from sampleflux.bag.items import item_data -from sampleflux.bag.sample import TypedSample -from sampleflux.sample import Sample - -__all__ = ["to_legacy", "to_typed", "ENCODE_KEY"] - -#: Metadata key under which :func:`to_legacy` stores the lossless typed-bag encoding. -ENCODE_KEY = "__typed__" - - -def to_legacy(sample: TypedSample) -> Sample: - """Lower a :class:`TypedSample` to a legacy ``Sample`` (lossless; see the module docstring).""" - inputs = sample.inputs() - targets = sample.targets() - legacy_input = item_data(next(iter(inputs.values()))) if inputs else None - legacy_target = item_data(next(iter(targets.values()))) if targets else None - encoded: List[Dict[str, Any]] = [ - {"key": f.key, "role": f.role, "type": f.item.type_name, "payload": f.item.payload, "attrs": f.item.attrs} - for f in encode_sample(sample) - ] - return Sample(input=legacy_input, target=legacy_target, metadata={ENCODE_KEY: {"fields": encoded}}) - - -def to_typed(sample: Sample, builder: Optional[Callable[[Sample], TypedSample]] = None) -> TypedSample: - """Lift a legacy ``Sample`` to a :class:`TypedSample`. - - A sample carrying an embedded encoding (produced by :func:`to_legacy`) is reconstructed - exactly. Otherwise ``builder`` is called to map the sample's fields to typed items; without - one, a clear error is raised (there is no universal legacy→typed mapping). - """ - meta = sample.metadata - if isinstance(meta, dict) and ENCODE_KEY in meta: - fields: Tuple[EncodedField, ...] = tuple( - EncodedField( - key=spec["key"], - role=spec["role"], - item=EncodedItem(type_name=spec["type"], payload=spec["payload"], attrs=spec["attrs"]), - ) - for spec in meta[ENCODE_KEY]["fields"] - ) - return decode_sample(fields) - if builder is not None: - return builder(sample) - raise ValueError( - "to_typed: legacy Sample has no embedded typed encoding — pass builder=... to map its " - "input/target/metadata onto typed items (per-dataset schema)." - ) diff --git a/sampleflux/bag/io.py b/sampleflux/bag/io.py index 63f56c9..52c822b 100644 --- a/sampleflux/bag/io.py +++ b/sampleflux/bag/io.py @@ -21,7 +21,7 @@ from typing import Any, Callable, Dict, Tuple, cast from sampleflux.bag.items import NDArrayItem, get_item_type, item_data -from sampleflux.bag.sample import Role, TypedSample +from sampleflux.bag.sample import Role, Sample __all__ = [ "EncodedItem", @@ -90,21 +90,21 @@ def decode_item(encoded: EncodedItem) -> Any: return cls(**encoded.attrs) -def encode_sample(sample: TypedSample) -> Tuple[EncodedField, ...]: +def encode_sample(sample: Sample) -> Tuple[EncodedField, ...]: """Encode every field of a sample, in insertion order.""" return tuple( EncodedField(key=key, role=sample.role_of(key), item=encode_item(item)) for key, item in sample.items() ) -def decode_sample(fields: Tuple[EncodedField, ...]) -> TypedSample: - """Rebuild a :class:`TypedSample` from encoded fields (order preserved).""" +def decode_sample(fields: Tuple[EncodedField, ...]) -> Sample: + """Rebuild a :class:`Sample` from encoded fields (order preserved).""" items: Dict[str, Any] = {} roles: Dict[str, Role] = {} for field in fields: items[field.key] = decode_item(field.item) roles[field.key] = field.role - return TypedSample(items, roles) + return Sample(items, roles) # --- the default structural codec ------------------------------------------- diff --git a/sampleflux/bag/sample.py b/sampleflux/bag/sample.py index aecf95a..c2152d6 100644 --- a/sampleflux/bag/sample.py +++ b/sampleflux/bag/sample.py @@ -1,4 +1,4 @@ -"""``TypedSample`` — the named bag of typed items that replaces ``Sample(input, target, metadata)``. +"""``Sample`` — the named bag of typed items that replaces ``Sample(input, target, metadata)``. A sample is an ordered mapping ``name -> item`` (see :mod:`sampleflux.bag.items`), plus a per-key ROLE tag. This gives every field BOTH a name (the key — the albumentations dispatch @@ -7,7 +7,7 @@ positions. A field can change role without moving keys; auxiliary items (masks, derived params) are simply tagged ``aux`` and excluded from both ``inputs()`` and ``targets()``. -``TypedSample`` is immutable — every mutator returns a NEW sample (copy-on-write), mirroring +``Sample`` is immutable — every mutator returns a NEW sample (copy-on-write), mirroring the ``Sample._replace`` idiom the legacy engine already relies on, so a transform never aliases its input. """ @@ -17,7 +17,7 @@ import numpy as np from typing_extensions import Literal, get_args -__all__ = ["TypedSample", "Role", "ROLES", "primary"] +__all__ = ["Sample", "Role", "ROLES", "primary"] #: The closed set of field roles. ``input`` / ``target`` drive the train boundary; ``aux`` is #: a helper field (mask, derived param) in neither; ``pred`` is a model prediction. Closed @@ -28,13 +28,13 @@ _DEFAULT_ROLE: Role = "input" -class TypedSample: +class Sample: """An ordered, immutable bag of typed items with per-field role tags. Construct from a mapping of items (roles default to ``input``); pass ``roles`` to tag specific keys:: - s = TypedSample( + s = Sample( {"image": Image(rgb), "regions": Regions(boxes), "class": Label("drone")}, roles={"regions": "target", "class": "target"}, ) @@ -54,9 +54,9 @@ def __init__( roles = roles or {} for key, role in roles.items(): if key not in self._fields: - raise KeyError(f"TypedSample: role given for unknown field {key!r}") + raise KeyError(f"Sample: role given for unknown field {key!r}") if role not in ROLES: - raise ValueError(f"TypedSample: invalid role {role!r} for {key!r} (allowed: {list(ROLES)})") + raise ValueError(f"Sample: invalid role {role!r} for {key!r} (allowed: {list(ROLES)})") self._roles: Dict[str, Role] = {key: roles.get(key, _DEFAULT_ROLE) for key in self._fields} # --- read views ------------------------------------------------------- @@ -115,36 +115,36 @@ def items_of_type(self, *types: type) -> Iterator[Tuple[str, Any]]: yield key, item # --- copy-on-write mutators ------------------------------------------ - def replace_field(self, key: str, item: Any) -> "TypedSample": + def replace_field(self, key: str, item: Any) -> "Sample": """A copy with ``key`` set to ``item`` (added if new; role preserved, else ``input``).""" fields = dict(self._fields) fields[key] = item - return TypedSample(fields, {**self._roles, key: self._roles.get(key, _DEFAULT_ROLE)}) + return Sample(fields, {**self._roles, key: self._roles.get(key, _DEFAULT_ROLE)}) - def set_role(self, key: str, role: Role) -> "TypedSample": + def set_role(self, key: str, role: Role) -> "Sample": """A copy with ``key``'s role set to ``role``.""" if key not in self._fields: - raise KeyError(f"TypedSample.set_role: unknown field {key!r}") + raise KeyError(f"Sample.set_role: unknown field {key!r}") if role not in ROLES: - raise ValueError(f"TypedSample.set_role: invalid role {role!r} (allowed: {list(ROLES)})") - return TypedSample(dict(self._fields), {**self._roles, key: role}) + raise ValueError(f"Sample.set_role: invalid role {role!r} (allowed: {list(ROLES)})") + return Sample(dict(self._fields), {**self._roles, key: role}) - def drop(self, key: str) -> "TypedSample": + def drop(self, key: str) -> "Sample": """A copy without ``key``.""" fields = dict(self._fields) roles = dict(self._roles) fields.pop(key, None) roles.pop(key, None) - return TypedSample(fields, roles) + return Sample(fields, roles) - def rename(self, src: str, dst: str) -> "TypedSample": + def rename(self, src: str, dst: str) -> "Sample": """A copy with field ``src`` renamed to ``dst`` (role travels; position moves to the end). Renaming onto an existing ``dst`` replaces it (last-write-wins, consistent with :meth:`merge`). Unknown ``src`` raises. """ if src not in self._fields: - raise KeyError(f"TypedSample.rename: unknown field {src!r}") + raise KeyError(f"Sample.rename: unknown field {src!r}") fields = dict(self._fields) roles = dict(self._roles) item = fields.pop(src) @@ -153,11 +153,11 @@ def rename(self, src: str, dst: str) -> "TypedSample": roles.pop(dst, None) fields[dst] = item roles[dst] = role - return TypedSample(fields, roles) + return Sample(fields, roles) # --- fan-in ------------------------------------------------------------ @classmethod - def merge(cls, *samples: "TypedSample") -> "TypedSample": + def merge(cls, *samples: "Sample") -> "Sample": """The ordered UNION of several samples' fields — the typed fan-in primitive. Fields AND their roles are united in listed order; on a key collision the @@ -169,15 +169,15 @@ def merge(cls, *samples: "TypedSample") -> "TypedSample": fields: Dict[str, Any] = {} roles: Dict[str, Role] = {} for sample in samples: - if not isinstance(sample, TypedSample): - raise TypeError(f"TypedSample.merge: expected TypedSample, got {type(sample).__name__}") + if not isinstance(sample, Sample): + raise TypeError(f"Sample.merge: expected Sample, got {type(sample).__name__}") fields.update(sample._fields) roles.update(sample._roles) return cls(fields, roles) # --- equality / repr -------------------------------------------------- def __eq__(self, other: object) -> bool: - if not isinstance(other, TypedSample): + if not isinstance(other, Sample): return NotImplemented if self._roles != other._roles or list(self._fields) != list(other._fields): return False @@ -185,10 +185,10 @@ def __eq__(self, other: object) -> bool: def __repr__(self) -> str: parts = ", ".join(f"{key}={type(item).__name__}[{self._roles[key]}]" for key, item in self._fields.items()) - return f"TypedSample({parts})" + return f"Sample({parts})" -def primary(sample: TypedSample, role: Role = "input") -> Tuple[str, Any]: +def primary(sample: Sample, role: Role = "input") -> Tuple[str, Any]: """The FIRST field of ``role`` in insertion order, as ``(key, item)``. The sanctioned answer to "the input" / "the target" of a bag: engines, ``bind``, and diff --git a/sampleflux/bag/transform.py b/sampleflux/bag/transform.py index e88c05d..8f9f4f3 100644 --- a/sampleflux/bag/transform.py +++ b/sampleflux/bag/transform.py @@ -31,7 +31,7 @@ from sampleflux.bag.dispatch import Kernel, dispatch, register_kernel from sampleflux.bag.items import item_data, with_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample __all__ = [ "Transform", @@ -72,11 +72,11 @@ def kernel(cls, item_type: type) -> Callable[[Kernel], Kernel]: """Register a kernel for ``item_type`` on this transform (decorator over :func:`register_kernel`).""" return register_kernel(cls, item_type) - def get_params(self, sample: TypedSample) -> Dict[str, Any]: + def get_params(self, sample: Sample) -> Dict[str, Any]: """Sample the shared parameters for one call. Default: no params.""" return {} - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: params = self.get_params(sample) out = sample for key, item in sample.items(): @@ -88,7 +88,7 @@ def __call__(self, sample: TypedSample) -> TypedSample: out = out.replace_field(key, kernel(item, params)) return out - def decode(self, sample: TypedSample) -> TypedSample: + def decode(self, sample: Sample) -> Sample: """The inverse transform (for visualization / back-projection). Not defined by default.""" raise NotImplementedError(f"{type(self).__name__} defines no decode (inverse)") @@ -160,7 +160,7 @@ class Pipeline: def __init__(self, transforms: Sequence[Any]) -> None: self.transforms: List[Transform] = [coerce_transform(t) for t in transforms] - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: for transform in self.transforms: sample = transform(sample) return sample @@ -181,7 +181,7 @@ def __init__(self, fn: Callable[[Any], Any], handles: Sequence[type], only: Opti self._fn = fn self.handles = tuple(handles) - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: out = sample for key, item in sample.items(): if self.only is not None and key not in self.only: diff --git a/sampleflux/collate.py b/sampleflux/collate.py index 002784e..6fcdfeb 100644 --- a/sampleflux/collate.py +++ b/sampleflux/collate.py @@ -2,35 +2,25 @@ Batching in sampleflux is two-stage: the engine groups carriers (``Flux.batch`` / ``FlowGraph.batch`` yield ``list``\\ s of N items) and a COLLATE function stacks a group -into one batched carrier. Historically each consumer package shipped its own collate -(classification / segmentation / detection, with two divergent metadata conventions); -this registry gives them ONE addressable home without changing any of them — consumers -``register_collate`` their task collates additively, and callers dispatch by key or by -the DETECTED carrier kind (:func:`sampleflux.kinds.classify_carrier`). +into one batched carrier. This registry gives consumer packages ONE addressable home for +their task collates — consumers ``register_collate`` their task collates additively, and +callers dispatch by key or by the default typed collate. The string keys primarily serve AI-callable (MCP) tool surfaces, which pass JSON-serializable names — never function objects — and enumerate the legal values via :func:`registered_collates`; in Python (and in YAML via a dotted ``!ref:`` to the function), passing a collate function directly remains the normal path. -Defaults registered here: - -- ``"sample"`` — stacks ``input``/``target`` (torch-first, numpy fallback, else kept as a - list) and gathers per-item metadata into the LIST form (``Sample.is_batched`` True — - the marainer/sonair convention). -- ``"pair"`` — a metadata-free 2-tuple batch: ``(stacked inputs, stacked targets)``. -- ``"value"`` — bare values stacked directly. - -Consumer conventions are deliberately NOT unified here (deltaid/raidar's -``{"per_sample": [...]}`` nesting stays theirs — see TASKS.md); the registry is additive. +The default registered here is ``"typed"`` — N :class:`~sampleflux.bag.sample.Sample` bags +collated into ONE batched bag (payloads stacked per field, per-item attrs as lists, roles +preserved). Consumer conventions are deliberately NOT unified here; the registry is additive. """ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from loggair import get_logger -from sampleflux.kinds import classify_carrier -from sampleflux.sample import InputMeta, Sample, TargetMeta +from sampleflux.bag.sample import Sample logger = get_logger(__name__) @@ -38,11 +28,11 @@ _REGISTRY: Dict[str, CollateFn] = {} -__all__ = ["CollateFn", "collate", "get_collate", "register_collate", "registered_collates"] +__all__ = ["CollateFn", "collate", "get_collate", "register_collate", "registered_collates", "typed_collate"] def register_collate(key: str) -> Callable[[CollateFn], CollateFn]: - """Register a collate function under ``key`` (a kind name or a task alias). + """Register a collate function under ``key`` (a task alias). Usable as a decorator:: @@ -79,18 +69,12 @@ def registered_collates() -> Tuple[str, ...]: def collate(items: Sequence[Any], key: Optional[str] = None) -> Any: """Collate ``items`` into one batched carrier. - ``key`` picks a registered collate explicitly; omitted, the DETECTED kind of the - first item dispatches — a ``TypedSample`` batch routes to ``"typed"``, everything - else through the classic carrier classifier (``sample`` / ``pair`` / ``value``). - An empty batch raises. + ``key`` picks a registered collate explicitly; omitted, the default ``"typed"`` collate + is used (every carrier is a :class:`~sampleflux.bag.sample.Sample` bag). An empty batch raises. """ - from sampleflux.bag.sample import TypedSample - if not items: raise ValueError("collate: cannot collate an empty batch") - if key is None: - key = "typed" if isinstance(items[0], TypedSample) else classify_carrier(items[0]) - return get_collate(key)(items) + return get_collate(key or "typed")(items) def _stack(values: List[Any]) -> Any: @@ -115,67 +99,29 @@ def _stack(values: List[Any]) -> Any: return list(values) -@register_collate("sample") -def sample_collate(items: Sequence[Any]) -> Sample: - """Default Sample collate: stacked input/target + LIST-form metadata (``is_batched`` True).""" - samples = [item if isinstance(item, Sample) else Sample.from_any(item) for item in items] - return Sample( - input=_stack([s.input for s in samples]), - target=_stack([s.target for s in samples]), - metadata=[dict(s.meta) for s in samples], - ) - - -@register_collate("pair") -def pair_collate(items: Sequence[Any]) -> Tuple[Any, Any]: - """Default pair collate: ``(stacked inputs, stacked targets)`` — no metadata anywhere.""" - return _stack([item[0] for item in items]), _stack([item[1] for item in items]) - - -@register_collate("value") -def value_collate(items: Sequence[Any]) -> Any: - """Default value collate: the bare values stacked.""" - return _stack(list(items)) - - -@register_collate("input_meta") -def input_meta_collate(items: Sequence[Any]) -> InputMeta: - """Default InputMeta collate: stacked inputs + the per-item metadata dicts as a list.""" - return InputMeta(_stack([item.input for item in items]), [dict(item.metadata) for item in items]) - - -@register_collate("target_meta") -def target_meta_collate(items: Sequence[Any]) -> TargetMeta: - """Default TargetMeta collate: stacked targets + the per-item metadata dicts as a list.""" - return TargetMeta(_stack([item.target for item in items]), [dict(item.metadata) for item in items]) - - @register_collate("typed") def typed_collate(items: Sequence[Any]) -> Any: - """The typed-bag collate: N ``TypedSample``\\ s → ONE batched ``TypedSample``. + """The typed-bag collate: N ``Sample``\\ s → ONE batched ``Sample``. Per field (union of keys is NOT taken — every sample must carry the same fields, a mismatch raises): payloads are stacked via :func:`_stack` (torch → stacked tensor, numpy → stacked array, else a list) and each declared item attr becomes a LIST of per-item values. Array items come back as the SAME item type over the stacked payload; - wrapper items likewise (attrs as lists). Roles are preserved. This single convention - replaces both classic batch shapes (the list-form batched metadata and the - ``{"per_sample": [...]}`` dict-nest) — trainers read ``primary(batch)`` / - ``batch.targets()``. + wrapper items likewise (attrs as lists). Roles are preserved. Trainers read + ``primary(batch)`` / ``batch.targets()``. """ from sampleflux.bag.io import EncodedItem, decode_item, encode_item - from sampleflux.bag.sample import TypedSample if not items: raise ValueError("typed_collate: cannot collate an empty batch") first = items[0] - if not isinstance(first, TypedSample): - raise TypeError(f"typed_collate: expected TypedSample items, got {type(first).__name__}") + if not isinstance(first, Sample): + raise TypeError(f"typed_collate: expected Sample items, got {type(first).__name__}") keys = list(first.keys()) for i, sample in enumerate(items): - if not isinstance(sample, TypedSample) or list(sample.keys()) != keys: + if not isinstance(sample, Sample) or list(sample.keys()) != keys: raise ValueError( - f"typed_collate: item {i} fields {list(sample.keys()) if isinstance(sample, TypedSample) else '?'} " + f"typed_collate: item {i} fields {list(sample.keys()) if isinstance(sample, Sample) else '?'} " f"do not match the batch fields {keys} — collate requires a homogeneous batch." ) fields: Dict[str, Any] = {} @@ -185,4 +131,4 @@ def typed_collate(items: Sequence[Any]) -> Any: stacked_payload = _stack([e.payload for e in encoded]) if encoded[0].payload is not None else None batched_attrs = {name: [e.attrs.get(name) for e in encoded] for name in encoded[0].attrs} fields[key] = decode_item(EncodedItem(type_name=type_name, payload=stacked_payload, attrs=batched_attrs)) - return TypedSample(fields, {key: first.role_of(key) for key in keys}) + return Sample(fields, {key: first.role_of(key) for key in keys}) diff --git a/sampleflux/core.py b/sampleflux/core.py index d33ea3d..eb5da29 100644 --- a/sampleflux/core.py +++ b/sampleflux/core.py @@ -1,23 +1,7 @@ import concurrent.futures -import json import multiprocessing from contextlib import nullcontext -from functools import lru_cache -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Collection, - Dict, - Iterable, - Iterator, - List, - NamedTuple, - Optional, - Tuple, - Union, - cast, -) +from typing import Any, Callable, Collection, Dict, Iterable, Iterator, List, NamedTuple, Optional, Tuple, Union, cast import torch.utils.data from confluid import configurable @@ -26,197 +10,32 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.items import item_data, with_data +from sampleflux.bag.sample import Role, Sample, primary from sampleflux.context import Context, activate from sampleflux.projection import ProjectionField -from sampleflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, InputMeta, Pair, Sample, TargetMeta - -if TYPE_CHECKING: # pragma: no cover - typing only - from sampleflux.typespec import SampleType logger = get_logger(__name__) +# Role a projection field maps onto in the typed bag (``metadata`` -> the ``aux`` role). +_PROJECTION_ROLES: Dict[str, str] = {"input": "input", "target": "target", "metadata": "aux"} -@lru_cache(maxsize=None) -def _serialized_type_keys(produces: "SampleType") -> Tuple[str, str]: - """Serialize an op's ``PRODUCES`` to the two stored-type JSON strings, memoized per spec so the - ``datasets.Features`` build happens once per distinct spec rather than once per sample.""" - features, extras = produces.to_hf_features() - return json.dumps(features.to_dict()), json.dumps(extras) - - -def _refresh_type(sample: Sample, op: Any) -> Sample: - """Keep a sample's stored type honest after an op — only when the sample already carries one. - - Default (untracked) pipelines never stamp a type, so this is a no-op and metadata is byte-identical - to before. When a stored type IS present (set via :meth:`Sample.with_type` or loaded from a typed - dataset), an op that declares ``PRODUCES`` refreshes it; an op that declares none drops it so - :meth:`Sample.describe` falls back to inference rather than reporting a stale type. - """ - if not any(key in sample.meta for key in TYPE_KEYS): - return sample - produces = getattr(op, "PRODUCES", None) - if produces is not None: - features_key, spec_key = _serialized_type_keys(produces) - return sample._replace(metadata={**sample.meta, FEATURES_KEY: features_key, SPEC_KEY: spec_key}) - return sample._replace(metadata={k: v for k, v in sample.meta.items() if k not in TYPE_KEYS}) - - -def _as_carrier(item: Any, native: bool) -> Any: - """The stream carrier for a raw source item. - A :class:`TypedSample` passes VERBATIM on every route — the typed carrier is - first-class and is never coerced into the legacy triple. ``native`` mode keeps any - other carrier as-is (pair/value lanes); the default coerces to the legacy ``Sample``. - """ - if native or isinstance(item, TypedSample): - return item - return Sample.from_any(item) +def _op_expands(op: Any) -> bool: + """True when an op is a 1→N expanding op (explicit ``EXPANDS = True`` class attribute).""" + return bool(getattr(op, "EXPANDS", False)) def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: - """Apply one op and refresh the stored type. The single op-application chokepoint shared by the - sequential, parallel (via :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths. - - The op's introspected contract (:func:`sampleflux.kinds.op_contract`) picks the BINDING: - a classic sample/untyped op receives the Sample verbatim (today's fast path); a - field-scoped op (``input`` / ``target`` / ``pair`` / ``input_meta`` / ``target_meta`` — - packed views or unpacked separate arguments) receives exactly its declared view and the - result merges back with the untouched fields preserved (:func:`_apply_view`). - """ - from sampleflux.kinds import op_contract - - if isinstance(sample, TypedSample): - # The TYPED carrier: transforms take the whole bag verbatim — the kinds field-scope - # binding and the reserved-key stored-type refresh are legacy-Sample concepts. - return op(sample) - - contract = op_contract(op) - if contract.accepts in ("sample", "any", "value") and contract.style == "packed": - result = op(sample) - if result is None: - return None - return _refresh_type(result, op) - return _apply_view(sample, op, contract) - - -def _view_error(op: Any, scope: str, result: Any) -> TypeError: - return TypeError( - f"{type(op).__name__}: a {scope!r}-scope op must return the matching view/tuple, a full " - f"Sample, or None — got {type(result).__name__}" - ) - + """Apply one op to the typed :class:`Sample` bag verbatim. -# One argument of an unpacked op, bound from the sample per its declared field scope. -_BIND_GET: Dict[str, Callable[[Sample], Any]] = { - "input": lambda s: s.input, - "target": lambda s: s.target, - "metadata": lambda s: s.meta, - "input_meta": lambda s: s.input_meta(), - "target_meta": lambda s: s.target_meta(), -} - - -def _apply_bindings(sample: Sample, op: Any, bindings: Tuple[str, ...]) -> Optional[Sample]: - """Apply an UNPACKED op — each argument bound per its declared field scope — and merge back. - - Handles EVERY combination the binding resolver produces: the classic - ``f(input, target)`` / ``f(input, target, metadata)``, the meta forms - ``f(input, metadata)`` / ``f(target, metadata)``, and mixed VIEW arguments like - ``f(im: InputMeta, tm: TargetMeta)`` or ``f(x: Input, tm: TargetMeta)``. The result - must be ``None`` (drop), a full ``Sample`` (takes over), or a tuple of the SAME arity - — each element merged per its binding (a view/2-tuple element for a ``*_meta`` binding - replaces value + metadata; a bare element replaces only the value). Metadata-bearing - elements merge left-to-right (the LAST metadata write wins — they usually share the - one live dict anyway, which the op may also mutate in place). - """ - args = [_BIND_GET[b](sample) for b in bindings] - result = op(*args) - if result is None: - return None - if isinstance(result, Sample): - return _refresh_type(result, op) - # A NAMED view is itself a tuple — returning ONE view from a multi-binding op would be - # silently misread as two elements, so it only counts as the whole result at arity 1. - is_single_view = isinstance(result, (InputMeta, TargetMeta, Pair)) - if (is_single_view and len(bindings) != 1) or not (isinstance(result, tuple) and len(result) == len(bindings)): - raise TypeError( - f"{type(op).__name__}: an unpacked op bound as {bindings!r} must return a tuple of the " - f"same arity, a full Sample, or None — got {type(result).__name__}" - ) - updates: Dict[str, Any] = {} - for binding, element in zip(bindings, result): - if binding in ("input", "target", "metadata"): - updates[binding] = element - else: # input_meta / target_meta - field = "input" if binding == "input_meta" else "target" - if isinstance(element, tuple) and len(element) == 2: - updates[field] = element[0] - updates["metadata"] = element[1] - else: # bare value: only the field changes (in-place meta mutation is already live) - updates[field] = element - return _refresh_type(sample._replace(**updates), op) - - -def _apply_view(sample: Sample, op: Any, contract: Any) -> Optional[Sample]: - """Bind a field-scoped op's declared view from ``sample``, apply, and merge the result back. - - Unpacked ops route through :func:`_apply_bindings` (per-argument scopes). Packed - single-view scopes (``None`` always drops; a returned ``Sample`` always takes over; - metadata dicts are handed live, so in-place mutation propagates): - - - ``input`` / ``target`` — the bare value in, the new value out (other fields kept); - - ``metadata`` — the dict in, the (new) dict out; - - ``pair`` — a `Pair` in (a plain-tuple-annotated op indexes it identically), a - 2-tuple out replaces input+target (metadata kept); - - ``input_meta`` / ``target_meta`` — the named view in; a view/2-tuple out replaces - value + metadata; a bare value out replaces only the value. + The single op-application chokepoint shared by the sequential, parallel (via + :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths. A transform + takes the whole bag and returns a new bag (or ``None`` to drop the sample); composing + ops (``Parallel`` / ``Enable`` / ``TransformChain`` / ``RandomApply`` / the context ops) + route their inner ops through here so every op is applied identically. """ - if contract.style == "unpacked" and contract.bindings: - return _apply_bindings(sample, op, contract.bindings) - scope = contract.accepts - - if scope == "input" or scope == "target": - field = scope - result = op(getattr(sample, field)) - if result is None: - return None - return _refresh_type(sample._replace(**{field: result}), op) - - if scope == "metadata": - result = op(sample.meta) - if result is None: - return None - if isinstance(result, Sample): - return _refresh_type(result, op) - if isinstance(result, dict): - return _refresh_type(sample._replace(metadata=result), op) - raise _view_error(op, scope, result) - - if scope == "pair": - result = op(Pair(sample.input, sample.target)) - if result is None: - return None - if isinstance(result, Sample): - return _refresh_type(result, op) - if isinstance(result, tuple) and len(result) == 2: - return _refresh_type(sample._replace(input=result[0], target=result[1]), op) - raise _view_error(op, scope, result) - - if scope in ("input_meta", "target_meta"): - field = "input" if scope == "input_meta" else "target" - result = op(sample.input_meta() if scope == "input_meta" else sample.target_meta()) - if result is None: - return None - if isinstance(result, Sample): - return _refresh_type(result, op) - if isinstance(result, tuple) and len(result) == 2: - return _refresh_type(sample._replace(**{field: result[0], "metadata": result[1]}), op) - return _refresh_type(sample._replace(**{field: result}), op) - - # A packed "sample"-scope op took the _apply_op fast path; anything else is defensive. - result = op(sample) # pragma: no cover - return None if result is None else _refresh_type(result, op) # pragma: no cover + return cast(Optional[Sample], op(sample)) def _describe_deferred_source(source: Any) -> str: @@ -266,13 +85,6 @@ class FilterOp: passes when the predicate returns ``True`` and is dropped otherwise (``__call__`` returns ``None``, which every engine route treats as "skip this sample"). - Example:: - - keep_loud = FilterOp(p=lambda s: float(s.input.max()) > 0.1) - flux = Flux(source=src, ops=[keep_loud]) - # equivalently, via the fluent API (which constructs this op): - flux = Flux(source=src).filter(lambda s: float(s.input.max()) > 0.1) - Args: p: Predicate ``Sample -> bool``; the sample passes through when it returns ``True``, else is dropped. Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. @@ -292,24 +104,17 @@ def __call__(self, s: Sample) -> Optional[Sample]: class WrappedOp: """Configurable transformation wrapper with smart mapping. - The op form of :meth:`Flux.map` — lifts a plain function over one Sample slot. The + The op form of :meth:`Flux.map` — lifts a plain function over one Sample field. The callable is ALWAYS stored as its importable ``module:function`` path (via :mod:`sampleflux.discovery`), so the op pickles across ``spawn`` workers and serializes into Confluid YAML verbatim; the live function resolves lazily on first call. - Example:: - - op = WrappedOp(f="numpy:sqrt", s="input") # dotted path — resolved lazily - flux = Flux(source=src, ops=[op]) - # equivalently, from a live callable via the fluent API (which constructs - # this op and stores np.sqrt as the string "numpy:sqrt"): - flux = Flux(source=src).map(np.sqrt) - Args: f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. - s: Which Sample slot to transform — ``"input"`` (default), ``"target"``, or ``"all"`` (the whole Sample). + s: Which field to transform — ``"input"`` (default, the primary input field's payload), + ``"target"`` (the primary target field's payload), or ``"all"`` (the whole ``Sample`` bag). kw: Extra keyword arguments forwarded to the wrapped callable on every call (defaults to none). """ @@ -333,112 +138,55 @@ def func(self) -> Callable: return self._func_cache def __call__(self, sample: Sample) -> Optional[Sample]: - try: - if self.s == "input": - new_input = self.func(sample.input, **self.kw) - return sample._replace(input=new_input) - elif self.s == "target": - new_target = self.func(sample.target, **self.kw) - return sample._replace(target=new_target) - elif self.s == "all": - return cast(Sample, self.func(sample, **self.kw)) - return sample - except Exception as e: - raise e + if self.s == "all": + return cast(Sample, self.func(sample, **self.kw)) + role: Role = "input" if self.s == "input" else "target" + key, item = primary(sample, role) + new_data = self.func(item_data(item), **self.kw) + return sample.replace_field(key, with_data(item, new_data)) class _Carried(NamedTuple): - """A carrier travelling the streamed route together with its per-sample Context. - - ``sample`` is a :class:`Sample` on the default route; under ``Flux(native=True)`` it - may be any native carrier (a metadata-free pair, a bare value). - """ + """A :class:`Sample` travelling the streamed route together with its per-sample Context.""" sample: Any ctx: Context -def _apply_op_native(carrier: Any, op: Any) -> Any: - """Apply one op to a NATIVE carrier (Sample / pair / bare value / a field view). - - Adaptation rules (the op's contract via :func:`sampleflux.kinds.op_contract`): - - - a **Sample** carrier routes through :func:`_apply_op` (which binds every scope); - - an **any-op** receives the carrier verbatim (untyped ops behave exactly as today); - - two NATIVE fast lanes keep metadata-free data metadata-free: a pair-scope op on a - pair carrier (result stays a pair) and an input-scope op on a bare value (result - stays a bare value); - - everything else PROMOTES the carrier to a Sample view (``Sample.from_any`` — view - types like ``InputMeta`` coerce field-correctly) — promotion is one-way and sticky, - so op-written metadata is never dropped. - """ - from sampleflux.kinds import classify_carrier, op_contract - - if isinstance(carrier, TypedSample): - return op(carrier) # the typed carrier is applied verbatim, never promoted - contract = op_contract(op) - if isinstance(carrier, Sample): - return _apply_op(carrier, op) - if contract.accepts == "any": - return op(carrier) - kind = classify_carrier(carrier) - if contract.accepts == "pair" and kind == "pair": - result = op(carrier[0], carrier[1]) if contract.style == "unpacked" else op(tuple(carrier)) - if result is None: - return None - if isinstance(result, (Sample, tuple)): - return result - raise _view_error(op, "pair", result) - if contract.accepts == "input" and kind == "value": - return op(carrier) - return _apply_op(Sample.from_any(carrier), op) # promotion is sticky - - -def _expand(op: Any, carrier: Any) -> List[Any]: - """Run a 1→N EXPANDING op and return its flattened, type-refreshed children.""" - raw = op(carrier) +def _expand(op: Any, sample: Any) -> List[Any]: + """Run a 1→N EXPANDING op and return its flattened children.""" + raw = op(sample) if raw is None: return [] - children: List[Any] = [] - for child in raw: - if child is None: - continue - children.append(_refresh_type(child, op) if isinstance(child, Sample) else child) - return children + return [child for child in raw if child is not None] -def _worker_task(sample: Any, ops: List[Any], native: bool = False) -> Optional[Any]: +def _worker_task(sample: Any, ops: List[Any]) -> Optional[Any]: """Single-result worker for STRICTLY 1→1 op lists (the ``Parallel`` op's contract). Kept for callers that need exactly one carrier back; expanding ops raise here — route expanding pipelines through :func:`_worker_task_multi`. """ - results = _worker_task_multi(sample, ops, native=native, allow_expansion=False) + results = _worker_task_multi(sample, ops, allow_expansion=False) return results[0] if results else None -def _worker_task_multi(sample: Any, ops: List[Any], native: bool = False, allow_expansion: bool = True) -> List[Any]: +def _worker_task_multi(sample: Any, ops: List[Any], allow_expansion: bool = True) -> List[Any]: """Top-level helper for multiprocess workers. Must be at top level for pickling. - Runs one source carrier through the op list and returns EVERY resulting carrier — - usually one, zero when filtered, several when a 1→N EXPANDING op fired (detected via - :func:`sampleflux.kinds.op_contract`; each expansion child continues through the - REMAINING ops with a shallow copy of the per-sample Context, depth-first so sibling - order matches the nested-loop intuition). + Runs one source :class:`Sample` through the op list and returns EVERY resulting sample — + usually one, zero when filtered, several when a 1→N EXPANDING op fired; each expansion + child continues through the REMAINING ops with a shallow copy of the per-sample Context, + depth-first so sibling order matches the nested-loop intuition. - Activates ONE fresh per-carrier :class:`~sampleflux.context.Context` around the op - loop so context ops (``Save``/``Use``/``Apply``/``Capture``/``Mix``) can move data - between the linear stream and named cells — the executor itself stays a plain - ``for op in ops`` loop. Contexts are created inside the worker (spawn-safe: ops - pickle, a Context never crosses a process boundary). - - ``native=True`` keeps the carrier's own kind (Sample / pair / value) and adapts it - per op via :func:`_apply_op_native` instead of coercing everything to Sample. + Activates ONE fresh per-sample :class:`~sampleflux.context.Context` around the op loop so + context ops (``Save``/``Use``/``Apply``/``Capture``/``MergeFields``) can move data between + the linear stream and named cells — the executor itself stays a plain ``for op in ops`` + loop. Contexts are created inside the worker (spawn-safe: ops pickle, a Context never + crosses a process boundary). """ from collections import deque - from sampleflux.kinds import op_contract - pending: "deque[Tuple[Any, Context, int]]" = deque([(sample, Context(), 0)]) out: List[Any] = [] while pending: @@ -449,7 +197,7 @@ def _worker_task_multi(sample: Any, ops: List[Any], native: bool = False, allow_ while i < len(ops): op = ops[i] i += 1 - if op_contract(op).expands: + if _op_expands(op): if not allow_expansion: raise TypeError( f"op {type(op).__name__!r} is a 1→N expanding op, which this strictly " @@ -460,13 +208,12 @@ def _worker_task_multi(sample: Any, ops: List[Any], native: bool = False, allow_ alive = False break # Depth-first: the first child continues inline; its siblings go to the - # FRONT of the queue (reversed, so sibling order is preserved) — output - # order matches the nested-loop intuition even for chained expansions. + # FRONT of the queue (reversed, so sibling order is preserved). for child in reversed(children[1:]): pending.appendleft((child, ctx.copy(), i)) current = children[0] continue - result = _apply_op_native(current, op) if native else _apply_op(current, op) + result = _apply_op(current, op) if result is None: alive = False break @@ -487,14 +234,6 @@ class JointFlux: materialization. For an indexable (random-access) concatenation of raw sources, use ``ConcatSource`` instead. - Example:: - - clean = Flux(source=day_one, ops=[normalize]) - augmented = Flux(source=day_two, ops=[normalize, augment]) - both = JointFlux(fluxes=[clean, augmented]) # len == len(clean) + len(augmented) - # or wrapped back into an engine (equivalent fluent form): - flux = Flux.joint([clean, augmented]) # == Flux(source=JointFlux([...])) - Args: fluxes: The Flux streams to concatenate; iteration walks them in order and length is their sum. Defaults to ``None`` ⇒ an empty joint stream (zero-arg construction). @@ -520,27 +259,15 @@ class Flux(torch.utils.data.Dataset[Sample]): The primary stream engine for SampleFlux. Wraps any iterable or indexed dataset and provides a functional API. - Annotation design (kept intentionally ``Any``): - Per the SampleFlux mandate "Functional Purity: Transforms are plain - Python callables. Never introduce base classes or complex inheritance - for data operations.", ``source`` is duck-typed (any iterable; the - Indexable protocol if ``__getitem__``/``__len__`` are present) and - ``ops`` is a list of bare callables ``Sample -> Optional[Sample]``. - No ``Source`` or ``Op`` ABC is introduced. - - Downstream auto-gen pydantic mirrors (``confluid.to_pydantic``) - coerce abstract iterable types to ``Any`` so identity-tracked - serialization (e.g. shared-source dataset-split patterns in - navigaitor) works correctly — see - ``confluid/pydantic_export.py:_ITER_TYPES_AS_ANY``. + Every carrier is a typed :class:`~sampleflux.bag.sample.Sample` bag, passed through the op + chain verbatim (no coercion). ``source`` is duck-typed (any iterable; the Indexable + protocol if ``__getitem__``/``__len__`` are present) and ``ops`` is a list of bare + transforms ``Sample -> Optional[Sample]``. Args: - source: Any iterable or indexable dataset (duck-typed) to wrap; ``None`` yields an empty stream. - ops: Ordered callables ``Sample -> Optional[Sample]`` applied lazily on access (``None`` = no ops). + source: Any iterable or indexable dataset (duck-typed) yielding ``Sample`` bags; ``None`` = empty stream. + ops: Ordered transforms ``Sample -> Optional[Sample]`` applied lazily on access (``None`` = no ops). chunk_size: Parallel-processing chunk size; ``0`` (the default) processes sequentially. - native: Opt-in multi-type mode — carriers keep their own kind (Sample / metadata-free pair / - bare value) and each op is adapted per its introspected contract (``sampleflux.kinds``). - ``False`` (the default) coerces every item to ``Sample`` exactly as before. """ def __init__( @@ -548,11 +275,9 @@ def __init__( source: Optional[Iterable[Any]] = None, ops: Optional[List[Any]] = None, chunk_size: Optional[int] = 0, - native: bool = False, ) -> None: self.source = source self.ops: List[Any] = ops or [] - self.native = bool(native) self._workers = 1 self._chunk_size = chunk_size or 0 # Populated on first random access when the source is iterable-only @@ -560,14 +285,7 @@ def __init__( self._indexable_cache: Optional[List[Any]] = None def _guard_live_source(self) -> Any: - """Return the source, surfacing a clear error when it's still a Fluid marker. - - Flux does not materialize deferred Confluid markers itself — that's - Confluid's job — but if a user hands Flux a deferred Class/Instance - marker we raise with an actionable message instead of letting the - failure surface as ``num_samples=0`` or a generic ``TypeError`` deep - inside torch's DataLoader. - """ + """Return the source, surfacing a clear error when it's still a Fluid marker.""" if isinstance(self.source, _ConfluidFluid): raise TypeError(_fluid_source_guidance(self.source)) return self.source @@ -586,17 +304,10 @@ def joint(cls, fluxes: List["Flux"]) -> "Flux": def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": """Attach an ops-only Confluid YAML (e.g. one exported by a pipeline-authoring tool) to ``source``. - ``path`` is the ``{ops: [!class:...()]}`` document produced by - an external graph exporter's ops-export (the CLI or the - canvas Export button). It also accepts an inline YAML string (``confluid.load`` - handles both). - - The op markers are **materialized to live callables** before being attached: - ``confluid.load`` leaves ``!class:`` markers nested under a mapping key deferred - (its final flow pass doesn't descend dict→list), so a plain ``load(path)["ops"]`` - would hand :class:`Flux` deferred ``Instance`` markers — which iteration rejects by - design (see :meth:`_guard_live_source` / ``_check_ops_materialized``). Routing through - :func:`confluid.materialize` flows the top-level list of markers into live ops. + ``path`` is the ``{ops: [!class:...()]}`` document produced by an external graph + exporter's ops-export. Op markers are materialized to live callables before being + attached (``confluid.load`` leaves ``!class:`` markers nested under a mapping key + deferred, so ``confluid.materialize`` flows them into live ops). """ loaded = _confluid_load(path) raw_ops = loaded.get("ops", []) if isinstance(loaded, dict) else [] @@ -605,12 +316,7 @@ def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Fl @classmethod def from_flow_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": - """Attach a ``{flow: {...}}`` graph document to ``source``, LOWERED to the serial form. - - The named-step flow document (see :mod:`sampleflux.flow`) is compiled into a flat - context-ops list via :func:`sampleflux.flow.to_ops`, so the graph executes on this - plain serial engine. ``FlowGraph.from_yaml`` is the native-engine twin. - """ + """Attach a ``{flow: {...}}`` graph document to ``source``, LOWERED to the serial form.""" from sampleflux.flow import flow_yaml_to_flux return cast("Flux", flow_yaml_to_flux(path, source=source)) @@ -618,16 +324,12 @@ def from_flow_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "F @property def _expands(self) -> bool: """True when any (materialized) op is a 1→N expanding op — the pipeline is then iterable-only.""" - from sampleflux.kinds import op_contract - - return any(not isinstance(op, _ConfluidFluid) and op_contract(op).expands for op in self.ops) + return any(not isinstance(op, _ConfluidFluid) and _op_expands(op) for op in self.ops) def _guard_not_expanding(self, operation: str) -> None: if self._expands: - from sampleflux.kinds import op_contract - culprit = next( - type(op).__name__ for op in self.ops if not isinstance(op, _ConfluidFluid) and op_contract(op).expands + type(op).__name__ for op in self.ops if not isinstance(op, _ConfluidFluid) and _op_expands(op) ) raise TypeError( f"Flux.{operation}: the pipeline contains the 1→N expanding op {culprit!r}, so the " @@ -637,13 +339,7 @@ def _guard_not_expanding(self, operation: str) -> None: ) def __len__(self) -> int: - """Return the length of the underlying source if available. - - Surfaces a clear error when the source is still a deferred Confluid - marker so downstream callers (e.g. torch's DataLoader) don't end up - reporting the opaque ``num_samples=0``, and when the pipeline contains - a 1→N expanding op (iterable-only — the true length is unknowable). - """ + """Return the length of the underlying source if available.""" from collections.abc import Sized source = self._guard_live_source() @@ -653,22 +349,7 @@ def __len__(self) -> int: return 0 def __getitem__(self, index: int) -> Any: - """Random access: get the i-th sample with ops applied. - - Supports three source shapes: - - - **Indexable** (``__getitem__`` present) — delegates directly. - - **Iterable with ``__len__``** (map-style-but-stream, like - :class:`waivefront.regions_source.RegionsJsonSource`) — materializes - the full source into a list on first access, caches it on the Flux - instance, and indexes into the cache on every subsequent call. - The list is built once per Flux lifetime, not once per epoch. - - **Bare iterator** (no ``__len__``) — raises ``TypeError``. Caching - a one-shot iterator silently would consume the user's source; if - random access is genuinely needed, either give the source a - ``__len__`` or wrap with ``list(source)`` explicitly at the call - site. - """ + """Random access: get the i-th sample with ops applied.""" source = self._guard_live_source() if source is None: raise TypeError("Flux source is None — cannot index. Pass a DataSource / iterable to Flux(source=...).") @@ -678,9 +359,7 @@ def __getitem__(self, index: int) -> Any: raw = source[index] elif hasattr(source, "__len__"): if self._indexable_cache is None: - logger.debug( - f"Flux: materializing iterable-only source " f"{type(source).__name__} for map-style random access." - ) + logger.debug(f"Flux: materializing iterable-only source {type(source).__name__} for random access.") self._indexable_cache = list(source) raw = self._indexable_cache[index] else: @@ -691,10 +370,10 @@ def __getitem__(self, index: int) -> Any: "it in ``list(...)`` before handing it to Flux." ) _check_ops_materialized(self.ops) - sample: Any = _as_carrier(raw, self.native) + sample: Any = raw with activate(Context()): for op in self.ops: - result = _apply_op_native(sample, op) if self.native else _apply_op(sample, op) + result = _apply_op(sample, op) if result is None: raise IndexError(f"Sample {index} filtered out by {op}") sample = result @@ -704,7 +383,6 @@ def to_sink(self, sink: Any) -> None: """Write the entire flux to a DataSink.""" from sampleflux.storage.base import Storage - # Open sink if it's a context-aware storage; otherwise no-op context. target_sink: Any = sink if isinstance(sink, Storage) else nullcontext() with target_sink: @@ -713,29 +391,17 @@ def to_sink(self, sink: Any) -> None: sink.flush() def parallel(self, workers: int = 4) -> "Flux": - """ - Enable multiprocess execution for the pipeline. - - Args: - workers: Number of worker processes to spawn. - """ + """Enable multiprocess execution for the pipeline.""" self._workers = workers return self def batch(self, chunk_size: int) -> "Flux": - """ - Group samples into chunks (lists of N samples). - - Args: - chunk_size: Number of samples per chunk. - """ + """Group samples into chunks (lists of N samples).""" self._chunk_size = chunk_size return self def map(self, func: Callable, select: str = "input", **kwargs: Any) -> "Flux": - """ - Append a transformation to the flux. - """ + """Append a transformation to the flux.""" op = WrappedOp(func, select, kwargs) self.ops.append(op) return self @@ -746,15 +412,7 @@ def filter(self, predicate: Callable[[Sample], bool]) -> "Flux": return self def __iter__(self) -> Iterator[Any]: - """Execute the pipeline lazily. - - Routing: - * Any op exposes a callable ``stream`` attribute (e.g. - :class:`sampleflux.ops.parallel.Parallel`) → :meth:`_iter_streamed`, - which composes the upstream iterator through stream-level ops. - * Else ``self._workers > 1`` → legacy :meth:`_iter_parallel`. - * Else :meth:`_iter_sequential`. - """ + """Execute the pipeline lazily.""" if not self._guard_live_source(): return @@ -778,23 +436,7 @@ def __iter__(self) -> Iterator[Any]: yield from it def _iter_streamed(self) -> Iterator[Sample]: - """Mixed per-sample / stream-level op chain. - - Per-sample ops are applied via ``op(sample)``. Ops that implement - ``.stream(sample_iter)`` (e.g. - :class:`sampleflux.ops.parallel.Parallel`) are handed the upstream - generator and yield transformed samples themselves. ``None`` results - are filtered, matching :meth:`_iter_sequential`. - - Each sample travels with its own per-sample :class:`Context` (a private - ``(sample, ctx)`` carrier between per-sample stages), activated around - every ``_apply_op`` call. A stream-level op is a Context boundary: the - carrier is stripped to a bare sample before ``op.stream(...)`` (raising - if cells are still live — cross-``Parallel`` graphs are a documented v1 - limit; ``Parallel``'s INNER chain gets its own contexts via - :func:`_worker_task`), and samples emerging downstream get fresh - contexts. - """ + """Mixed per-sample / stream-level op chain (a stream-level op exposes ``.stream``).""" source = self._guard_live_source() if source is None: return @@ -802,12 +444,10 @@ def _iter_streamed(self) -> Iterator[Sample]: def to_carried() -> Iterator[Optional[_Carried]]: for item in source: - yield _Carried(_as_carrier(item, self.native), Context()) + yield _Carried(item, Context()) def per_sample(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[_Carried]]: - from sampleflux.kinds import op_contract - - expands = op_contract(op).expands + expands = _op_expands(op) for c in stream: if c is None: continue @@ -815,7 +455,7 @@ def per_sample(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Option if expands: children = _expand(op, c.sample) else: - s = _apply_op_native(c.sample, op) if self.native else _apply_op(c.sample, op) + s = _apply_op(c.sample, op) if expands: for j, child in enumerate(children): yield _Carried(child, c.ctx if j == 0 else c.ctx.copy()) @@ -857,8 +497,7 @@ def _iter_sequential(self) -> Iterator[Sample]: return _check_ops_materialized(self.ops) for item in source: - sample = _as_carrier(item, self.native) - yield from _worker_task_multi(sample, self.ops, native=self.native) + yield from _worker_task_multi(item, self.ops) def _iter_parallel(self) -> Iterator[Sample]: """Multiprocess execution engine.""" @@ -873,8 +512,7 @@ def _iter_parallel(self) -> Iterator[Sample]: with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: futures = [] for item in source: - sample = _as_carrier(item, self.native) - futures.append(executor.submit(_worker_task_multi, sample, self.ops, self.native)) + futures.append(executor.submit(_worker_task_multi, item, self.ops)) for future in futures: yield from future.result() @@ -886,17 +524,13 @@ def collect(self) -> List[Sample]: def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: """Yield pipeline-output Samples carrying only ``fields`` (the projection primitive). - Implements :class:`sampleflux.projection.SupportsProjection`. Flux must run - its op chain to produce each Sample (an op may consume the input), so this - is the generic "iterate, then drop unrequested fields" form — it cannot - skip input construction the way a leaf source (e.g. an image dataset that - reads only the label column) can. Lazy: a generator. ``fields`` is a - subset of ``{"input", "target", "metadata"}``. + Implements :class:`sampleflux.projection.SupportsProjection`. Flux must run its op + chain to produce each Sample (an op may consume the input), so this is the generic + "iterate, then keep only fields of the requested roles" form. ``fields`` is a subset + of ``{"input", "target", "metadata"}`` (mapped onto the ``input`` / ``target`` / + ``aux`` roles). Lazy: a generator. """ - want = frozenset(fields) + want_roles = {_PROJECTION_ROLES[f] for f in fields} for sample in self: - yield Sample( - input=sample.input if "input" in want else None, - target=sample.target if "target" in want else None, - metadata=sample.meta if "metadata" in want else {}, - ) + keep = [k for k in sample.keys() if sample.role_of(k) in want_roles] + yield Sample({k: sample[k] for k in keep}, {k: sample.role_of(k) for k in keep}) diff --git a/sampleflux/flow.py b/sampleflux/flow.py index d9fef07..fb8ce4d 100644 --- a/sampleflux/flow.py +++ b/sampleflux/flow.py @@ -6,7 +6,7 @@ write it); the flat context-ops form (:mod:`sampleflux.ops.context`) is the serial execution format the plain :class:`~sampleflux.core.Flux` engine runs. The two convert **bidirectionally**: :func:`to_ops` lowers a flow into a flat op list, :func:`from_ops` -lifts a flat op list back into a flow — with execution parity in both directions. +lifts a flat op list back — with execution parity in both directions. .. code-block:: yaml @@ -15,25 +15,22 @@ rescaled: !class:sampleflux.ops.numpy.RescaleOp() # input: previous step masked: !class:waivefront.SegmentOp() {from: spec} # 2nd reader of spec = fan-out thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: masked} - denoised: !class:waivefront.torchsig.processing.NoiseFloorOp() - from: rescaled - bind: {low_level: thresh} # per-sample param := thresh's result - out: {from: denoised, target_from: masked} # pure fan-in (no op) + out: {from: masked, merge_from: [rescaled]} # typed fan-in (no op) outputs: out -Step grammar (the four RESERVED step keys, stripped before the op is built): +Step grammar (the three RESERVED step keys, stripped before the op is built): - ``from:`` — the step supplying this step's input sample. Omitted = the previous step (the first step reads the source sample). Must name an EARLIER step: document order is the schedule, so forward references are errors and cycles are inexpressible. -- ``target_from:`` / ``metadata_from:`` — fan-in: compose the incoming sample's target / - metadata from another step's result before the op runs (the ``Mix`` slot semantics — - a Sample result contributes its corresponding field, metadata merges last-write-wins). +- ``merge_from:`` — typed fan-in: UNION another step's fields into this step's incoming + sample before the op runs (the ``MergeFields`` slot semantics — last-write-wins on a + key collision, in listed order). - ``bind:`` — ``{param: ref}`` per-sample parameters: ``ref`` is a step name (its result - sample's ``input``, or the raw value) or ``step.attr`` (the step op's live ``@output`` - after it ran — lowered through ``Capture``; stochastic-correct). + sample's primary input, ``step[key]`` for a named field, or the raw value) or + ``step.attr`` (the step op's live ``@output`` after it ran — lowered through ``Capture``). -A step may be a plain mapping with no op (``out: {from: a, target_from: b}``) — a pure +A step may be a plain mapping with no op (``out: {from: a, merge_from: [b]}``) — a pure fan-in/identity step; ``{}`` is the identity (used to give the source a referable name). ``outputs:`` names the step whose result the pipeline yields (default: the last step). @@ -51,13 +48,12 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.bag.sample import TypedSample, primary -from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Mix, Save, Use, _read_output -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample, primary +from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Save, Use, _read_output logger = get_logger(__name__) -RESERVED_STEP_KEYS = ("from", "target_from", "metadata_from", "merge_from", "bind") +RESERVED_STEP_KEYS = ("from", "merge_from", "bind") """Step-grammar keys stripped from a step mapping before the op is constructed.""" __all__ = ["FlowGraph", "FlowStep", "from_ops", "parse_flow", "to_ops", "RESERVED_STEP_KEYS"] @@ -69,8 +65,6 @@ class FlowStep(NamedTuple): name: str op: Optional[Any] # live op callable; None = pure fan-in / identity step from_: Optional[str] # None = previous step (first step: the source sample) - target_from: Optional[str] - metadata_from: Optional[str] bind: Dict[str, str] # param -> "step" | "step.attr" | "step[key]" merge_from: Tuple[str, ...] = () # typed fan-in: union these steps' FIELDS, in slot order @@ -80,7 +74,7 @@ class _BindRef(NamedTuple): step: str attr: Optional[str] # "step.attr" = the step op's @output attribute - key: Optional[str] # "step[key]" = the named FIELD of the step's TypedSample result + key: Optional[str] # "step[key]" = the named FIELD of the step's Sample result def _split_bind_ref(ref: str) -> _BindRef: @@ -109,7 +103,6 @@ def _check_reserved_collision(op: Any, step_name: str) -> None: Reserved keys are stripped from the step mapping before the op is built, so such a param could never be configured inline — fail loudly instead of silently stealing it. - (``from`` is a Python keyword and can never be a param, but the others could.) """ try: params = inspect.signature(type(op).__init__).parameters @@ -135,9 +128,7 @@ def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[Li Validates: step names carry no dots, every reference points to an EARLIER step. ``build=False`` keeps a marker step UNBUILT (the op stays a Fluid marker) — for - structural consumers (converters/importers) that must not materialize ops (hoisted - dotted ``!ref:`` values would resolve outside their document); such steps skip the - reserved-ctor-param check and their live construction happens at first call. + structural consumers (converters/importers) that must not materialize ops. """ if not isinstance(flow_doc, dict) or not flow_doc: raise ValueError("flow: expected a non-empty mapping of step-name -> op") @@ -181,14 +172,11 @@ def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[Li _check_reserved_collision(op, name) from_ = reserved.get("from") - target_from = reserved.get("target_from") - metadata_from = reserved.get("metadata_from") - for key, ref in (("from", from_), ("target_from", target_from), ("metadata_from", metadata_from)): - if ref is not None and str(ref) not in seen: - raise ValueError( - f"flow step {name!r}: {key}: {ref!r} does not name an EARLIER step " - f"(document order is the schedule; steps so far: {seen!r})" - ) + if from_ is not None and str(from_) not in seen: + raise ValueError( + f"flow step {name!r}: from: {from_!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) merge_raw = reserved.get("merge_from") merge_from: Tuple[str, ...] = () if merge_raw is not None: @@ -199,11 +187,6 @@ def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[Li f"flow step {name!r}: merge_from: {ref!r} does not name an EARLIER step " f"(document order is the schedule; steps so far: {seen!r})" ) - if target_from is not None or metadata_from is not None: - raise ValueError( - f"flow step {name!r}: merge_from (typed fan-in) and target_from/metadata_from " - "(legacy fan-in) are mutually exclusive on one step" - ) bind_raw = reserved.get("bind") or {} if not isinstance(bind_raw, dict): raise TypeError(f"flow step {name!r}: bind must be a mapping of param -> step[.output]") @@ -219,8 +202,6 @@ def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[Li name=name, op=op, from_=None if from_ is None else str(from_), - target_from=None if target_from is None else str(target_from), - metadata_from=None if metadata_from is None else str(metadata_from), bind=bind, merge_from=merge_from, ) @@ -238,10 +219,9 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T Slot granularity matters: one consumer step may read the SAME producer through several slots (its input AND a ``bind`` param), and only the ``"in"`` slot of the immediately - following step can ride the linear stream. Slots: ``"in"`` (input), ``"target"``, - ``"meta"``, ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A - ``bind`` step-result reference counts; an ``@output`` (``step.attr``) reference does - NOT (it reads the op instance, not the result cell). + following step can ride the linear stream. Slots: ``"in"`` (input), ``"merge"``, + ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A ``bind`` step-result + reference counts; an ``@output`` (``step.attr``) reference does NOT. """ readers: Dict[str, List[Tuple[int, str]]] = {s.name: [] for s in steps} for i, step in enumerate(steps): @@ -249,10 +229,6 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T source = step.from_ or implicit if source is not None: readers[source].append((i, "in")) - if step.target_from is not None: - readers[step.target_from].append((i, "target")) - if step.metadata_from is not None: - readers[step.metadata_from].append((i, "meta")) for ref in step.merge_from: readers[ref].append((i, "merge")) for ref in step.bind.values(): @@ -279,7 +255,7 @@ class FlowGraph(torch.utils.data.Dataset[Sample]): between the two is a pinned contract. Args: - source: Any iterable or indexable dataset (duck-typed) to wrap; ``None`` yields an empty stream. + source: Any iterable or indexable dataset (duck-typed) yielding ``Sample`` bags; ``None`` = empty stream. flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. outputs: Name of the step whose result is yielded. Blank (default) = the last step. chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single samples. @@ -371,63 +347,34 @@ def read_result(name: str, *, copy: bool) -> Any: for step in steps: # 1. the input sample (implicit stream reads move; explicit fan-out reads copy) if step.from_ is not None: - base = read_result(step.from_, copy=True) + sample = read_result(step.from_, copy=True) elif prev is not None: - base = read_result(prev, copy=False) + sample = read_result(prev, copy=False) else: - base = seed - # The TYPED carrier passes through verbatim; everything else coerces to Sample. - sample: Any = base if isinstance(base, TypedSample) else Sample.from_any(base) + sample = seed - # 2a. typed fan-in: UNION the merge_from steps' fields (slot order, last wins) + # 2. typed fan-in: UNION the merge_from steps' fields (slot order, last wins) if step.merge_from: - if not isinstance(sample, TypedSample): + if not isinstance(sample, Sample): raise TypeError( - f"flow step {step.name!r}: merge_from is the TYPED fan-in but the carrier is " - f"{type(sample).__name__} — use target_from/metadata_from for legacy Samples." + f"flow step {step.name!r}: merge_from is the typed fan-in but the carrier is " + f"{type(sample).__name__} — expected a Sample." ) merged = [sample] for ref in step.merge_from: value = read_result(ref, copy=True) - if not isinstance(value, TypedSample): + if not isinstance(value, Sample): raise TypeError( f"flow step {step.name!r}: merge_from step {ref!r} holds " - f"{type(value).__name__}, expected a TypedSample" + f"{type(value).__name__}, expected a Sample" ) merged.append(value) - sample = TypedSample.merge(*merged) - - # 2b. legacy fan-in slots (Mix semantics) - if step.target_from is not None or step.metadata_from is not None: - if isinstance(sample, TypedSample): - raise TypeError( - f"flow step {step.name!r}: target_from/metadata_from are the LEGACY fan-in " - "but the carrier is a TypedSample — use merge_from." - ) - metadata = dict(sample.meta) - target = sample.target - if step.target_from is not None: - value = read_result(step.target_from, copy=True) - target = value.target if isinstance(value, Sample) else value - if isinstance(value, Sample): - metadata.update(value.meta) - if step.metadata_from is not None: - value = read_result(step.metadata_from, copy=True) - extra = value.meta if isinstance(value, Sample) else value - if not isinstance(extra, dict): - raise TypeError( - f"flow step {step.name!r}: metadata_from holds {type(extra).__name__}, " - "expected a Sample or a dict" - ) - metadata.update(extra) - sample = sample._replace(target=target, metadata=metadata) + sample = Sample.merge(*merged) # 3. per-sample parameter binds if step.op is not None: op = step.op - from sampleflux.kinds import op_contract as _op_contract - - if _op_contract(op).expands: + if getattr(op, "EXPANDS", False): raise NotImplementedError( f"flow step {step.name!r}: {type(op).__name__!r} is a 1→N expanding op — " "FlowGraph steps are strictly 1→1 (a named-step env has one result per step). " @@ -445,11 +392,9 @@ def read_result(name: str, *, copy: bool) -> Any: ) else: value = read_result(parsed.step, copy=False) - if isinstance(value, TypedSample): + if isinstance(value, Sample): # "step[key]" = the named field; bare "step" = the primary input. value = value[parsed.key] if parsed.key else primary(value)[1] - elif isinstance(value, Sample): - value = value.input setattr(op, param, value) result = op(sample) if result is None: @@ -483,18 +428,12 @@ def _iter_samples(self) -> Iterator[Sample]: return assert self.source is not None for item in self.source: - seed = item if isinstance(item, TypedSample) else Sample.from_any(item) - result = self._run(seed) + result = self._run(item) if result is not None: yield result def _iter_parallel(self) -> Iterator[Sample]: - """Multiprocess execution — delegates to the serial engine over the LOWERED op list. - - Lowering + Flux's spawn pool is the sanctioned parallel path (one worker - implementation, guaranteed parity by the to_ops contract); a native process pool - here would duplicate it for no gain. - """ + """Multiprocess execution — delegates to the serial engine over the LOWERED op list.""" from sampleflux.core import Flux assert self.source is not None @@ -518,8 +457,7 @@ def __getitem__(self, index: int) -> Any: f"FlowGraph source {type(self.source).__name__} does not support indexing; " "wrap it in a list or use iteration." ) - seed = raw if isinstance(raw, TypedSample) else Sample.from_any(raw) - result = self._run(seed) + result = self._run(raw) if result is None: raise IndexError(f"Sample {index} filtered out by the flow") return result @@ -566,10 +504,6 @@ def to_ops(steps: Union[Sequence[FlowStep], Dict[str, Any]], outputs: str = "") outputs = outputs or (parsed[-1].name if parsed else "") readers = _result_readers(parsed, outputs) - # Which step results must live in a cell? Every read EXCEPT the one that can ride the - # linear stream: the immediately-next step's INPUT slot, or the final output read when - # this is the last step. Slot granularity matters — a consumer may read the same - # producer through its input slot AND a bind slot (only the input slot can stream). needs_cell: Dict[str, bool] = {} cell_reads_left: Dict[str, int] = {} for i, step in enumerate(parsed): @@ -608,7 +542,7 @@ def take_cell(name: str) -> Tuple[str, bool]: cell, last = take_cell(step.from_) ops.append(Use(name=cell, drop=last)) - # 2. fan-in slots — typed union (MergeFields) or the legacy Mix slots + # 2. fan-in slot — typed union (MergeFields) if step.merge_from: merge_drops: List[str] = [] merge_cells: List[str] = [] @@ -618,20 +552,6 @@ def take_cell(name: str) -> Tuple[str, bool]: if last: merge_drops.append(cell) ops.append(MergeFields(sources=merge_cells, drop=merge_drops)) - if step.target_from is not None or step.metadata_from is not None: - drops: List[str] = [] - kwargs: Dict[str, Any] = {} - if step.target_from is not None: - cell, last = take_cell(step.target_from) - kwargs["target_from"] = cell - if last: - drops.append(cell) - if step.metadata_from is not None: - cell, last = take_cell(step.metadata_from) - kwargs["metadata_from"] = cell - if last: - drops.append(cell) - ops.append(Mix(drop=drops, **kwargs)) # 3. the op, wrapped for binds (Apply) and @output captures (Capture) emitted: Optional[Any] = step.op @@ -659,7 +579,7 @@ def take_cell(name: str) -> Tuple[str, bool]: else: emitted = Capture(op=emitted, captures={a: f"{step.name}.{a}" for a in captures}) ops.append(emitted) - elif step.target_from is None and step.metadata_from is None and step.from_ is None and i == 0: + elif step.merge_from is None and step.from_ is None and i == 0: # identity first step ({}: names the source) — nothing to run pass @@ -687,7 +607,7 @@ def take_cell(name: str) -> Tuple[str, bool]: # --------------------------------------------------------------------------- -_CONTEXT_OP_CLASSES = (Save, Use, Drop, Apply, Capture, Mix, MergeFields) +_CONTEXT_OP_CLASSES = (Save, Use, Drop, Apply, Capture, MergeFields) def _ctx_view(raw: Any) -> Optional[type]: @@ -732,23 +652,19 @@ def from_ops(ops: Sequence[Any], outputs: str = "") -> Tuple[Dict[str, Any], str """Lift a flat op list into a ``(flow_mapping, outputs)`` pair. Context ops are absorbed into step grammar: ``Save`` names the preceding step (or an - identity first step for a source fork), ``Use`` starts a branch (``from:``), ``Mix`` - becomes ``target_from``/``metadata_from`` on the following step (or a pure fan-in - step), ``Apply``/``Capture`` unwrap into ``bind:`` references, and ``Drop`` vanishes - (liveness is recomputed on lowering). A plain linear list lifts to a linear flow with - auto-generated step names. The result round-trips: ``to_ops(from_ops(ops))`` is - execution-equivalent to ``ops``. - - Accepts LIVE ops or confluid ``Instance``/``Class`` MARKERS interchangeably (the - graph exporter lifts compiled marker lists without materializing them, keeping - hoisted-constant ``!ref:``\\ s intact); a real op arrives in the flow mapping verbatim - (marker in, marker out). + identity first step for a source fork), ``Use`` starts a branch (``from:``), + ``MergeFields`` becomes ``merge_from`` on the following step (or a pure fan-in step), + ``Apply``/``Capture`` unwrap into ``bind:`` references, and ``Drop`` vanishes (liveness + is recomputed on lowering). A plain linear list lifts to a linear flow with + auto-generated step names. The result round-trips. + + Accepts LIVE ops or confluid ``Instance``/``Class`` MARKERS interchangeably. """ flow_map: Dict[str, Dict[str, Any]] = {} taken: Dict[str, int] = {} prev_name: Optional[str] = None capture_cells: Dict[str, str] = {} # cell -> "step.attr" bind ref - pending: Dict[str, Any] = {} # accumulating step grammar (from/target_from/...) + pending: Dict[str, Any] = {} # accumulating step grammar (from/merge_from/...) def cell_ref(cell: str) -> str: """Map a cell name to its bind reference (an @output capture or a step result).""" @@ -776,7 +692,6 @@ def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: if prev_name is not None: # rename the just-flushed step to the cell name entry = flow_map.pop(prev_name) - # keep bind refs pointing at the old auto name consistent for e in flow_map.values(): b = e.get("bind") if isinstance(e, dict) else None if b: @@ -798,17 +713,6 @@ def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: if view is Use: pending["from"] = cell_ref(str(_ctx_field(raw, "name", ""))) continue - if view is Mix: - mix_grammar: Dict[str, Any] = {} - if _ctx_field(raw, "input_from", ""): - mix_grammar["from"] = cell_ref(str(_ctx_field(raw, "input_from"))) - if _ctx_field(raw, "target_from", ""): - mix_grammar["target_from"] = cell_ref(str(_ctx_field(raw, "target_from"))) - if _ctx_field(raw, "metadata_from", ""): - mix_grammar["metadata_from"] = cell_ref(str(_ctx_field(raw, "metadata_from"))) - pending.update(mix_grammar) - pending["__mix_pending__"] = True - continue if view is MergeFields: sources = [cell_ref(str(c)) for c in (_ctx_field(raw, "sources", None) or [])] pending["merge_from"] = sources @@ -837,7 +741,7 @@ def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: for cell, attr in captures.items(): capture_cells[cell] = f"{name}.{attr}" - # A trailing Mix (or Use) with no following op = a pure fan-in step. + # A trailing MergeFields (or Use) with no following op = a pure fan-in step. if pending: pending.pop("__mix_pending__", None) flush_step(None) diff --git a/sampleflux/kinds.py b/sampleflux/kinds.py deleted file mode 100644 index d726c01..0000000 --- a/sampleflux/kinds.py +++ /dev/null @@ -1,333 +0,0 @@ -"""Op-kind introspection — WHAT a transform processes and HOW it wants to be called. - -The taxonomy is a grid over two axes, detected from ``__call__``'s signature so ops stay -plain callables (no base classes) and a visual editor can surface the names later: - -**Field scope** (:data:`SampleKind`) — which part of the ``Sample(input, target, -metadata)`` triple the transform processes: - -======================== ========================== ============================ -scope without metadata with metadata -======================== ========================== ============================ -input only ``input`` (bare value) ``input_meta`` (`InputMeta`) -target only ``target`` (bare value) ``target_meta`` (`TargetMeta`) -both ``pair`` (`(input,target)`) ``sample`` (the full triple) -======================== ========================== ============================ - -plus ``value`` (a bare carrier of unknown role, runtime classification only) and ``any`` -(untyped — receives whatever flows, exactly today's behavior). - -**Call style** (:data:`CallStyle`) — packed (ONE argument: the ``Sample`` / a tuple / a -view) or unpacked (the fields as SEPARATE arguments): - -- ``__call__(self, sample: Sample)`` → sample, packed -- ``__call__(self, input, target, metadata)`` → sample, unpacked (3 required args) -- ``__call__(self, pair: tuple)`` / ``(p: Pair)`` → pair, packed -- ``__call__(self, input, target)`` → pair, unpacked (2 required args) -- ``__call__(self, v: InputMeta)`` → input_meta, packed -- ``__call__(self, input, metadata)`` → input_meta, unpacked (2nd arg named ``metadata``/``meta``) -- ``__call__(self, target, metadata)`` → target_meta, unpacked (1st arg named ``target``) -- ``__call__(self, x: Input)`` → input, bare value (`Input`/`Target` Annotated aliases, - or mark your own type: ``Annotated[np.ndarray, INPUT]``) -- untyped single argument → any (unchanged) - -The ENGINE binds the declared view from whatever carrier flows and merges the result -back, preserving untouched fields (see ``core._apply_op``). Arity counts REQUIRED -parameters only, so an existing op with optional extras keeps today's behavior. Explicit -class attributes (``SAMPLE_KIND_IN`` / ``SAMPLE_KIND_OUT`` / ``EXPANDS`` / -``CALL_STYLE``) override detection for callables introspection can't read. - -The same introspection powers 1→N detection: a ``-> Iterator[Sample]`` / -``-> Iterable[Sample]`` return annotation (or ``EXPANDS = True``) marks an EXPANDING op, -which makes the pipeline iterable-only (see ``Flux.__len__``/``__getitem__``). -""" - -import collections.abc -import inspect -from dataclasses import dataclass -from typing import Annotated, Any, Dict, Literal, Tuple, Union, get_args, get_origin, get_type_hints - -from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta - -SampleKind = Literal["sample", "pair", "input", "target", "metadata", "input_meta", "target_meta", "value", "any"] -"""The field-scope taxonomy — see the module docstring grid.""" - -CallStyle = Literal["packed", "unpacked"] -"""How the op wants its view: one packed argument, or the fields as separate arguments.""" - -SAMPLE_KINDS: Tuple[str, ...] = get_args(SampleKind) -CALL_STYLES: Tuple[str, ...] = get_args(CallStyle) - -_META_PARAM_NAMES = frozenset({"metadata", "meta"}) -_TARGET_PARAM_NAMES = frozenset({"target"}) - -__all__ = [ - "CALL_STYLES", - "CallStyle", - "INPUT", - "Input", - "METADATA", - "MetaDict", - "OpContract", - "SAMPLE_KINDS", - "SampleKind", - "TARGET", - "Target", - "classify_carrier", - "op_contract", -] - -_EXPANDING_ORIGINS = ( - list, - set, - frozenset, - collections.abc.Iterable, - collections.abc.Iterator, - collections.abc.Generator, - collections.abc.Sequence, -) - - -class _KindMark: - """PEP-593 marker naming the field a bare-value annotation binds (``Annotated[T, INPUT]``).""" - - __slots__ = ("kind",) - - def __init__(self, kind: str) -> None: - self.kind = kind - - def __repr__(self) -> str: # pragma: no cover - debug aid - return f"KindMark({self.kind})" - - -INPUT = _KindMark("input") -TARGET = _KindMark("target") -METADATA = _KindMark("metadata") - -Input = Annotated[Any, INPUT] -"""Annotation alias: the op processes the BARE input value (``Annotated[T, INPUT]`` keeps a real T).""" - -Target = Annotated[Any, TARGET] -"""Annotation alias: the op processes the BARE target value (``Annotated[T, TARGET]`` keeps a real T).""" - -MetaDict = Annotated[Any, METADATA] -"""Annotation alias: the op processes the metadata DICT (a plain ``dict`` annotation works too).""" - - -@dataclass(frozen=True) -class OpContract: - """What an op consumes/produces, how it is called, and whether it expands 1→N. - - For an UNPACKED op, ``bindings`` lists each required parameter's field scope in - order (e.g. ``("input_meta", "target_meta")`` for ``f(im: InputMeta, tm: TargetMeta)``) - — the engine binds each argument independently and merges each returned element back. - ``accepts`` stays the grid SUMMARY of the covered fields (what a visual editor surfaces). - """ - - accepts: SampleKind = "any" - produces: SampleKind = "any" - expands: bool = False - style: CallStyle = "packed" - bindings: Tuple[str, ...] = () - - -_ANY_CONTRACT = OpContract() -_contract_cache: Dict[type, OpContract] = {} - - -def classify_carrier(obj: Any) -> SampleKind: - """The carrier kind of a runtime object. - - The named views are checked BEFORE the generic tuple rule — an ``InputMeta`` IS a - 2-tuple and would otherwise misclassify as a pair. - """ - if isinstance(obj, Sample): - return "sample" - if isinstance(obj, InputMeta): - return "input_meta" - if isinstance(obj, TargetMeta): - return "target_meta" - if isinstance(obj, Pair): - return "pair" - if isinstance(obj, tuple) and len(obj) == 2: - return "pair" - return "value" - - -def _unwrap_optional(anno: Any) -> Any: - """``Optional[X]`` / ``Union[X, None]`` -> ``X`` (multi-arm unions are left as-is).""" - if get_origin(anno) is Union: - args = [a for a in get_args(anno) if a is not type(None)] - if len(args) == 1: - return args[0] - return anno - - -def _kind_mark_of(anno: Any) -> Any: - """The ``_KindMark`` on an ``Annotated[...]`` layer, or None.""" - if get_origin(anno) is Annotated: - for meta in get_args(anno)[1:]: - if isinstance(meta, _KindMark): - return meta - return None - - -def _kind_of(anno: Any) -> SampleKind: - """The field scope an annotation names; unknown/absent/Any -> ``any``.""" - anno = _unwrap_optional(anno) - mark = _kind_mark_of(anno) - if mark is not None: - return mark.kind # type: ignore[no-any-return] - if get_origin(anno) is Annotated: - return _kind_of(get_args(anno)[0]) - if anno is inspect.Parameter.empty or anno is Any or anno is None: - return "any" - if anno is Sample: - return "sample" - if anno is InputMeta: - return "input_meta" - if anno is TargetMeta: - return "target_meta" - if anno is Pair: - return "pair" - if anno is tuple or get_origin(anno) is tuple: - return "pair" - if anno is dict or get_origin(anno) is dict: - return "metadata" - if isinstance(anno, type) and issubclass(anno, Sample): - return "sample" - return "any" - - -def _is_meta_annotation(anno: Any) -> bool: - """True when an annotation names a metadata dict (``Dict[str, ...]`` / ``dict``).""" - anno = _unwrap_optional(anno) - return anno is dict or get_origin(anno) is dict - - -def _return_contract(anno: Any) -> Tuple[SampleKind, bool]: - """(produced kind, expands) from a return annotation.""" - anno = _unwrap_optional(anno) - origin = get_origin(anno) - if origin in _EXPANDING_ORIGINS or (isinstance(anno, type) and anno in _EXPANDING_ORIGINS): - args = get_args(anno) - element = args[0] if args else Any - return _kind_of(element), True - return _kind_of(anno), False - - -# Per-parameter binding vocabulary: which field scope one argument of an UNPACKED op binds. -_PARAM_BINDINGS = ("input", "target", "metadata", "input_meta", "target_meta") -# Positional defaults — the classic AI convention: f(input, target[, metadata]). -_POSITIONAL_DEFAULTS = ("input", "target", "metadata") -_INPUT_PARAM_NAMES = frozenset({"input"}) - -# Which fields each binding covers, for the grid summary. -_BINDING_FIELDS: Dict[str, frozenset] = { - "input": frozenset({"i"}), - "target": frozenset({"t"}), - "metadata": frozenset({"m"}), - "input_meta": frozenset({"i", "m"}), - "target_meta": frozenset({"t", "m"}), -} -_FIELDS_TO_KIND: Dict[frozenset, str] = { - frozenset({"i", "t", "m"}): "sample", - frozenset({"i", "t"}): "pair", - frozenset({"i", "m"}): "input_meta", - frozenset({"t", "m"}): "target_meta", - frozenset({"i"}): "input", - frozenset({"t"}): "target", - frozenset({"m"}): "metadata", -} - - -def _param_binding(param: Any, anno: Any, position: int) -> str: - """One required parameter's field binding: annotation wins, then the name, then position. - - Positional defaults are the classic ``f(input, target, metadata)`` convention, so an - unannotated/unnamed multi-arg op keeps the old behavior; a view annotation - (``InputMeta``/``TargetMeta``/``Input``/``Target``/``dict``) or a recognised name - (``input``/``target``/``metadata``/``meta``) overrides its slot. - """ - kind = _kind_of(anno) - if kind in _PARAM_BINDINGS: - return kind - if param.name in _INPUT_PARAM_NAMES: - return "input" - if param.name in _TARGET_PARAM_NAMES: - return "target" - if param.name in _META_PARAM_NAMES: - return "metadata" - return _POSITIONAL_DEFAULTS[position] - - -def _bindings_summary(bindings: Tuple[str, ...]) -> SampleKind: - """The grid-summary kind of a binding list (the covered fields).""" - covered: frozenset = frozenset().union(*(_BINDING_FIELDS[b] for b in bindings)) - return _FIELDS_TO_KIND.get(covered, "sample") # type: ignore[return-value] - - -def op_contract(op: Any) -> OpContract: - """The introspected (cached per type) contract of an op — scope, call style, expansion. - - Explicit class attributes win: ``SAMPLE_KIND_IN`` / ``SAMPLE_KIND_OUT`` (a - :data:`SampleKind`), ``CALL_STYLE`` (a :data:`CallStyle`), and ``EXPANDS`` (bool) — - the escape hatch for callables introspection can't read. Annotation resolution - failures degrade to ``any``/packed so an untyped or exotic op behaves exactly as - today. Arity counts REQUIRED parameters (no default) only, so an op with optional - extras after its sample argument keeps single-argument semantics. - """ - cls = type(op) - cached = _contract_cache.get(cls) - if cached is not None: - return _explicit_overrides(op, cached) - - accepts: SampleKind = "any" - produces: SampleKind = "any" - expands = False - style: CallStyle = "packed" - bindings: Tuple[str, ...] = () - call = getattr(cls, "__call__", None) - if call is not None: - try: - signature = inspect.signature(call) - hints = get_type_hints(call, include_extras=True) - except Exception: # noqa: BLE001 - degrade to "any" on ANY introspection failure - signature, hints = None, {} - if signature is not None: - params = [ - p - for name, p in signature.parameters.items() - if name != "self" and p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) - ] - required = [p for p in params if p.default is inspect.Parameter.empty] - arity = len(required) if required else min(len(params), 1) - if arity == 1 and params: - accepts = _kind_of(hints.get(params[0].name, params[0].annotation)) - elif 2 <= arity <= 3: - bindings = tuple(_param_binding(p, hints.get(p.name, p.annotation), i) for i, p in enumerate(required)) - accepts, style = _bindings_summary(bindings), "unpacked" - # arity 0 or > 3: leave "any"/packed — the op is called with the carrier verbatim. - produces, expands = _return_contract(hints.get("return", inspect.Parameter.empty)) - - bindings = bindings if style == "unpacked" else () - contract = OpContract(accepts=accepts, produces=produces, expands=expands, style=style, bindings=bindings) - _contract_cache[cls] = contract - return _explicit_overrides(op, contract) - - -def _explicit_overrides(op: Any, base: OpContract) -> OpContract: - """Apply the ``SAMPLE_KIND_IN``/``SAMPLE_KIND_OUT``/``EXPANDS``/``CALL_STYLE`` escape hatches.""" - kind_in = getattr(op, "SAMPLE_KIND_IN", None) - kind_out = getattr(op, "SAMPLE_KIND_OUT", None) - expands = getattr(op, "EXPANDS", None) - call_style = getattr(op, "CALL_STYLE", None) - if kind_in is None and kind_out is None and expands is None and call_style is None: - return base - return OpContract( - accepts=kind_in if kind_in in SAMPLE_KINDS else base.accepts, - produces=kind_out if kind_out in SAMPLE_KINDS else base.produces, - expands=bool(expands) if expands is not None else base.expands, - style=call_style if call_style in CALL_STYLES else base.style, - bindings=base.bindings, - ) diff --git a/sampleflux/labels.py b/sampleflux/labels.py index 57c4289..0021761 100644 --- a/sampleflux/labels.py +++ b/sampleflux/labels.py @@ -26,7 +26,7 @@ from confluid import configurable -from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp +from sampleflux.ops.target import DecodeTarget, EncodeTarget @configurable @@ -73,13 +73,13 @@ def inverse(self) -> Dict[int, str]: """``id → name`` lookup (the inverse of :attr:`mapping`).""" return {v: k for k, v in self._require().items()} - def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTargetOp: - """Return an :class:`~sampleflux.ops.target.EncodeTargetOp` that maps name → id via this map.""" - return EncodeTargetOp(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) + def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTarget: + """Return an :class:`~sampleflux.ops.target.EncodeTarget` transform that maps name → id via this map.""" + return EncodeTarget(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) - def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTargetOp: - """Return a :class:`~sampleflux.ops.target.DecodeTargetOp` that maps id → name via this map.""" - return DecodeTargetOp(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) + def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTarget: + """Return a :class:`~sampleflux.ops.target.DecodeTarget` transform that maps id → name via this map.""" + return DecodeTarget(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) @classmethod def fit(cls, targets: Iterable[Any]) -> "LabelMap": diff --git a/sampleflux/ops/__init__.py b/sampleflux/ops/__init__.py index 87508f8..2c69b7a 100644 --- a/sampleflux/ops/__init__.py +++ b/sampleflux/ops/__init__.py @@ -1,83 +1,74 @@ """ -SampleFlux operations. +SampleFlux operations (typed-bag :class:`~sampleflux.Sample` transforms). Submodules: - - sampleflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, - ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp, SqueezeOp, - UnsqueezeOp, MinOp, MaxOp, MedianOp, PercentileOp, StatsOp (ndarray) - - sampleflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp, SqueezeOp, - UnsqueezeOp (tensor) + - sampleflux.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / + connected_component_bboxes / resolve_expression helpers) + - sampleflux.ops.torch: ToTensor (+ to_tensor helper) + - sampleflux.ops.image: ConvertToImage (+ value_to_image / normalize_to_uint8 …) + - sampleflux.ops.target: MetadataToTarget, EncodeTarget, DecodeTarget, + CocoToTorchVisionDetection, MasksToDetectionBoxes + - sampleflux.ops.structure: SetRole, RenameField, DropField, CopyField, SelectFields - sampleflux.ops.parallel: Parallel (worker-pool sub-pipeline) - sampleflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - sampleflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - - sampleflux.ops.configure: ConfigureOp (per-sample parameter injection — the helios Configure pattern) - - sampleflux.ops.formula: FormulaOp (math formula over sample.input — the Math node's op form) + - sampleflux.ops.configure: ConfigureOp (per-sample parameter injection) + - sampleflux.ops.formula: FormulaOp (math formula over the primary input) - sampleflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - sampleflux.ops.transform_chain: TransformChain (sequential op-chain grouping) - - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, Mix (per-sample Context - graph plane — the flat-list building blocks a branchy flow: document lowers to) - - sampleflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - - sampleflux.ops.swap: SwapInputTargetOp - - sampleflux.ops.stash: StashInputOp, UnstashInputOp, StashTargetOp, UnstashTargetOp - (metadata-bus snapshots — only for crossing a Parallel boundary or persisting - a snapshot into a sink; graph wiring uses sampleflux.ops.context) - - sampleflux.ops.target: MetadataToTargetOp, EncodeTargetOp, DecodeTargetOp (target field) - -Flat imports default to torch variants for the data ops; flow / copy / -swap / stash / target utilities are field-agnostic. + - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-sample + Context graph plane — the flat-list building blocks a branchy flow: document lowers to) + - sampleflux.ops.debug: PrintSampleOp (per-sample summary probe) """ from sampleflux.ops.configure import ConfigureOp -from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use -from sampleflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp +from sampleflux.ops.context import Apply, Capture, Drop, MergeFields, Save, Use +from sampleflux.ops.debug import PrintSampleOp from sampleflux.ops.enable import Enable from sampleflux.ops.formula import FormulaOp +from sampleflux.ops.image import ConvertToImage +from sampleflux.ops.numpy import ConnectedComponents, Threshold from sampleflux.ops.parallel import Parallel from sampleflux.ops.random_apply import RandomApply from sampleflux.ops.sink import SampleSinkOp -from sampleflux.ops.stash import StashInputOp, StashTargetOp, UnstashInputOp, UnstashTargetOp -from sampleflux.ops.swap import SwapInputTargetOp +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole from sampleflux.ops.target import ( - CocoToTorchVisionDetectionOp, - DecodeTargetOp, - EncodeTargetOp, - MasksToDetectionBoxesOp, - MetadataToTargetOp, + CocoToTorchVisionDetection, + DecodeTarget, + EncodeTarget, + MasksToDetectionBoxes, + MetadataToTarget, ) -from sampleflux.ops.torch import RescaleOp, SqueezeOp, StandardizeOp, ToTensorOp, UnsqueezeOp +from sampleflux.ops.torch import ToTensor from sampleflux.ops.transform_chain import TransformChain __all__ = [ - "ConfigureOp", - "CopyInputOp", - "CopyMetadataOp", - "CopySampleOp", "Apply", "Capture", - "CopyTargetOp", - "DecodeTargetOp", + "CocoToTorchVisionDetection", + "ConfigureOp", + "ConnectedComponents", + "ConvertToImage", + "CopyField", + "DecodeTarget", "Drop", + "DropField", "Enable", - "Mix", - "Save", - "Use", + "EncodeTarget", "FormulaOp", - "EncodeTargetOp", - "MetadataToTargetOp", - "CocoToTorchVisionDetectionOp", - "MasksToDetectionBoxesOp", + "MasksToDetectionBoxes", + "MergeFields", + "MetadataToTarget", "Parallel", + "PrintSampleOp", "RandomApply", - "RescaleOp", + "RenameField", + "Save", "SampleSinkOp", - "SqueezeOp", - "StandardizeOp", - "StashInputOp", - "StashTargetOp", - "SwapInputTargetOp", + "SelectFields", + "SetRole", + "Threshold", + "ToTensor", "TransformChain", - "ToTensorOp", - "UnstashInputOp", - "UnstashTargetOp", - "UnsqueezeOp", + "Use", ] diff --git a/sampleflux/ops/albumentations.py b/sampleflux/ops/albumentations.py index f05dd38..d219147 100644 --- a/sampleflux/ops/albumentations.py +++ b/sampleflux/ops/albumentations.py @@ -25,13 +25,14 @@ :class:`~sampleflux.ops.torchvision.TorchvisionTransformOp`, which emits CHW torch tensors. """ -from typing import Any, Dict, List, Literal, Optional, Tuple +from typing import Any, List, Literal, Optional import numpy as np from confluid import configurable from loggair import get_logger -from sampleflux.sample import Sample +from sampleflux.bag.items import Mask, item_data, with_data +from sampleflux.bag.sample import Sample, primary logger = get_logger(__name__) @@ -159,41 +160,23 @@ def pipeline(self) -> Any: return self._pipeline def __call__(self, sample: Sample) -> Sample: - image = _as_array(sample.input) + key, item = primary(sample, "input") + image = _as_array(item_data(item)) if self.target == "mask": - out = self.pipeline(image=image, mask=_as_array(sample.target)) - return sample._replace(input=out["image"], target=out["mask"]) + mask_field = next(iter(sample.items_of_type(Mask)), None) + if mask_field is None: + raise ValueError("AlbumentationsOp(target='mask'): no Mask field in the sample to transform jointly.") + mkey, mitem = mask_field + out = self.pipeline(image=image, mask=_as_array(item_data(mitem))) + result = sample.replace_field(key, with_data(item, out["image"])) + return result.replace_field(mkey, with_data(mitem, out["mask"])) if self.target == "boxes": - new_input, new_target = self._apply_boxes(image, sample.target) - return sample._replace(input=new_input, target=new_target) - out = self.pipeline(image=image) - return sample._replace(input=out["image"]) - - def _apply_boxes(self, image: np.ndarray, target: Any) -> Tuple[Any, Dict[str, Any]]: - """Route the torchvision detection dict through albumentations' bbox machinery.""" - import torch - - pipeline = self.pipeline - if not isinstance(target, dict) or "boxes" not in target or "labels" not in target: - raise TypeError( - f"AlbumentationsOp(target='boxes'): sample.target must be the torchvision detection " - f"dict {{'boxes': [N,4] xyxy, 'labels': [N]}}; got {type(target).__name__}. Wire " - "CocoToTorchVisionDetectionOp / MasksToDetectionBoxesOp upstream." - ) - if "bboxes" not in getattr(pipeline, "processors", {}): - raise ValueError( - "AlbumentationsOp(target='boxes'): the prebuilt Compose was built without bbox_params. " - "Construct it as A.Compose([...], bbox_params=A.BboxParams(format='pascal_voc', " - "label_fields=['labels'])) — or pass 'transforms' and let the op add them." + raise NotImplementedError( + "AlbumentationsOp(target='boxes') is not yet ported to the typed-bag Regions target " + "(migration follow-up); use target='none' or 'mask'." ) - boxes = np.asarray(target["boxes"], dtype=np.float32).reshape(-1, 4) - labels = [int(v) for v in np.asarray(target["labels"]).reshape(-1)] - out = pipeline(image=image, bboxes=boxes.tolist(), labels=labels) - out_boxes = np.asarray(out["bboxes"], dtype=np.float32).reshape(-1, 4) - new_target = dict(target) - new_target["boxes"] = torch.as_tensor(out_boxes, dtype=torch.float32) - new_target["labels"] = torch.as_tensor(list(out["labels"]), dtype=torch.int64).reshape(-1) - return out["image"], new_target + out = self.pipeline(image=image) + return sample.replace_field(key, with_data(item, out["image"])) __all__ = ["AlbumentationsOp", "TargetMode"] diff --git a/sampleflux/ops/configure.py b/sampleflux/ops/configure.py index d5cec8f..0647ed3 100644 --- a/sampleflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -17,7 +17,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.sample import Sample +from sampleflux.bag.items import item_data +from sampleflux.bag.sample import Sample, primary @configurable(category="op", group="compose") @@ -87,8 +88,8 @@ def __call__(self, sample: Sample) -> Optional[Sample]: if result is None: return None # the compute chain filtered the sample (FilterOp semantics) current = result - value = current.input - sample.meta[self.key or self.param] = value + # The computed value is the primary input item's payload of the side-branch result. + value = item_data(primary(current)[1]) target = cast(Any, self.target) setattr(target, self.param, value) return _apply_op(sample, target) diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py index 1e7de0d..f834c2c 100644 --- a/sampleflux/ops/context.py +++ b/sampleflux/ops/context.py @@ -19,9 +19,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.bag.sample import TypedSample, primary +from sampleflux.bag.sample import Sample, primary from sampleflux.context import require -from sampleflux.sample import Sample _MISSING = object() @@ -47,18 +46,15 @@ def _read_output(op: Any, name: str) -> Any: return _MISSING -def _cell_field(value: Any, field: str, key: str = "") -> Any: +def _cell_field(value: Any, field: str = "input", key: str = "") -> Any: """A cell's contribution to a value slot. - A legacy ``Sample`` cell contributes its named field; a ``TypedSample`` cell - contributes the ``key``-named item when ``key`` is given, else its PRIMARY input-role - item (:func:`~sampleflux.bag.sample.primary` — the sanctioned "the input" accessor); - a raw cell value is used verbatim. + A :class:`~sampleflux.bag.sample.Sample` cell contributes the ``key``-named item when + ``key`` is given, else its PRIMARY input-role item (:func:`primary` — the sanctioned + "the input" accessor); a raw cell value is used verbatim. """ - if isinstance(value, TypedSample): - return value[key] if key else primary(value)[1] if isinstance(value, Sample): - return getattr(value, field) + return value[key] if key else primary(value)[1] return value @@ -114,9 +110,7 @@ def __call__(self, sample: Any) -> Any: ctx.delete(self.name) else: value = deepcopy(value) - if isinstance(value, TypedSample): - return value # the typed carrier passes through verbatim (never coerced) - return Sample.from_any(value) + return value @configurable(category="op", group="structure") @@ -159,7 +153,7 @@ class Apply: op: The op to configure and apply; required at call time, validated lazily. param: Attribute name on ``op`` to set with the cell value; required at call time. source: Context cell holding the value; required at call time, validated lazily. - key: For a TypedSample cell — the named field to contribute. Blank (default) = the primary input field. + key: For a Sample cell — the named field to contribute. Blank (default) = the primary input field. drop: When True, free the source cell after reading it. """ @@ -274,75 +268,12 @@ def close(self) -> None: close_fn() -@configurable(category="op", group="structure") -class Mix: - """Fan-in: compose one sample from Context cells and the incoming stream sample. - - Each named slot reads its cell — a Sample cell contributes its corresponding field, a - raw cell value is used verbatim — while an unnamed slot keeps the incoming sample's - field. Metadata merges incoming-first, then each named cell's metadata in slot order - (``input_from``, ``target_from``, ``metadata_from`` — last write wins), so branch - traceability survives the merge and ``metadata_from`` has the final say. - - Args: - input_from: Context cell providing the mixed ``input``. Blank (default) = keep the incoming input. - target_from: Context cell providing the mixed ``target``. Blank (default) = keep the incoming target. - metadata_from: Context cell whose metadata merges LAST (wins conflicts). Blank (default) = none. - drop: Context cells to free after mixing (defaults to none). - """ - - def __init__( - self, - input_from: str = "", - target_from: str = "", - metadata_from: str = "", - drop: Optional[List[str]] = None, - ) -> None: - # Lazy / zero-arg: store config only; cell names are resolved at first call. - self.input_from = str(input_from) - self.target_from = str(target_from) - self.metadata_from = str(metadata_from) - self.drop = list(drop) if drop else [] - - def __call__(self, sample: Sample) -> Sample: - ctx = require("Mix") - - mixed_input = sample.input - mixed_target = sample.target - metadata: Dict[str, Any] = dict(sample.meta) - - for cell_name, field in ((self.input_from, "input"), (self.target_from, "target")): - if not cell_name: - continue - value = ctx.get(cell_name) - if field == "input": - mixed_input = _cell_field(value, "input") - else: - mixed_target = _cell_field(value, "target") - if isinstance(value, Sample): - metadata.update(value.meta) - if self.metadata_from: - value = ctx.get(self.metadata_from) - extra = value.meta if isinstance(value, Sample) else value - if not isinstance(extra, dict): - raise TypeError( - f"Mix: metadata_from cell {self.metadata_from!r} holds {type(extra).__name__}, " - "expected a Sample or a dict" - ) - metadata.update(extra) - - for cell_name in self.drop: - ctx.delete(cell_name) - - return Sample(input=mixed_input, target=mixed_target, metadata=metadata) - - @configurable(category="op", group="structure") class MergeFields: - """Typed fan-in: UNION the named cells' fields into the incoming :class:`TypedSample`. + """Typed fan-in: UNION the named cells' fields into the incoming :class:`Sample`. The typed replacement for :class:`Mix`'s metadata dict-merge: each source cell (a - ``TypedSample`` saved by an earlier branch) contributes its FIELDS and ROLES, united in + ``Sample`` saved by an earlier branch) contributes its FIELDS and ROLES, united in listed order with last-write-wins on a key collision (the deterministic slot-order rule; avoid a deliberate collision by renaming on the producing branch — ``sampleflux.ops.structure.RenameField``). ``keys`` selects a subset of a source's @@ -365,24 +296,24 @@ def __init__( self.keys = list(keys) if keys else [] self.drop = list(drop) if drop else [] - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if not self.sources: raise ValueError("MergeFields: 'sources' (the context cells to union) is required") - if not isinstance(sample, TypedSample): + if not isinstance(sample, Sample): raise TypeError( - f"MergeFields: the incoming carrier is {type(sample).__name__}, expected TypedSample — " + f"MergeFields: the incoming carrier is {type(sample).__name__}, expected Sample — " "typed fan-in unions named fields (legacy Sample fan-in is Mix)." ) ctx = require("MergeFields") merged = sample for cell_name in self.sources: value = ctx.get(cell_name) - if not isinstance(value, TypedSample): - raise TypeError(f"MergeFields: cell {cell_name!r} holds {type(value).__name__}, expected a TypedSample") + if not isinstance(value, Sample): + raise TypeError(f"MergeFields: cell {cell_name!r} holds {type(value).__name__}, expected a Sample") if self.keys: keep = [k for k in self.keys if k in value] - value = TypedSample({k: value[k] for k in keep}, {k: value.role_of(k) for k in keep}) - merged = TypedSample.merge(merged, value) + value = Sample({k: value[k] for k in keep}, {k: value.role_of(k) for k in keep}) + merged = Sample.merge(merged, value) for cell_name in self.drop: ctx.delete(cell_name) return merged diff --git a/sampleflux/ops/copy.py b/sampleflux/ops/copy.py deleted file mode 100644 index a4b342d..0000000 --- a/sampleflux/ops/copy.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Defensive deepcopy ops. - -Use these when a downstream op mutates ``sample.input`` / ``sample.target`` -in place and you want later readers (or external references) to see the -pre-mutation value — e.g. before handing a sample to an in-place library -call, or to decouple a snapshot from the live stream. -""" - -import copy -from typing import Any - -from confluid import configurable - -from sampleflux.sample import Sample - - -@configurable(category="op", group="structure") -class CopySampleOp: - """Deepcopy of input, target, and metadata.""" - - def __call__(self, sample: Sample) -> Sample: - return Sample( - input=copy.deepcopy(sample.input), - target=copy.deepcopy(sample.target), - metadata=copy.deepcopy(sample.meta), - ) - - -@configurable(category="op", group="structure") -class CopyInputOp: - """Deepcopy ``sample.input``.""" - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=copy.deepcopy(sample.input)) - - -@configurable(category="op", group="structure") -class CopyTargetOp: - """Deepcopy ``sample.target``.""" - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(target=copy.deepcopy(sample.target)) - - -@configurable(category="op", group="structure") -class CopyMetadataOp: - """Deepcopy ``sample.meta``. - - The replacement dict is a fresh object, so subsequent in-place writes - on the new metadata won't be seen by other holders of the old dict. - """ - - def __call__(self, sample: Sample) -> Sample: - new_meta: Any = copy.deepcopy(sample.meta) - return sample._replace(metadata=new_meta) diff --git a/sampleflux/ops/debug.py b/sampleflux/ops/debug.py index af56a9e..287da4a 100644 --- a/sampleflux/ops/debug.py +++ b/sampleflux/ops/debug.py @@ -5,7 +5,8 @@ from confluid import configurable from loggair import get_logger -from sampleflux.sample import Sample +from sampleflux.bag.items import item_data +from sampleflux.bag.sample import Sample logger = get_logger(__name__) @@ -104,9 +105,11 @@ def __call__(self, sample: Sample) -> Sample: def _format(self, sample: Sample) -> str: parts = [f"[{self.label} #{self._count}]"] - if self.include_data: - parts.append(f"input={_summarize(sample.input)}") - parts.append(f"target={_summarize(sample.target)}") - if self.include_metadata: - parts.append(f"metadata={_summarize_metadata(sample.metadata)}") + for key in sample.keys(): + role = sample.role_of(key) + if role == "aux" and not self.include_metadata: + continue + if role != "aux" and not self.include_data: + continue + parts.append(f"{key}[{role}]={_summarize(item_data(sample[key]))}") return " ".join(parts) diff --git a/sampleflux/ops/enable.py b/sampleflux/ops/enable.py index 6e1d967..b194fd5 100644 --- a/sampleflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -11,7 +11,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample logger = get_logger(__name__) diff --git a/sampleflux/ops/formula.py b/sampleflux/ops/formula.py index 82da10f..accbb7f 100644 --- a/sampleflux/ops/formula.py +++ b/sampleflux/ops/formula.py @@ -14,7 +14,8 @@ from confluid import configurable -from sampleflux.sample import Sample +from sampleflux.bag.items import item_data, with_data +from sampleflux.bag.sample import Sample, primary # Every public ``math`` symbol + the scalar built-in helpers, mirroring the canvas Math # node's namespace. The bound variable shadows same-named constants (e.g. ``e``). @@ -39,9 +40,10 @@ def __init__(self, formula: str = "a", var: str = "a") -> None: def __call__(self, sample: Sample) -> Sample: if not self.formula.strip(): raise ValueError("FormulaOp: 'formula' must be a non-empty expression") - namespace = {**_FORMULA_NAMESPACE, self.var: sample.input} + key, item = primary(sample, "input") + namespace = {**_FORMULA_NAMESPACE, self.var: item_data(item)} try: value = eval(self.formula, {"__builtins__": {}}, namespace) # noqa: S307 - restricted namespace except Exception as exc: raise ValueError(f"FormulaOp: formula {self.formula!r} failed: {exc}") from exc - return sample._replace(input=value) + return sample.replace_field(key, with_data(item, value)) diff --git a/sampleflux/ops/image.py b/sampleflux/ops/image.py index 733341b..3cab695 100644 --- a/sampleflux/ops/image.py +++ b/sampleflux/ops/image.py @@ -28,15 +28,35 @@ from sampleflux.bag.items import Image as ImageItem from sampleflux.bag.items import NDArrayItem, item_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample, primary from sampleflux.bag.transform import Transform -from sampleflux.sample import Sample -from sampleflux.typespec import ArrayType as _ArrayType -from sampleflux.typespec import PythonType, SampleType, UnionType logger = get_logger("sampleflux.ops.image") +def normalize_to_uint8( + arr: np.ndarray, + vmin: Optional[float] = None, + vmax: Optional[float] = None, +) -> np.ndarray: + """Min-max normalize ``arr`` to ``uint8`` in ``[0, 255]``. + + ``vmin`` / ``vmax`` pin the scale when given (clamping out-of-range values); otherwise the + array's finite min / max are used. Non-finite entries are folded to the bounds; a degenerate + range yields all-zeros. The single quantization source of truth (the 2-D-map / float-array + paths of :func:`value_to_image` call it directly). + """ + arr = np.asarray(arr).astype(np.float32) + finite = arr[np.isfinite(arr)] + lo = float(vmin) if vmin is not None else (float(finite.min()) if finite.size else 0.0) + hi = float(vmax) if vmax is not None else (float(finite.max()) if finite.size else 0.0) + if hi <= lo: + return np.zeros(arr.shape, dtype=np.uint8) + filled = np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) + norm = (filled - lo) / (hi - lo) + return np.asarray(np.clip(norm, 0.0, 1.0) * 255.0, dtype=np.uint8) + + # Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob # across the workspace (``value_to_image`` / ``sample_to_image`` / ``ConvertToImageOp`` and, via # re-export, waivefront's renderers) AND for GUI colormap dropdowns (which read ``COLORMAPS``). @@ -116,7 +136,7 @@ def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: arr = arr.astype(np.uint8) * 255 if arr.ndim == 2: - return np.array(_apply_colormap(NormalizeToUint8Op.normalize_to_uint8(arr), colormap)) + return np.array(_apply_colormap(normalize_to_uint8(arr), colormap)) if arr.ndim == 3: # Normalize channel position to trailing (HWC). if arr.shape[0] in (1, 3, 4) and arr.shape[2] not in (1, 3, 4): @@ -130,7 +150,7 @@ def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: arr = arr[..., :3] else: # 2 channels (or other) — replicate the first arr = np.repeat(arr[..., :1], 3, axis=2) - return arr if arr.dtype == np.uint8 else NormalizeToUint8Op.normalize_to_uint8(arr) + return arr if arr.dtype == np.uint8 else normalize_to_uint8(arr) return _text_to_image(f"input ndim={arr.ndim}, shape={arr.shape}") @@ -175,19 +195,18 @@ def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 5 def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: - """Render ``sample.input`` to an ``(H, W, 3)`` uint8 RGB image for display. + """Render a sample's primary input to an ``(H, W, 3)`` uint8 RGB image for display. - Thin wrapper over :func:`value_to_image` (which does the modality-agnostic - rendering) applied to ``sample.input``. Kept as the canonical "preview a - sample" entry point for SampleFlux pipelines; use :func:`value_to_image` - directly to render an arbitrary value such as ``sample.target``. + Thin wrapper over :func:`value_to_image` (which does the modality-agnostic rendering) + applied to the payload of the sample's primary ``input``-role field. Use + :func:`value_to_image` directly to render an arbitrary field payload. Args: - sample: The Sample to preview; its ``input`` field is rendered. + sample: The Sample to preview; its primary ``input`` field is rendered. colormap: Colormap applied to 2-D maps — one of the supported names (see ``Colormap``; ``"gray"`` = greyscale). max_size: Maximum length in pixels of the longest image side; larger renders are downscaled. """ - return value_to_image(sample.input, colormap=colormap, max_size=max_size) + return value_to_image(item_data(primary(sample, "input")[1]), colormap=colormap, max_size=max_size) # --------------------------------------------------------------------------- # @@ -575,146 +594,13 @@ def draw_text( return np.array(img) -@configurable(category="op", group="image") -class ConvertToImageOp: - """Convert ``sample.input`` (array / tensor / 2-D map / PIL image) into a PIL image. - - The generic image-conversion op — normalize → colormap → (flip) → resize. - It is modality-agnostic: a dB spectrogram, a segmentation logit map, a CHW - tensor, or an already-PIL image all become a ``PIL.Image.Image`` on - ``sample.input``. Domain overlays are a SEPARATE concern — chain - ``waivefront.visualizers.RenderOverlaysOp`` after this op to draw - signal-region rectangles; this op never draws annotations. - - Rendering uses :func:`value_to_image`'s core (so 2-D maps are colormapped, - 3-D arrays treated as images, bool masks become 0/255, floats min-max - normalized). Sizing: - - * ``width`` and ``height`` both > 0 → resize to exactly that raster - (e.g. a spectrogram rendered to ``1024x512`` for downstream detectors). - * otherwise → bound the longest side by ``max_size``, preserving aspect. - - ``flip_vertical=True`` mirrors the image top-to-bottom — used when the source - array's row 0 is the *bottom* of the desired image (a spectrogram stores - row 0 = f_min but display wants f_max at the top, so overlay pixel math - lines up). The final ``image_width_px`` / ``image_height_px`` are published - to ``sample.meta`` so downstream consumers (e.g. a detector - back-projecting pixel boxes to signal regions) can read the raster size. - - Args: - colormap: Colormap applied to 2-D maps — a supported ``Colormap`` name (``"gray"`` = greyscale). - width: Exact output width in pixels; resize to ``(width, height)`` when both width and height are > 0. - height: Exact output height in pixels; resize to ``(width, height)`` when both width and height are > 0. - max_size: When ``width``/``height`` aren't both set, bound the longest side to this many pixels (aspect kept). - flip_vertical: Mirror the image top-to-bottom (e.g. spectrogram row 0 = f_min → display f_max at the top). - """ - - ACCEPTS = SampleType(input=UnionType((PythonType("PIL.Image.Image"), _ArrayType(frameworks={"numpy", "torch"})))) - PRODUCES = SampleType(input=PythonType("PIL.Image.Image")) - - def __init__( - self, - colormap: Colormap = "gray", - width: int = 0, - height: int = 0, - max_size: int = 512, - flip_vertical: bool = False, - ) -> None: - self.colormap: Colormap = colormap - self.width = int(width) - self.height = int(height) - self.max_size = int(max_size) - self.flip_vertical = bool(flip_vertical) - - def __call__(self, sample: Sample) -> Sample: - rgb = _render_rgb(sample.input, self.colormap) - if self.flip_vertical: - rgb = rgb[::-1, :, :] - if self.width > 0 and self.height > 0: - img = Image.fromarray(rgb).resize( - (self.width, self.height), - resample=Image.Resampling.BILINEAR, - ) - else: - img = Image.fromarray(_bound_longest_side(rgb, self.max_size)) - - sample.meta["image_width_px"] = img.width - sample.meta["image_height_px"] = img.height - return sample._replace(input=img) - - -@configurable(category="op", group="image") -class NormalizeToUint8Op: - """Min-max normalize ``sample.input`` to a ``uint8`` array in ``[0, 255]``. - - The generic value→``uint8`` conversion step, decoupled from any colormap or - PIL rendering (that is :class:`ConvertToImageOp`). Useful as a standalone - quantization stage — e.g. turning a dB spectrogram or a logit map into a - display-ready 8-bit grid — and as the shared math behind the renderers in - this module (:func:`value_to_image` calls :meth:`normalize_to_uint8` - directly for its 2-D-map and float-array paths). - - By default the scale is taken from the array's own finite min/max (per-array - auto-contrast). Supply ``vmin`` / ``vmax`` to pin a *fixed* range instead so - successive samples are quantized on a common scale (e.g. a constant dB window - across a dataset) — values outside the range clamp to ``0`` / ``255``. - - Non-finite entries (``NaN`` / ``±inf``) are folded to the low / high bound - before scaling; a degenerate range (``vmax <= vmin``, or a flat array under - auto bounds) maps to all-zeros to avoid a divide-by-zero. - - Args: - vmin: Lower bound mapped to ``0``; ``None`` (default) uses the array's finite minimum. - vmax: Upper bound mapped to ``255``; ``None`` (default) uses the array's finite maximum. - """ - - ACCEPTS = SampleType(input=_ArrayType(frameworks={"numpy", "torch"})) - PRODUCES = SampleType(input=_ArrayType(dtype="uint8", frameworks={"numpy"})) - - def __init__(self, vmin: Optional[float] = None, vmax: Optional[float] = None) -> None: - self.vmin = None if vmin is None else float(vmin) - self.vmax = None if vmax is None else float(vmax) - - @staticmethod - def normalize_to_uint8( - arr: np.ndarray, - vmin: Optional[float] = None, - vmax: Optional[float] = None, - ) -> np.ndarray: - """Min-max normalize ``arr`` to ``uint8`` in ``[0, 255]``. - - ``vmin`` / ``vmax`` pin the scale when given (clamping out-of-range - values); otherwise the array's finite min / max are used. Non-finite - entries are folded to the bounds; a degenerate range yields all-zeros. - """ - arr = np.asarray(arr).astype(np.float32) - finite = arr[np.isfinite(arr)] - lo = float(vmin) if vmin is not None else (float(finite.min()) if finite.size else 0.0) - hi = float(vmax) if vmax is not None else (float(finite.max()) if finite.size else 0.0) - if hi <= lo: - return np.zeros(arr.shape, dtype=np.uint8) - filled = np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) - norm = (filled - lo) / (hi - lo) - # np.asarray (not .astype) so the return type is ndarray under stub - # versions where clip-arithmetic degrades to Any. - return np.asarray(np.clip(norm, 0.0, 1.0) * 255.0, dtype=np.uint8) - - def __call__(self, sample: Sample) -> Sample: - if self.vmin is not None and self.vmax is not None and self.vmin >= self.vmax: - raise ValueError(f"NormalizeToUint8Op: vmin must be < vmax; got vmin={self.vmin!r}, vmax={self.vmax!r}") - arr = sample.input - if isinstance(arr, torch.Tensor): - arr = arr.detach().cpu().numpy() - return sample._replace(input=self.normalize_to_uint8(arr, self.vmin, self.vmax)) - - @configurable(category="op", group="image") class ConvertToImage(Transform): """Typed twin of :class:`ConvertToImageOp` — an array-bearing field → an ``Image`` item. The typed-bag counterpart of :class:`ConvertToImageOp`: instead of rendering ``sample.input`` into a PIL image in place, it reads an array-bearing field from a - :class:`~sampleflux.TypedSample` and writes a fresh :class:`~sampleflux.Image` item + :class:`~sampleflux.Sample` and writes a fresh :class:`~sampleflux.Image` item (HWC ``uint8`` RGB) under ``output``, tagged with the ``input`` role (it is the pipeline's working image). Any other field passes through untouched. @@ -764,7 +650,7 @@ def __init__( self.field = field self.output = output - def _find_source(self, sample: TypedSample) -> Any: + def _find_source(self, sample: Sample) -> Any: """Resolve the payload to render (``self.field`` or the first array-bearing item).""" if self.field: if self.field not in sample.keys(): @@ -776,7 +662,7 @@ def _find_source(self, sample: TypedSample) -> Any: return item_data(item) raise ValueError(f"ConvertToImage: no array-bearing field in sample (fields: {list(sample.keys())})") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: rgb = _render_rgb(self._find_source(sample), self.colormap) if self.flip_vertical: rgb = rgb[::-1, :, :] @@ -794,8 +680,7 @@ def __call__(self, sample: TypedSample) -> TypedSample: "Colormap", "COLORMAPS", "ConvertToImage", - "ConvertToImageOp", - "NormalizeToUint8Op", + "normalize_to_uint8", "value_to_image", "sample_to_image", "select_channel", diff --git a/sampleflux/ops/metadata.py b/sampleflux/ops/metadata.py deleted file mode 100644 index 16057cf..0000000 --- a/sampleflux/ops/metadata.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Metadata-manipulation ops.""" - -import fnmatch -from typing import List, Optional, Tuple - -from confluid import configurable - -from sampleflux.sample import Sample - - -def _matches_any(key: str, patterns: Tuple[str, ...]) -> bool: - """True if ``key`` matches ANY ``fnmatch`` glob in ``patterns`` (case-sensitive).""" - return any(fnmatch.fnmatchcase(key, pattern) for pattern in patterns) - - -@configurable(category="op", group="structure") -class DropMetadataOp: - """Remove metadata keys matching glob patterns (a pass-through ``Sample -> Sample`` op). - - A key is DROPPED when it matches an ``exclude`` pattern AND does NOT match any ``include`` - pattern — so ``include`` PROTECTS keys and takes priority over ``exclude`` (the rsync / - gitignore include-wins model). Strips bookkeeping you don't want a downstream sink to - serialise — e.g. a bulky ``Stash*Op`` snapshot (a stashed complex signal) kept on the - metadata bus for a ``Parallel`` crossing. The replacement metadata is a - fresh dict (copy-on-write); ``input`` / ``target`` are untouched. Single-sample only (reads - ``sample.meta``), like ``CopyMetadataOp`` — drop keys before collation. - - Args: - exclude: Glob patterns (``fnmatch``: ``*`` = any run, ``?`` = one char, ``[seq]`` = a set) - for keys to REMOVE. A pattern with NO wildcards matches that key exactly. Case-sensitive. - E.g. ``spec_*`` removes every spec-prefixed snapshot; with no ``exclude`` nothing - is dropped. - include: Glob patterns for keys to KEEP even when they match ``exclude`` — higher priority, - so it carves exceptions out of ``exclude``. E.g. ``exclude=["spec_*"]`` + - ``include=["spec_keep"]`` drops every ``spec_*`` key EXCEPT the protected one. ``include`` - only ever protects against ``exclude`` (with no ``exclude`` it has no effect). - """ - - def __init__(self, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> None: - self.exclude = exclude - self.include = include - - def __call__(self, sample: Sample) -> Sample: - exclude = tuple(self.exclude or ()) - include = tuple(self.include or ()) - kept = { - key: value - for key, value in sample.meta.items() - if not (_matches_any(key, exclude) and not _matches_any(key, include)) - } - return sample._replace(metadata=kept) diff --git a/sampleflux/ops/numpy.py b/sampleflux/ops/numpy.py index a0c6e7c..91dddb1 100644 --- a/sampleflux/ops/numpy.py +++ b/sampleflux/ops/numpy.py @@ -1,21 +1,15 @@ import operator import os import re -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import numpy as np from confluid import configurable from loggair import get_logger from sampleflux.bag.items import Mask, NDArrayItem, Regions, item_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.bag.transform import Transform -from sampleflux.sample import Sample -from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType - -# Common shorthands for the numpy ops' declared types. -_NDARRAY = ArrayType(frameworks={"numpy"}) -_NUMERIC_OR_PIL = UnionType((ArrayType(dtype="numeric", frameworks={"numpy"}), PythonType("PIL.Image.Image"))) logger = get_logger(__name__) @@ -23,37 +17,33 @@ _EXPR_PATTERN = re.compile(r"\{(\w+)\}|\$(\w+)") -def resolve_expression(value: str, sample: Sample) -> str: - """Substitute ``{key}`` from ``sample.meta`` and ``$NAME`` from ``os.environ``. +def resolve_expression(value: str, meta: Optional[Dict[str, Any]] = None) -> str: + """Substitute ``{key}`` from ``meta`` and ``$NAME`` from ``os.environ``. - Returns the substituted string verbatim — the caller is responsible for - any further casting (e.g. ``float(...)`` for a numeric expression). + Returns the substituted string verbatim — the caller is responsible for any further + casting (e.g. ``float(...)`` for a numeric expression). In the typed-bag model an item + owns its own metadata (there is no shared sample dict), so ``meta`` is usually empty and + only literals / ``$ENV`` expressions resolve; a ``{key}`` bound then raises ``KeyError``. Args: - value: Expression string with ``{meta_key}`` and/or ``$ENV_VAR`` placeholders - (a plain literal returns unchanged). - sample: The Sample whose ``metadata`` supplies the ``{key}`` substitutions. - - Examples: - ``"5.5"`` → ``"5.5"`` (no substitution) - ``"{reference_snr_level}"`` → ``str(metadata["reference_snr_level"])`` - ``"-{reference_snr_level}"`` → ``"-"`` (sign passes through to ``float()``) - ``"$REF_SNR"`` → ``os.environ["REF_SNR"]`` + value: Expression string with ``{meta_key}`` and/or ``$ENV_VAR`` placeholders. + meta: Metadata dict supplying the ``{key}`` substitutions (defaults to empty). Raises: KeyError: A referenced metadata key or environment variable is missing. """ + meta = meta or {} def _repl(match: "re.Match[str]") -> str: meta_key = match.group(1) env_name = match.group(2) if meta_key is not None: - if meta_key not in sample.meta: + if meta_key not in meta: raise KeyError( f"resolve_expression: metadata key {meta_key!r} missing in {value!r}; " - f"available keys: {sorted(sample.meta)}" + f"available keys: {sorted(meta)}" ) - return str(sample.meta[meta_key]) + return str(meta[meta_key]) assert env_name is not None if env_name not in os.environ: raise KeyError(f"resolve_expression: environment variable {env_name!r} missing in {value!r}") @@ -62,398 +52,88 @@ def _repl(match: "re.Match[str]") -> str: return _EXPR_PATTERN.sub(_repl, value) -@configurable(category="op", group="numpy") -class StandardizeOp: - """ - Standardizes ndarray values with given mean and standard deviation. - - Formula: output = (input - mean) / std - - mean/std can be a single float (applied uniformly) or a sequence of - per-channel values that broadcasts over [C, H, W] format. - - Handles PIL images by converting to ndarray first. - - Args: - mean: Mean to subtract — a single float (uniform) or a per-channel sequence broadcasting over [C, H, W]. - std: Standard deviation to divide by — a single float (uniform) or a per-channel sequence. - """ - - ACCEPTS = SampleType(input=_NUMERIC_OR_PIL) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - - def __init__(self, mean: Union[float, Sequence[float]] = 0.0, std: Union[float, Sequence[float]] = 1.0): - # Lazy / zero-arg: store config only. The defaults (mean 0, std 1) are an identity standardize. - self.mean = mean - self.std = std - - def __call__(self, sample: Sample) -> Sample: - arr = sample.input - - # Handle PIL / PngImageFile - if hasattr(arr, "convert"): - arr = np.array(arr) - - if not isinstance(arr, np.ndarray): - raise TypeError(f"StandardizeOp expects an np.ndarray, got {type(arr).__name__}") - - if arr.dtype == np.float64: - arr = arr.astype(np.float64) - else: - arr = arr.astype(np.float32) - - if isinstance(self.mean, (int, float)): - mean_a = np.array([self.mean], dtype=arr.dtype) - else: - mean_a = np.array(self.mean, dtype=arr.dtype) - - if isinstance(self.std, (int, float)): - std_a = np.array([self.std], dtype=arr.dtype) - else: - std_a = np.array(self.std, dtype=arr.dtype) - - # Reshape to [C, 1, 1, ...] for broadcasting over [C, H, W] - mean_a = mean_a.reshape(-1, *([1] * (arr.ndim - 1))) - std_a = std_a.reshape(-1, *([1] * (arr.ndim - 1))) - - arr = (arr - mean_a) / std_a - - return sample._replace(input=arr) - - -def _require_ndarray(sample: Sample, op_name: str) -> np.ndarray: - arr = sample.input - if not isinstance(arr, np.ndarray): - raise TypeError(f"{op_name} expects an np.ndarray on sample.input, got {type(arr).__name__}") - return arr - - -@configurable(category="op", group="numpy") -class SqueezeOp: - """Remove size-1 axes from an ``np.ndarray``. - - Args: - axis: Axis index to remove. When ``None`` (default), all size-1 axes are removed. - When specified, the axis must have size 1 (numpy raises ``ValueError`` otherwise). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: Optional[int] = None) -> None: - self.axis = axis - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "SqueezeOp") - out = np.squeeze(arr) if self.axis is None else np.squeeze(arr, axis=self.axis) - return sample._replace(input=out) - - -@configurable(category="op", group="numpy") -class UnsqueezeOp: - """Insert a size-1 axis at the specified position in an ``np.ndarray``. - - Args: - axis: Axis index at which the new dimension is inserted. Default ``0``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: int = 0) -> None: - self.axis = axis - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "UnsqueezeOp") - return sample._replace(input=np.expand_dims(arr, axis=self.axis)) - - -@configurable(category="op", group="numpy") -class ClipPercentilesOp: - """Clip ``sample.input`` to ``[p_low, p_high]`` percentiles of finite values. - - Percentiles are computed over only finite entries — ``inf`` / ``-inf`` / - ``nan`` are excluded from the percentile estimate. ``np.clip`` then maps - ``+inf`` to the upper bound and ``-inf`` to the lower bound; ``nan`` - survives unchanged. Chain :class:`ReplaceNonFiniteOp` upstream if remaining - ``nan`` values matter. - - Args: - low: Lower percentile in ``[0, 100)``. Default ``2.0``. - high: Upper percentile in ``(0, 100]``, must be ``> low``. Default ``98.0``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, low: float = 2.0, high: float = 98.0) -> None: - # Lazy / zero-arg: store config only; the bound relationship is validated lazily in __call__. - self.low = float(low) - self.high = float(high) - - def __call__(self, sample: Sample) -> Sample: - if not (0.0 <= self.low < self.high <= 100.0): - raise ValueError(f"ClipPercentilesOp: require 0 <= low < high <= 100; got low={self.low}, high={self.high}") - arr = _require_ndarray(sample, "ClipPercentilesOp") - finite = np.isfinite(arr) - if not finite.any(): - logger.warning("ClipPercentilesOp: input is entirely non-finite; passing through") - return sample - lo = float(np.percentile(arr[finite], self.low)) - hi = float(np.percentile(arr[finite], self.high)) - return sample._replace(input=np.clip(arr, lo, hi)) - - -@configurable(category="op", group="numpy") -class RescaleOp: - """Affine rescale ``sample.input`` from ``[in_min, in_max]`` to ``[out_min, out_max]``. - - The default ``out_min=0.0`` / ``out_max=1.0`` covers the common - ``[0, 255] -> [0, 1]`` image-normalization case. PIL inputs are - converted to ndarray; integer dtypes are promoted to ``float32`` - (``float64`` is preserved). - - Args: - in_min: Lower edge of the input range. Default ``0.0``. - in_max: Upper edge of the input range, must be ``> in_min``. Default ``1.0``. - out_min: Lower edge of the output range. Default ``0.0``. - out_max: Upper edge of the output range, must be ``> out_min``. Default ``1.0``. - clip: When True (default), clamp values outside ``[in_min, in_max]`` - before rescaling. When False, extrapolate linearly. - """ - - ACCEPTS = SampleType(input=_NUMERIC_OR_PIL) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - - def __init__( - self, - in_min: float = 0.0, - in_max: float = 1.0, - out_min: float = 0.0, - out_max: float = 1.0, - clip: bool = True, - ) -> None: - # Lazy / zero-arg: store config only; the bound relationships are validated lazily in __call__. - self.in_min = float(in_min) - self.in_max = float(in_max) - self.out_min = float(out_min) - self.out_max = float(out_max) - self.clip = bool(clip) - - def __call__(self, sample: Sample) -> Sample: - if not (self.in_min < self.in_max): - raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={self.in_min}, in_max={self.in_max}") - if not (self.out_min < self.out_max): - raise ValueError( - f"RescaleOp: require out_min < out_max; got out_min={self.out_min}, out_max={self.out_max}" - ) - arr = sample.input - if hasattr(arr, "convert"): - arr = np.array(arr) - if not isinstance(arr, np.ndarray): - raise TypeError(f"RescaleOp expects an np.ndarray, got {type(arr).__name__}") - arr = arr.astype(np.float64 if arr.dtype == np.float64 else np.float32) - src = np.clip(arr, self.in_min, self.in_max) if self.clip else arr - scaled = (src - self.in_min) / (self.in_max - self.in_min) - out = scaled * (self.out_max - self.out_min) + self.out_min - return sample._replace(input=out) - - -@configurable(category="op", group="numpy") -class ReplaceNonFiniteOp: - """Replace ``inf`` / ``-inf`` / ``nan`` entries in ``sample.input``. - - Args: - value: Replacement specifier. Either: - - * a ``float`` / ``int`` — literal replacement value; - * the string ``"min"`` — replace with the array's finite min; - * the string ``"max"`` — replace with the array's finite max. - - Default ``"min"``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, value: Union[float, int, str] = "min") -> None: - # Lazy / zero-arg: store config only; the 'min'/'max' string is validated lazily in __call__. - self.value = value - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "ReplaceNonFiniteOp") - non_finite = ~np.isfinite(arr) - if not non_finite.any(): - return sample - if isinstance(self.value, str): - if self.value not in ("min", "max"): - raise ValueError(f"ReplaceNonFiniteOp: value string must be 'min' or 'max'; got {self.value!r}") - finite = ~non_finite - if not finite.any(): - logger.warning("ReplaceNonFiniteOp: array is entirely non-finite; passing through") - return sample - finite_values = arr[finite] - repl = float(finite_values.min() if self.value == "min" else finite_values.max()) - else: - repl = float(self.value) - return sample._replace(input=np.where(non_finite, repl, arr)) - - -# ThresholdOp comparison selectors. Closed ``Literal``s (workspace "prefer closed -# Literals over bare strings" mandate) so GUIs / schema generators render the choice -# as a dropdown and the allowed operators stay machine-introspectable via -# ``typing.get_args(...)``. Two distinct types because the lower bound only sensibly -# uses ``>`` / ``>=`` and the upper bound only ``<`` / ``<=``. +# Threshold comparison selectors. Closed ``Literal``s so GUIs / schema generators render the +# choice as a dropdown and the allowed operators stay machine-introspectable via +# ``typing.get_args(...)``. Two distinct types because the lower bound only sensibly uses +# ``>`` / ``>=`` and the upper bound only ``<`` / ``<=``. LowComparison = Literal[">", ">="] HighComparison = Literal["<", "<="] -# Operator dispatch. The dict keys are the single runtime source of truth's -# consumers — ``tests/test_ops.py`` pins ``set(_LOW_COMPARISONS) == get_args(LowComparison)`` -# (and likewise for high) so the map can never drift from the Literal. _LOW_COMPARISONS: Dict[str, Callable[[Any, float], Any]] = {">": operator.gt, ">=": operator.ge} _HIGH_COMPARISONS: Dict[str, Callable[[Any, float], Any]] = {"<": operator.lt, "<=": operator.le} -@configurable(category="op", group="numpy") -class ThresholdOp: - """Threshold ``sample.input`` (ndarray) into a boolean mask using one or both bounds. - - Which mask is produced depends on *which* bounds are set (presence-driven), and the - comparison applied for each is selected by ``low_op`` / ``high_op``: - - * only ``low_level`` → ``input low_level`` (values above the floor) - * only ``high_level`` → ``input high_level`` (values below the ceiling) - * both → both conditions AND-ed together (band-pass) - - ``low_op`` is ``">"`` (strict, the default) or ``">="`` (inclusive); ``high_op`` is - ``"<"`` (strict, the default) or ``"<="`` (inclusive). So the defaults yield the OPEN - interval ``low_level < input < high_level``, while ``low_op=">="`` + ``high_op="<="`` - yield the CLOSED interval ``low_level <= input <= high_level``. - - At least one of ``low_level`` / ``high_level`` MUST be provided; passing - neither raises ``ValueError`` when the op is applied (the zero-arg default is - deferred-valid so the op stays constructible, per the lazy-init convention). - - Each bound is either a numeric literal or a string expression resolved via - :func:`resolve_expression` against ``sample.meta`` and ``os.environ``: - - * ``5.5`` or ``"5.5"`` — fixed bound - * ``"{reference_snr_level}"`` — looks up ``metadata["reference_snr_level"]`` - * ``"-{reference_snr_level}"`` — negated lookup (the leading ``-`` is - carried through ``float(...)`` after substitution) - * ``"$REF_SNR"`` / ``"-$REF_SNR"`` — environment-variable lookup - - Records each resolved bound that was applied under ``metadata["threshold_low"]`` - / ``metadata["threshold_high"]`` for traceability. - - Args: - low_level: Lower bound (numeric literal or expression) compared with ``low_op`` when set; - ``None`` disables the lower bound. - high_level: Upper bound (numeric literal or expression) compared with ``high_op`` when set; - ``None`` disables the upper bound. - low_op: Lower-bound comparison — ``">"`` (strict, default) or ``">="`` (inclusive). - high_op: Upper-bound comparison — ``"<"`` (strict, default) or ``"<="`` (inclusive). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=ArrayType(dtype="bool", frameworks={"numpy"})) - - def __init__( - self, - low_level: Optional[Union[float, int, str]] = None, - high_level: Optional[Union[float, int, str]] = None, - low_op: LowComparison = ">", - high_op: HighComparison = "<", - ) -> None: - # Lazy / zero-arg: store config only; the "at least one bound" requirement is validated - # lazily in __call__ so the op stays constructible with no arguments. - self.low_level = low_level - self.high_level = high_level - self.low_op = low_op - self.high_op = high_op - - def _resolve(self, bound: Optional[Union[float, int, str]], sample: Sample) -> float: - if bound is None: - raise ValueError("ThresholdOp._resolve called with None — bound was not filtered by __call__") - if isinstance(bound, str): - resolved = resolve_expression(bound, sample) - try: - return float(resolved) - except (TypeError, ValueError) as exc: - raise ValueError( - f"ThresholdOp: expression {bound!r} resolved to {resolved!r}, which is not a number" - ) from exc - # Any non-string numeric: a Python int/float, a NumPy scalar (e.g. the float32 a value-chain - # MaxOp → FormulaOp → ConfigureOp injects into low_level per sample), or a 0-d array — anything - # float() accepts. A list / multi-D array / complex value fails float() and raises the TypeError. +def _resolve_bound(bound: Union[float, int, str], meta: Optional[Dict[str, Any]]) -> float: + """Resolve a threshold bound (literal / numeric / ``resolve_expression`` string) to a float.""" + if isinstance(bound, str): + resolved = resolve_expression(bound, meta) try: - return float(bound) + return float(resolved) except (TypeError, ValueError) as exc: - raise TypeError( - f"ThresholdOp bounds must be a number or expression string; got {type(bound).__name__}" + raise ValueError( + f"threshold: expression {bound!r} resolved to {resolved!r}, which is not a number" ) from exc + try: + return float(bound) + except (TypeError, ValueError) as exc: + raise TypeError(f"threshold bounds must be a number or expression string; got {type(bound).__name__}") from exc + + +def threshold_array( + arr: np.ndarray, + low_level: Optional[Union[float, int, str]] = None, + high_level: Optional[Union[float, int, str]] = None, + low_op: LowComparison = ">", + high_op: HighComparison = "<", + meta: Optional[Dict[str, Any]] = None, +) -> np.ndarray: + """Threshold ``arr`` into a boolean mask using one or both bounds. + + * only ``low_level`` → ``arr low_level`` (values above the floor) + * only ``high_level`` → ``arr high_level`` (values below the ceiling) + * both → both conditions AND-ed together (band-pass) - def __call__(self, sample: Sample) -> Sample: - arr = sample.input - if not isinstance(arr, np.ndarray): - raise TypeError(f"ThresholdOp expects an np.ndarray on sample.input, got {type(arr).__name__}") - - low_level = self.low_level - high_level = self.high_level - # Treat empty string (blank STRING widget left unset) as None ("disabled"). - if isinstance(low_level, str) and low_level.strip() == "": - low_level = None - if isinstance(high_level, str) and high_level.strip() == "": - high_level = None - - mask: Optional[np.ndarray] = None - if low_level is not None: - low = self._resolve(low_level, sample) - if np.isnan(low): - logger.warning( - f"ThresholdOp: resolved low_level is NaN; no values will be above the threshold. " - f"Expression was {self.low_level!r} resolved to {low!r}" - ) - else: - sample.meta["threshold_low"] = low - mask = _LOW_COMPARISONS[self.low_op](arr, low) - if high_level is not None: - high = self._resolve(high_level, sample) - if np.isnan(high): - logger.warning( - f"ThresholdOp: resolved high_level is NaN; no values will be below the threshold. " - f"Expression was {self.high_level!r} resolved to {high!r}" - ) - else: - sample.meta["threshold_high"] = high - below = _HIGH_COMPARISONS[self.high_op](arr, high) - mask = below if mask is None else (mask & below) - if mask is None: - raise ValueError("ThresholdOp requires at least one of 'low_level' / 'high_level'") - return sample._replace(input=mask) + At least one of ``low_level`` / ``high_level`` MUST be provided. + """ + if not isinstance(arr, np.ndarray): + raise TypeError(f"threshold_array expects an np.ndarray, got {type(arr).__name__}") + if isinstance(low_level, str) and low_level.strip() == "": + low_level = None + if isinstance(high_level, str) and high_level.strip() == "": + high_level = None + + mask: Optional[np.ndarray] = None + if low_level is not None: + low = _resolve_bound(low_level, meta) + if np.isnan(low): + logger.warning(f"threshold_array: resolved low_level is NaN ({low_level!r}); no values pass the floor.") + else: + mask = _LOW_COMPARISONS[low_op](arr, low) + if high_level is not None: + high = _resolve_bound(high_level, meta) + if np.isnan(high): + logger.warning(f"threshold_array: resolved high_level is NaN ({high_level!r}); no values pass the ceiling.") + else: + below = _HIGH_COMPARISONS[high_op](arr, high) + mask = below if mask is None else (mask & below) + if mask is None: + raise ValueError("threshold_array requires at least one of 'low_level' / 'high_level'") + return mask @configurable(category="op", group="numpy") class Threshold(Transform): - """Typed twin of :class:`ThresholdOp` — an array-bearing field → a boolean ``Mask`` item. + """An array-bearing field → a boolean ``Mask`` item. - The typed-bag counterpart of :class:`ThresholdOp`: it reads the array at ``field`` (blank = - the first array-bearing item in the bag) and thresholds it into a boolean mask with the SAME - bound / comparison / expression math — this twin REUSES the legacy op verbatim, so the booleans - are identical — writing a :class:`~sampleflux.Mask` item under ``output`` tagged ``aux`` (a - threshold mask is an intermediate that a later op — e.g. :class:`ConnectedComponents` — - consumes, not a model input or target). Any other field passes through untouched. - - Which mask is produced depends on which bounds are set, and the comparison for each is picked - by ``low_op`` / ``high_op`` (see :class:`ThresholdOp` for the full presence-driven rules and - the open-vs-closed interval semantics). At least one of ``low_level`` / ``high_level`` MUST be - set; passing neither raises ``ValueError`` when applied (the zero-arg default stays - constructible per the lazy-init convention). + Reads the array at ``field`` (blank = the first array-bearing item in the bag) and thresholds + it into a boolean mask with the bound / comparison / expression math (:func:`threshold_array`), + writing a :class:`~sampleflux.Mask` item under ``output`` tagged ``aux`` (a threshold mask is an + intermediate that a later op — e.g. :class:`ConnectedComponents` — consumes). Any other field + passes through untouched. Each bound is a numeric literal or a ``resolve_expression`` string — ``5.5`` / ``"5.5"`` (literal) or ``"$REF_SNR"`` (environment variable). NOTE: ``{meta_key}`` expressions have no - typed metadata source in the bag model (an item owns its own metadata; there is no shared - sample dict), so only literals and ``$ENV`` resolve here — a ``{key}`` bound raises ``KeyError``. + typed metadata source in the bag model, so only literals and ``$ENV`` resolve here. Args: low_level: Lower bound (numeric literal or ``$ENV`` expression) compared with ``low_op`` when set; @@ -487,7 +167,7 @@ def __init__( self.field = field self.output = output - def _find_array(self, sample: TypedSample) -> np.ndarray: + def _find_array(self, sample: Sample) -> np.ndarray: """Resolve the array to threshold (``self.field`` or the first array-bearing item).""" if self.field: if self.field not in sample.keys(): @@ -502,12 +182,9 @@ def _find_array(self, sample: TypedSample) -> np.ndarray: return data raise ValueError(f"Threshold: no array-bearing field in sample (fields: {list(sample.keys())})") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: arr = self._find_array(sample) - # Reuse the legacy op's threshold math VERBATIM on a shim Sample so the booleans are - # identical; the shim's empty metadata is why only literals / $ENV bounds resolve here. - legacy = ThresholdOp(self.low_level, self.high_level, self.low_op, self.high_op) - mask = legacy(Sample(input=arr, target=None, metadata={})).input + mask = threshold_array(arr, self.low_level, self.high_level, self.low_op, self.high_op) out = sample.replace_field(self.output, Mask(mask)) return out.set_role(self.output, "aux") @@ -518,11 +195,9 @@ def connected_component_bboxes( """Label connected ``True`` regions of a 2-D bool mask → ``(row_min, row_max, col_min, col_max)`` inclusive tuples. Components smaller than ``min_area_bins`` are dropped. ``connectivity`` is ``4`` - (orthogonal neighbors) or ``8`` (orthogonal + diagonal). This is the shared scipy - core behind :class:`ConnectedComponentsOp` (signal-domain bin bboxes on ``input``) - AND :class:`sampleflux.ops.target.MasksToDetectionBoxesOp` (its ``connected=True`` - mode, which lifts the tuples to xyxy-pixel detection boxes). Requires ``scipy`` - (``pip install sampleflux[vision]``). + (orthogonal neighbors) or ``8`` (orthogonal + diagonal). Shared by :class:`ConnectedComponents` + AND :func:`sampleflux.ops.target.masks_to_detection` (its ``connected=True`` mode). Requires + ``scipy`` (``pip install sampleflux[vision]``). """ if min_area_bins < 1: raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") @@ -559,203 +234,17 @@ def connected_component_bboxes( @configurable(category="op", group="numpy") -class MinOp: - """Reduce ``sample.input`` to its minimum value, ignoring NaN. - - Args: - axis: Axis along which to compute the minimum. ``None`` (default) reduces over all axes. - keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: Optional[int] = None, keepdims: bool = False) -> None: - self.axis = axis - self.keepdims = bool(keepdims) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "MinOp") - return sample._replace(input=np.nanmin(arr, axis=self.axis, keepdims=self.keepdims)) - - -@configurable(category="op", group="numpy") -class MaxOp: - """Reduce ``sample.input`` to its maximum value, ignoring NaN. - - Args: - axis: Axis along which to compute the maximum. ``None`` (default) reduces over all axes. - keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: Optional[int] = None, keepdims: bool = False) -> None: - self.axis = axis - self.keepdims = bool(keepdims) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "MaxOp") - return sample._replace(input=np.nanmax(arr, axis=self.axis, keepdims=self.keepdims)) - - -@configurable(category="op", group="numpy") -class MedianOp: - """Reduce ``sample.input`` to its median value, ignoring NaN. - - Args: - axis: Axis along which to compute the median. ``None`` (default) reduces over all axes. - keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, axis: Optional[int] = None, keepdims: bool = False) -> None: - self.axis = axis - self.keepdims = bool(keepdims) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "MedianOp") - return sample._replace(input=np.nanmedian(arr, axis=self.axis, keepdims=self.keepdims)) - - -@configurable(category="op", group="numpy") -class PercentileOp: - """Reduce ``sample.input`` to a 2-element array ``[p_low, p_high]``, ignoring NaN. - - Output shape when ``axis=None``: ``(2,)`` scalar pair. When ``axis=k``: - ``(2, …)`` stacked along a new leading dimension. - - Args: - low: Lower percentile in ``[0, 100]``. Default ``5.0``. - high: Upper percentile in ``[0, 100]``, should be ``> low``. Default ``95.0``. - axis: Axis along which to compute the percentiles. ``None`` (default) reduces over all axes. - keepdims: When ``True``, the reduced axes are retained with size 1 (default ``False``). - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__( - self, - low: float = 5.0, - high: float = 95.0, - axis: Optional[int] = None, - keepdims: bool = False, - ) -> None: - self.low = float(low) - self.high = float(high) - self.axis = axis - self.keepdims = bool(keepdims) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "PercentileOp") - p_low = np.nanpercentile(arr, self.low, axis=self.axis, keepdims=self.keepdims) - p_high = np.nanpercentile(arr, self.high, axis=self.axis, keepdims=self.keepdims) - return sample._replace(input=np.stack([p_low, p_high])) - - -@configurable(category="op", group="numpy") -class StatsOp: - """Compute summary statistics of ``sample.input`` and record them in metadata; input is passed through unchanged. - - Writes five scalar float keys to ``sample.metadata``: ``{prefix}min``, - ``{prefix}max``, ``{prefix}median``, ``{prefix}p_low``, ``{prefix}p_high``. - NaN values are excluded from all computations. - - Chain anywhere in a pipeline without disrupting the data flow — useful for - inspecting distribution properties during development or for downstream - normalisation decisions. - - Args: - low: Lower percentile bound (0–100). Default ``5.0``. - high: Upper percentile bound (0–100). Default ``95.0``. - prefix: Optional string prepended to every metadata key, e.g. ``"input_"`` to - distinguish multiple ``StatsOp`` invocations in one pipeline. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, low: float = 5.0, high: float = 95.0, prefix: str = "") -> None: - self.low = float(low) - self.high = float(high) - self.prefix = prefix - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "StatsOp") - p = self.prefix - meta = dict(sample.meta) - meta[f"{p}min"] = float(np.nanmin(arr)) - meta[f"{p}max"] = float(np.nanmax(arr)) - meta[f"{p}median"] = float(np.nanmedian(arr)) - meta[f"{p}p_low"] = float(np.nanpercentile(arr, self.low)) - meta[f"{p}p_high"] = float(np.nanpercentile(arr, self.high)) - return sample._replace(metadata=meta) - - -@configurable(category="op", group="numpy") -class ConnectedComponentsOp: - """Label connected ``True`` regions of a boolean mask into bin-bbox tuples. - - Reads ``sample.input`` as a 2-D boolean ndarray; writes ``sample.input`` as - a list of ``(row_min, row_max, col_min, col_max)`` integer tuples (inclusive - bounds). Components smaller than ``min_area_bins`` are dropped. - - ``connectivity`` selects the neighborhood: - - * ``4`` — orthogonal neighbors only (N/S/E/W); diagonally touching - components stay separate. - * ``8`` — orthogonal + diagonal neighbors; diagonally touching - components merge. - - Requires ``scipy`` (install via ``pip install sampleflux[vision]``). - - Args: - min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). - connectivity: Pixel neighborhood — ``4`` (orthogonal only) or ``8`` (orthogonal + diagonal). - """ - - ACCEPTS = SampleType(input=ArrayType(ndim=2, dtype="bool", frameworks={"numpy"})) - PRODUCES = SampleType(input=PythonType("list")) +class ConnectedComponents(Transform): + """A boolean ``Mask`` → a ``Regions`` item. - def __init__(self, min_area_bins: int = 1, connectivity: int = 4) -> None: - # Lazy / zero-arg: store config only; bounds are validated lazily in __call__. - self.min_area_bins = int(min_area_bins) - self.connectivity = int(connectivity) - - def __call__(self, sample: Sample) -> Sample: - mask = sample.input - if not isinstance(mask, np.ndarray): - raise TypeError(f"ConnectedComponentsOp expects an np.ndarray on sample.input, got {type(mask).__name__}") - if mask.ndim != 2: - raise ValueError(f"ConnectedComponentsOp expects a 2-D mask; got shape {mask.shape}") - # Shared scipy core (also used by sampleflux.ops.target.MasksToDetectionBoxesOp); - # validates min_area_bins / connectivity and raises the scipy ImportError. - bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) - return sample._replace(input=bboxes) + Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, else the + first array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into + ``(row_min, row_max, col_min, col_max)`` inclusive bin-box tuples via + :func:`connected_component_bboxes`, writing them as a :class:`~sampleflux.Regions` item under + ``output`` tagged ``aux`` (RAW detections, not model predictions). Any other field passes through. - -@configurable(category="op", group="numpy") -class ConnectedComponents(Transform): - """Typed twin of :class:`ConnectedComponentsOp` — a boolean ``Mask`` → a ``Regions`` item. - - The typed-bag counterpart of :class:`ConnectedComponentsOp`: it reads the - :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, else the first - array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into - ``(row_min, row_max, col_min, col_max)`` inclusive bin-box tuples via the SAME shared - :func:`connected_component_bboxes` helper the legacy op uses (so the numbers are identical), - writing them as a :class:`~sampleflux.Regions` item under ``output``. That field is tagged - ``aux``: these are RAW detections (thresholded blobs), NOT model predictions — the ``pred`` - role is reserved for a detector's output. Any other field passes through untouched. - - The ``Regions.boxes`` list holds ``(row_min, row_max, col_min, col_max)`` tuples — the exact - generic bin-box format (row bounds first, then column bounds; inclusive) a downstream - back-projection reads to map bins to a signal / world coordinate frame. Components smaller than - ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or 8-neighborhood. Requires - ``scipy`` (``pip install sampleflux[vision]``). + Components smaller than ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or + 8-neighborhood. Requires ``scipy`` (``pip install sampleflux[vision]``). Args: min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). @@ -781,7 +270,7 @@ def __init__( self.field = field self.output = output - def _find_mask(self, sample: TypedSample) -> np.ndarray: + def _find_mask(self, sample: Sample) -> np.ndarray: """Resolve the mask to label (``self.field``, else the first ``Mask``, else the first array).""" if self.field: if self.field not in sample.keys(): @@ -811,8 +300,19 @@ def _find_mask(self, sample: TypedSample) -> np.ndarray: raise ValueError(f"ConnectedComponents expects a 2-D mask; got shape {data.shape}") return data - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: mask = self._find_mask(sample) bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) out = sample.replace_field(self.output, Regions(boxes=list(bboxes))) return out.set_role(self.output, "aux") + + +__all__ = [ + "resolve_expression", + "threshold_array", + "connected_component_bboxes", + "LowComparison", + "HighComparison", + "Threshold", + "ConnectedComponents", +] diff --git a/sampleflux/ops/parallel.py b/sampleflux/ops/parallel.py index a81427a..6904ebf 100644 --- a/sampleflux/ops/parallel.py +++ b/sampleflux/ops/parallel.py @@ -25,8 +25,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid +from sampleflux.bag.sample import Sample from sampleflux.core import _worker_task -from sampleflux.sample import Sample @configurable(category="op", group="compose") diff --git a/sampleflux/ops/random_apply.py b/sampleflux/ops/random_apply.py index 207d4f6..2e25058 100644 --- a/sampleflux/ops/random_apply.py +++ b/sampleflux/ops/random_apply.py @@ -14,7 +14,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample logger = get_logger(__name__) diff --git a/sampleflux/ops/sink.py b/sampleflux/ops/sink.py index b928ce8..cce5f4b 100644 --- a/sampleflux/ops/sink.py +++ b/sampleflux/ops/sink.py @@ -12,7 +12,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample logger = get_logger(__name__) diff --git a/sampleflux/ops/stash.py b/sampleflux/ops/stash.py deleted file mode 100644 index bad1adc..0000000 --- a/sampleflux/ops/stash.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Stash / unstash ``sample.input`` / ``sample.target`` to / from ``metadata``. - -Use ``StashInputOp(key)`` to snapshot the current ``sample.input`` under a -metadata key without changing ``sample.input``; ``UnstashInputOp(key)`` -restores it later. ``StashTargetOp`` / ``UnstashTargetOp`` are the exact -``sample.target`` counterparts. - -Graph WIRING is the job of the context ops (``sampleflux.ops.context`` — -``Save``/``Use``/``Mix`` over per-sample Context cells, see ``docs/graph.md``). -The stash family remains for the two jobs cells cannot do, because the -snapshot rides ``sample.metadata`` WITH the sample: - -* crossing a ``Parallel`` boundary — metadata travels through the stream - split/join; Context cells deliberately raise there; -* deliberately PERSISTING a snapshot into a sink (the metadata key is - serialised alongside the sample unless an ``Unstash*Op`` removes it). - -The ``Unstash*Op``\\ s default to ``copy=True`` (deepcopy) so two readers -of the same key are independent — each gets its own array to mutate. -Without the copy, an in-place op like ``ClipPercentilesOp`` after the first -restore would silently corrupt the stashed value seen by the second. -""" - -import copy as _copy - -from confluid import configurable - -from sampleflux.sample import Sample - - -@configurable(category="op", group="structure") -class StashInputOp: - """Copy ``sample.input`` into ``metadata[key]``; ``sample.input`` unchanged. - - Args: - key: Metadata key to write. - copy: When ``True``, deepcopy ``sample.input`` before stashing. - Defaults to ``False`` (cheap pointer alias) — the typical case - is that downstream ops use ``sample._replace(input=...)`` and - don't mutate the shared array in place. - """ - - def __init__(self, key: str = "", copy: bool = False) -> None: - # Lazy / zero-arg: store config only. - self.key = key - self.copy = copy - - def __call__(self, sample: Sample) -> Sample: - sample.meta[self.key] = _copy.deepcopy(sample.input) if self.copy else sample.input - return sample - - -@configurable(category="op", group="structure") -class UnstashInputOp: - """Set ``sample.input := metadata[key]``. - - Args: - key: Metadata key to read. - copy: When ``True`` (default), deepcopy the stashed value before - assigning. This prevents two branches that unstash the same - key from corrupting each other through downstream in-place - mutations. Set ``False`` only when the caller has audited - that no downstream op mutates the array in place. - remove: When ``True`` (default), DELETE the key from metadata after - restoring it — so the snapshot doesn't linger on the bus and - leak into a downstream sink. Set ``False`` to keep it (required - when the SAME key is unstashed again later, e.g. a fan-out that - restores the fork before several branches — only the LAST - unstash of a key may remove it). - """ - - def __init__(self, key: str = "", copy: bool = True, remove: bool = True) -> None: - # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. - self.key = key - self.copy = copy - self.remove = remove - - def __call__(self, sample: Sample) -> Sample: - value = sample.meta[self.key] - if self.copy: - value = _copy.deepcopy(value) - if self.remove: - del sample.meta[self.key] # key exists (just read above) - return sample._replace(input=value) - - -@configurable(category="op", group="structure") -class StashTargetOp: - """Copy ``sample.target`` into ``metadata[key]``; ``sample.target`` unchanged. - - Args: - key: Metadata key to write. - copy: When ``True``, deepcopy ``sample.target`` before stashing. - Defaults to ``False`` (cheap pointer alias) — the typical case - is that downstream ops use ``sample._replace(target=...)`` and - don't mutate the shared value in place. - """ - - def __init__(self, key: str = "", copy: bool = False) -> None: - # Lazy / zero-arg: store config only. - self.key = key - self.copy = copy - - def __call__(self, sample: Sample) -> Sample: - sample.meta[self.key] = _copy.deepcopy(sample.target) if self.copy else sample.target - return sample - - -@configurable(category="op", group="structure") -class UnstashTargetOp: - """Set ``sample.target := metadata[key]``. - - Args: - key: Metadata key to read. - copy: When ``True`` (default), deepcopy the stashed value before - assigning. This prevents two branches that unstash the same - key from corrupting each other through downstream in-place - mutations. Set ``False`` only when the caller has audited - that no downstream op mutates the value in place. - remove: When ``True`` (default), DELETE the key from metadata after - restoring it — so the snapshot doesn't linger on the bus and - leak into a downstream sink. Set ``False`` to keep it (required - when the SAME key is unstashed again later, e.g. a fan-out that - restores the fork before several branches — only the LAST - unstash of a key may remove it). - """ - - def __init__(self, key: str = "", copy: bool = True, remove: bool = True) -> None: - # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. - self.key = key - self.copy = copy - self.remove = remove - - def __call__(self, sample: Sample) -> Sample: - value = sample.meta[self.key] - if self.copy: - value = _copy.deepcopy(value) - if self.remove: - del sample.meta[self.key] # key exists (just read above) - return sample._replace(target=value) diff --git a/sampleflux/ops/structure.py b/sampleflux/ops/structure.py index 3c7a28a..a7ca899 100644 --- a/sampleflux/ops/structure.py +++ b/sampleflux/ops/structure.py @@ -1,9 +1,9 @@ -"""Structure ops for the typed bag — reshape a :class:`~sampleflux.bag.sample.TypedSample`'s fields. +"""Structure ops for the typed bag — reshape a :class:`~sampleflux.bag.sample.Sample`'s fields. The typed analogue of the classic triple-slot plumbing (``MetadataToTargetOp``, the stash/swap family): where the old model moved values between the fixed ``input``/``target`` slots and the shared metadata dict, the bag model just RENAMES, RETAGS, COPIES, or DROPS named fields. Each op -is a thin copy-on-write wrapper over a ``TypedSample`` mutator — no payload is touched. +is a thin copy-on-write wrapper over a ``Sample`` mutator — no payload is touched. All ops are lazy / zero-arg constructible (config validated in ``__call__``) and ``@configurable(category="op", group="structure")`` so they surface as canvas nodes. @@ -14,7 +14,7 @@ from confluid import configurable from typing_extensions import get_args -from sampleflux.bag.sample import ROLES, Role, TypedSample +from sampleflux.bag.sample import ROLES, Role, Sample __all__ = ["SetRole", "RenameField", "DropField", "CopyField", "SelectFields"] @@ -35,7 +35,7 @@ def __init__(self, key: str = "", role: Role = "input") -> None: self.key = key self.role = role - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if not self.key: raise ValueError("SetRole: 'key' (the field to retag) is required") if self.role not in get_args(Role): @@ -59,7 +59,7 @@ def __init__(self, src: str = "", dst: str = "") -> None: self.src = src self.dst = dst - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if not self.src or not self.dst: raise ValueError("RenameField: both 'src' and 'dst' are required") return sample.rename(self.src, self.dst) @@ -78,7 +78,7 @@ def __init__(self, key: str = "", missing_ok: bool = False) -> None: self.key = key self.missing_ok = missing_ok - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if not self.key: raise ValueError("DropField: 'key' (the field to remove) is required") if self.key not in sample: @@ -103,7 +103,7 @@ def __init__(self, src: str = "", dst: str = "", role: Optional[Role] = None) -> self.dst = dst self.role = role - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if not self.src or not self.dst: raise ValueError("CopyField: both 'src' and 'dst' are required") if self.src not in sample: @@ -123,10 +123,10 @@ class SelectFields: def __init__(self, keys: Optional[List[str]] = None) -> None: self.keys = list(keys) if keys else [] - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: if not self.keys: raise ValueError("SelectFields: 'keys' (the fields to keep) is required") missing = [k for k in self.keys if k not in sample] if missing: raise KeyError(f"SelectFields: unknown fields {missing} (fields: {list(sample.keys())})") - return TypedSample({k: sample[k] for k in self.keys}, {k: sample.role_of(k) for k in self.keys}) + return Sample({k: sample[k] for k in self.keys}, {k: sample.role_of(k) for k in self.keys}) diff --git a/sampleflux/ops/swap.py b/sampleflux/ops/swap.py deleted file mode 100644 index 69c237c..0000000 --- a/sampleflux/ops/swap.py +++ /dev/null @@ -1,17 +0,0 @@ -"""``SwapInputTargetOp`` — exchange ``sample.input`` and ``sample.target``. - -Useful when an op operates on ``sample.input`` but you want it applied to -the target instead: swap, run the op, swap back. -""" - -from confluid import configurable - -from sampleflux.sample import Sample - - -@configurable(category="op", group="structure") -class SwapInputTargetOp: - """Exchange ``sample.input`` ↔ ``sample.target``. Metadata unchanged.""" - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.target, target=sample.input) diff --git a/sampleflux/ops/target.py b/sampleflux/ops/target.py index df40ca4..609a559 100644 --- a/sampleflux/ops/target.py +++ b/sampleflux/ops/target.py @@ -1,33 +1,18 @@ -"""Move and encode the supervised ``target`` field. - -Companions to the input↔metadata movers (:class:`~sampleflux.ops.stash.StashInputOp` / -:class:`~sampleflux.ops.stash.UnstashInputOp`) and -:class:`~sampleflux.ops.swap.SwapInputTargetOp`: - -* :class:`MetadataToTargetOp` moves a value from ``metadata`` onto ``sample.target``. -* :class:`EncodeTargetOp` / :class:`DecodeTargetOp` map ``sample.target`` through an - explicit lookup and back — the declarative analogue of scikit-learn's - ``LabelEncoder``. The label→id mapping is pinned in config, NOT fitted from - whatever labels happen to appear, so train / eval / predict share one identical - ordering. - -The first three are deliberately small, value-agnostic plumbing ops (no ``ACCEPTS`` / -``PRODUCES`` contract, like ``copy`` / ``swap`` / ``stash``). The encoded value is -written verbatim (e.g. a plain ``int``); wrap it into a framework tensor downstream -(e.g. a collate function) when a loss needs one. - -* :class:`CocoToTorchVisionDetectionOp` is the one structured-target op here: it turns a - HuggingFace / COCO ``objects`` annotation (``{bbox, category}``) into the torchvision - detection target ``{"boxes": xyxy, "labels"}`` (torch tensors). It is the generic, - image-detection counterpart of waivefront's signal-domain ``RegionsToDetectionBoxesOp``. - -The typed-bag TWINS (:class:`MetadataToTarget` / :class:`EncodeTarget` / :class:`DecodeTarget` -and the two detection twins :class:`CocoToTorchVisionDetection` / :class:`MasksToDetectionBoxes`) -are the ``TypedSample`` counterparts of the legacy ``*Op`` classes above — STRICTLY ADDITIVE, the -legacy ops untouched. Each detection twin reads one source field and writes the torchvision -detection target as a :class:`~sampleflux.Regions` item (``boxes`` = the xyxy tensor, ``labels`` = -the class-id tensor) tagged ``target``, reusing its legacy op's conversion math VERBATIM (via a -shim :class:`~sampleflux.sample.Sample`) so the numbers are byte-identical. +"""Typed-bag target-shaping transforms. + +* :class:`MetadataToTarget` promotes a field / attr value into a target ``Label``. +* :class:`EncodeTarget` / :class:`DecodeTarget` map a class-name ``Label`` to a class-id + ``Label`` and back through an explicit lookup ``mapping`` — the declarative analogue of + scikit-learn's ``LabelEncoder``. The mapping is pinned in config, NOT fitted, so + train / eval / predict share one identical ordering. +* :class:`CocoToTorchVisionDetection` turns a HuggingFace / COCO ``objects`` annotation + (``{bbox, category}``) into a torchvision detection target rendered as a + :class:`~sampleflux.Regions` item. +* :class:`MasksToDetectionBoxes` derives detection boxes from a segmentation ``Mask``. + +The detection conversions are the modality-neutral, image-detection counterparts of +waivefront's signal-domain region ops. The encoded target value is written verbatim; wrap +it into a framework tensor downstream (e.g. a collate function) when a loss needs one. """ from typing import Any, Dict, Literal, Optional @@ -36,9 +21,8 @@ from confluid import configurable from sampleflux.bag.items import Label, Mask, Regions, item_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.bag.transform import Transform -from sampleflux.sample import Sample #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo #: fails at the call site and UIs / form-specs enumerate the choices. @@ -48,9 +32,7 @@ def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: Any, op_name: str) -> Any: """Return ``mapping[value]``, or ``default`` when missing and ``ignore_unknown``. - Shared by :class:`EncodeTargetOp` / :class:`DecodeTargetOp`. A plain - module-level function (NOT a base class) so the ops stay independent - callables — SampleFlux Functional Purity. + A plain module-level function shared by :class:`EncodeTarget` / :class:`DecodeTarget`. """ if value in mapping: return mapping[value] @@ -64,276 +46,112 @@ def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: ) -@configurable(category="op", group="structure") -class MetadataToTargetOp: - """Set ``sample.target := metadata[key]``; optionally copy it to ``metadata[target_key]``. - - The metadata→target counterpart of :class:`~sampleflux.ops.stash.StashInputOp` / - :class:`~sampleflux.ops.stash.UnstashInputOp` (which move input↔metadata). Typical - use: a raw label rides in ``metadata`` and must become the supervised ``target`` - before :class:`EncodeTargetOp` overwrites it with a class id. - - Args: - key: Metadata key to read the value from into ``sample.target`` (defaults to ``""``; a missing - or empty key surfaces as a ``KeyError`` when the op runs, per the lazy-init convention). - target_key: When set, the value is also written to ``metadata[target_key]`` - (so the raw label survives a later ``EncodeTargetOp`` and can be decoded - back). ``None`` (default) leaves ``metadata`` untouched. - """ - - def __init__(self, key: str = "", target_key: Optional[str] = None) -> None: - # Lazy / zero-arg: store config only; a missing key surfaces lazily as a KeyError in __call__. - self.key = str(key) - self.target_key = str(target_key) if target_key is not None else None - - def __call__(self, sample: Sample) -> Sample: - if self.key not in sample.meta: - raise KeyError( - f"MetadataToTargetOp: sample.meta has no key {self.key!r}. " f"Available keys: {sorted(sample.meta)}" - ) - value = sample.meta[self.key] - if self.target_key is not None: - sample.meta[self.target_key] = value - return sample._replace(target=value) - - -@configurable(category="op", group="structure") -class EncodeTargetOp: - """Encode ``sample.target`` through an explicit lookup ``mapping``. - - The declarative analogue of scikit-learn's ``LabelEncoder``: maps a raw target - (typically a string label) to its class id via a config-pinned ``mapping``. - Pinning the mapping — rather than fitting it from whatever labels appear — keeps - train / eval / predict on one identical label→id ordering. The plain mapping value - is written (framework-agnostic); tensorize the target downstream when a loss needs it. +def coco_to_detection( + objects: Any, + bbox_key: str = "bbox", + category_key: str = "category", + bbox_format: BBoxFormat = "xywh", + label_offset: int = 0, +) -> Dict[str, Any]: + """Convert a COCO / HuggingFace ``objects`` mapping to ``{"boxes": [N,4] xyxy, "labels": [N]}`` tensors. - Args: - mapping: Lookup from raw target → encoded value, e.g. ``{"DJI AVATA2": 2, ...}``. - Must be non-empty. - ignore_unknown: When ``False`` (default), raise on a target missing from - ``mapping``; when ``True``, substitute ``default``. - default: Value written for an unknown target when ``ignore_unknown=True``. - Defaults to ``0``. + Each box is, by COCO convention, ``[x, y, w, h]`` in absolute pixels; ``category`` is an + integer class id. An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the + negative-example contract torchvision detectors accept). """ - - def __init__( - self, mapping: Optional[Dict[Any, Any]] = None, ignore_unknown: bool = False, default: Any = 0 - ) -> None: - # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. - self.mapping = dict(mapping) if mapping else {} - self.ignore_unknown = bool(ignore_unknown) - self.default = default - - def __call__(self, sample: Sample) -> Sample: - if not self.mapping: - raise ValueError("EncodeTargetOp: mapping must contain at least one entry.") - encoded = _lookup(sample.target, self.mapping, self.ignore_unknown, self.default, "EncodeTargetOp") - return sample._replace(target=encoded) - - -@configurable(category="op", group="structure") -class DecodeTargetOp: - """Decode ``sample.target`` through a lookup ``mapping`` (inverse of :class:`EncodeTargetOp`). - - Maps an encoded target (e.g. an integer class id) back to its label (e.g. a class - name) — the readback half used in prediction / reporting. - - Args: - mapping: Lookup from encoded value → decoded value, e.g. ``{2: "DJI AVATA2", ...}``. - Must be non-empty. - ignore_unknown: When ``False`` (default), raise on a target missing from - ``mapping``; when ``True``, substitute ``default``. - default: Value written for an unknown target when ``ignore_unknown=True``. - Defaults to ``None``. - """ - - def __init__( - self, mapping: Optional[Dict[Any, Any]] = None, ignore_unknown: bool = False, default: Any = None - ) -> None: - # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. - self.mapping = dict(mapping) if mapping else {} - self.ignore_unknown = bool(ignore_unknown) - self.default = default - - def __call__(self, sample: Sample) -> Sample: - if not self.mapping: - raise ValueError("DecodeTargetOp: mapping must contain at least one entry.") - decoded = _lookup(sample.target, self.mapping, self.ignore_unknown, self.default, "DecodeTargetOp") - return sample._replace(target=decoded) - - -@configurable(category="op", group="structure") -class CocoToTorchVisionDetectionOp: - """Convert a HuggingFace / COCO ``objects`` annotation to a torchvision detection target. - - HuggingFace object-detection datasets (e.g. ``cppe-5``) carry per-image annotations as an - ``objects`` mapping — ``{"bbox": [[...], ...], "category": [...], ...}`` — where each box is, - by COCO convention, ``[x, y, w, h]`` in absolute pixels and ``category`` is an integer class - id. ``HuggingFaceSource(target_feature="objects")`` lands that mapping verbatim on - ``sample.target``; this op rewrites it to the shape ``raidar.detection.detection_collate_fn`` - and the detection trainer consume:: - - sample.target = {"boxes": [N, 4] float32 xyxy-pixel, "labels": [N] int64} - - The modality-neutral, image-detection counterpart of waivefront's signal-domain - :class:`~waivefront.targets.RegionsToDetectionBoxesOp` (which projects time/frequency - regions) — it lives in core sampleflux because the COCO→xyxy conversion is fully generic. - The input image is left untouched (tensorize it with :class:`~sampleflux.ops.torch.ToTensorOp`). - An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract - torchvision detectors accept). - - Args: - bbox_key: Key in the objects mapping holding per-box coordinates (default ``"bbox"``). - category_key: Key holding the per-box integer class ids (default ``"category"``). - bbox_format: Box layout in pixels — ``xywh`` (COCO, default), ``xyxy``, or ``cxcywh``; output is xyxy. - label_offset: Added to each class id (default ``0``). Set ``1`` to reserve class ``0`` for background. + import torch + + if not isinstance(objects, dict): + raise TypeError( + f"coco_to_detection: expected a COCO/HF objects mapping " + f"(a dict with {bbox_key!r}/{category_key!r}); got {type(objects).__name__}." + ) + raw_boxes = objects.get(bbox_key) or [] + raw_labels = objects.get(category_key) or [] + + if len(raw_boxes): + boxes = torch.as_tensor(raw_boxes, dtype=torch.float32).reshape(-1, 4) + if bbox_format == "xywh": # COCO: top-left + size + x, y, w, h = boxes.unbind(-1) + boxes = torch.stack([x, y, x + w, y + h], dim=-1) + elif bbox_format == "cxcywh": # center + size + cx, cy, w, h = boxes.unbind(-1) + boxes = torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1) + # "xyxy": already in the output layout + else: + boxes = torch.zeros((0, 4), dtype=torch.float32) + + if len(raw_labels): + labels = torch.as_tensor(list(raw_labels), dtype=torch.int64).reshape(-1) + label_offset + else: + labels = torch.zeros((0,), dtype=torch.int64) + + return {"boxes": boxes, "labels": labels} + + +def masks_to_detection( + mask: Any, + label: int = 1, + connected: bool = False, + min_area: int = 1, + connectivity: int = 4, +) -> Dict[str, Any]: + """Derive ``{"boxes": [N,4] xyxy, "labels": [N]}`` tensors from a 2-D integer segmentation mask. + + ``connected=False`` (default) — an instance mask: each distinct non-zero pixel value is + one object. ``connected=True`` — binarize then split into connected components. Every box + gets class id ``label``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. """ - - def __init__( - self, - bbox_key: str = "bbox", - category_key: str = "category", - bbox_format: BBoxFormat = "xywh", - label_offset: int = 0, - ) -> None: - # Lazy / zero-arg: store config only. The objects-shaped target is validated in __call__. - self.bbox_key = str(bbox_key) - self.category_key = str(category_key) - self.bbox_format = bbox_format - self.label_offset = int(label_offset) - - def __call__(self, sample: Sample) -> Sample: - # torch is imported lazily so this module stays import-light for the value-agnostic - # plumbing ops above (which need no framework). - import torch - - objects = sample.target - if not isinstance(objects, dict): - raise TypeError( - f"CocoToTorchVisionDetectionOp: sample.target must be a COCO/HF objects mapping " - f"(a dict with {self.bbox_key!r}/{self.category_key!r}); got {type(objects).__name__}. " - "Wire HuggingFaceSource(target_feature='objects') upstream." - ) - raw_boxes = objects.get(self.bbox_key) or [] - raw_labels = objects.get(self.category_key) or [] - - if len(raw_boxes): - boxes = torch.as_tensor(raw_boxes, dtype=torch.float32).reshape(-1, 4) - if self.bbox_format == "xywh": # COCO: top-left + size - x, y, w, h = boxes.unbind(-1) - boxes = torch.stack([x, y, x + w, y + h], dim=-1) - elif self.bbox_format == "cxcywh": # center + size - cx, cy, w, h = boxes.unbind(-1) - boxes = torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1) - # "xyxy": already in the output layout - else: - boxes = torch.zeros((0, 4), dtype=torch.float32) - - if len(raw_labels): - labels = torch.as_tensor(list(raw_labels), dtype=torch.int64).reshape(-1) + self.label_offset - else: - labels = torch.zeros((0,), dtype=torch.int64) - - return sample._replace(target={"boxes": boxes, "labels": labels}) + import torch + + if hasattr(mask, "convert"): # PIL image (e.g. an 'L' instance mask) + mask = np.array(mask) + mask = np.asarray(mask) + if mask.ndim != 2: + raise TypeError( + f"masks_to_detection: expected a 2-D segmentation mask; got shape {getattr(mask, 'shape', None)}." + ) + + boxes: list = [] + if connected: + from sampleflux.ops.numpy import connected_component_bboxes + + for r0, r1, c0, c1 in connected_component_bboxes(mask != 0, min_area, connectivity): + boxes.append((float(c0), float(r0), float(c1 + 1), float(r1 + 1))) + else: + for value in np.unique(mask): + if int(value) == 0: + continue + ys, xs = np.where(mask == value) + if int(ys.size) < min_area: + continue + boxes.append((float(xs.min()), float(ys.min()), float(xs.max() + 1), float(ys.max() + 1))) + + if boxes: + boxes_t = torch.tensor(boxes, dtype=torch.float32) + labels_t = torch.full((len(boxes),), label, dtype=torch.int64) + else: + boxes_t = torch.zeros((0, 4), dtype=torch.float32) + labels_t = torch.zeros((0,), dtype=torch.int64) + return {"boxes": boxes_t, "labels": labels_t} @configurable(category="op", group="structure") -class MasksToDetectionBoxesOp: - """Convert a segmentation MASK on ``sample.target`` to a torchvision detection target. - - Reads a 2-D integer mask (PIL ``L`` image or ndarray) and rewrites ``sample.target`` to - ``{"boxes": [N,4] float32 xyxy-pixel, "labels": [N] int64}`` — the tight per-object box. This is - the derivation the official torchvision **Penn-Fudan** object-detection tutorial performs (the - dataset ships masks, not boxes). Two object-separation modes: - - * ``connected=False`` (default) — an **instance mask**: each distinct non-zero pixel value is one - object (box = the tight extent of ``mask == value``). Penn-Fudan's ``instance_id`` mask (pixels - ``1..N``, one per pedestrian) is exactly this — exact even when objects touch. - * ``connected=True`` — a **binary / semantic mask**: binarize (non-zero), then split into connected - components via :func:`sampleflux.ops.numpy.connected_component_bboxes` (one box per blob). Use for - a semantic mask (all objects share one value) or a model's predicted foreground mask. - - Every box gets class id ``label`` (one foreground class; class 0 stays background — so a 1-class - dataset like Penn-Fudan derives ``num_classes = 2``). The input image is left untouched (tensorize - with :class:`~sampleflux.ops.torch.ToTensorOp` ``mode="RGB"``). An empty mask yields empty ``[0,4]`` / - ``[0]`` tensors (the negative-example contract torchvision detectors accept). - - Args: - label: Foreground class id assigned to every derived box (default ``1``; class 0 = background). - connected: True = connected-components on a binary mask; False (default) = each non-zero value is one instance. - min_area: Drop objects whose mask area (in pixels) is below this (default ``1``). - connectivity: Connected-components neighborhood when ``connected=True`` — ``4`` or ``8`` (default ``4``). - """ - - def __init__(self, label: int = 1, connected: bool = False, min_area: int = 1, connectivity: int = 4) -> None: - # Lazy / zero-arg: store config only; the mask shape is validated in __call__. - self.label = int(label) - self.connected = bool(connected) - self.min_area = int(min_area) - self.connectivity = int(connectivity) - - def __call__(self, sample: Sample) -> Sample: - import numpy as np - import torch - - mask = sample.target - if hasattr(mask, "convert"): # PIL image (e.g. an 'L' instance mask) - mask = np.array(mask) - mask = np.asarray(mask) - if mask.ndim != 2: - raise TypeError( - f"MasksToDetectionBoxesOp: sample.target must be a 2-D segmentation mask " - f"(PIL 'L' image or 2-D array); got shape {getattr(mask, 'shape', None)}. " - "Wire HuggingFaceSource(target_feature='') upstream." - ) - - boxes: list = [] - if self.connected: - from sampleflux.ops.numpy import connected_component_bboxes - - # row/col-inclusive (r0,r1,c0,c1) → xyxy-pixel (x0,y0,x1,y1) with exclusive far edge. - for r0, r1, c0, c1 in connected_component_bboxes(mask != 0, self.min_area, self.connectivity): - boxes.append((float(c0), float(r0), float(c1 + 1), float(r1 + 1))) - else: - for value in np.unique(mask): - if int(value) == 0: - continue - ys, xs = np.where(mask == value) - if int(ys.size) < self.min_area: - continue - boxes.append((float(xs.min()), float(ys.min()), float(xs.max() + 1), float(ys.max() + 1))) - - if boxes: - boxes_t = torch.tensor(boxes, dtype=torch.float32) - labels_t = torch.full((len(boxes),), self.label, dtype=torch.int64) - else: - boxes_t = torch.zeros((0, 4), dtype=torch.float32) - labels_t = torch.zeros((0,), dtype=torch.int64) - return sample._replace(target={"boxes": boxes_t, "labels": labels_t}) +class MetadataToTarget(Transform): + """Promote a field / attr value into a target ``Label``. + Reads a value from a SOURCE field (``field``; blank picks the first ``Label``, else the + first field) — either the field's natural value (a ``Label``'s ``.value``, otherwise the + item's array payload) or, when ``key`` is set, the named ATTRIBUTE of the source item — + and writes a fresh :class:`~sampleflux.Label` under ``output`` tagged ``target``. -@configurable(category="op", group="structure") -class MetadataToTarget(Transform): - """Typed twin of :class:`MetadataToTargetOp` — promote a field / attr value into a target ``Label``. - - The typed-bag counterpart of :class:`MetadataToTargetOp`. The legacy op copies - ``metadata[key]`` onto ``sample.target``, but the typed model has NO shared metadata dict — every - item OWNS its metadata, and the supervised label already rides a :class:`~sampleflux.Label` field. - So this twin reads a value from a SOURCE field (``field``; blank picks the first ``Label``, else - the first field) — either the field's natural value (a ``Label``'s ``.value``, otherwise the - item's array payload) or, when ``key`` is set, the named ATTRIBUTE of the source item — and writes - a fresh :class:`~sampleflux.Label` under ``output`` tagged ``target``. - - REDUNDANCY. In a typical typed classification pipeline the source emits the label directly as a - ``Label`` field already tagged ``target``, so this op is usually a NO-OP-ish re-home and is NOT - needed. It is provided for parity / config-compat with the legacy ``metadata → target`` step and - for the case where a label rode as another item's attribute (``key=``) and must become a - dedicated target ``Label``. + In a typical typed classification pipeline the source emits the label directly as a + ``Label`` field already tagged ``target``, so this op is usually a NO-OP-ish re-home; it + exists for the case where a label rode as another item's attribute (``key=``). Args: field: Source field to read; blank (default) picks the first ``Label`` field, else the first field. - key: Optional attribute name to read off the source item (e.g. a carried label attr); blank - (default) reads the item's natural value (a ``Label``'s ``.value``, else its array payload). + key: Optional attribute name to read off the source item; blank (default) reads the item's natural value. output: Field the target ``Label`` is written to (added if new); its role is set to ``target``. """ @@ -347,7 +165,7 @@ def __init__(self, field: str = "", key: str = "", output: str = "target") -> No self.key = str(key) self.output = str(output) - def _find_source(self, sample: TypedSample) -> str: + def _find_source(self, sample: Sample) -> str: """Resolve the KEY of the source field (``self.field``, else first ``Label``, else first field).""" if self.field: if self.field not in sample.keys(): @@ -361,7 +179,7 @@ def _find_source(self, sample: TypedSample) -> str: return key raise ValueError("MetadataToTarget: sample is empty — no source field to read") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: key = self._find_source(sample) item = sample[key] if self.key: @@ -380,19 +198,13 @@ def __call__(self, sample: TypedSample) -> TypedSample: @configurable(category="op", group="structure") class EncodeTarget(Transform): - """Typed twin of :class:`EncodeTargetOp` — a class-NAME ``Label`` → a class-ID ``Label`` (role ``target``). - - The typed-bag counterpart of :class:`EncodeTargetOp`: it reads a :class:`~sampleflux.Label` field - (``field``; blank picks the first ``Label``) whose ``.value`` is a raw class name and maps it to - its class id through the config-pinned ``mapping`` — the declarative ``LabelEncoder`` analogue. - This twin REUSES the legacy ``EncodeTargetOp`` verbatim (its non-empty-mapping validation AND its - shared ``_lookup`` logic), so the encoded value is byte-identical. The result is a new - :class:`~sampleflux.Label` (carrying the source label's ``classes`` vocabulary) written under - ``output`` — blank (default) replaces the source field in place — tagged ``target``. + """A class-NAME ``Label`` → a class-ID ``Label`` (role ``target``). - Pinning the mapping (rather than fitting it) keeps train / eval / predict on one identical - label→id ordering. The non-empty-mapping requirement is validated LAZILY when the op runs (the - zero-arg default stays constructible per the lazy-init convention). + Reads a :class:`~sampleflux.Label` field (``field``; blank picks the first ``Label``) + whose ``.value`` is a raw class name and maps it to its class id through the config-pinned + ``mapping`` — the declarative ``LabelEncoder`` analogue. The result is a new + :class:`~sampleflux.Label` (carrying the source label's ``classes`` vocabulary) written + under ``output`` — blank (default) replaces the source field in place — tagged ``target``. Args: mapping: Lookup from raw label name → class id, e.g. ``{"DJI AVATA2": 2, ...}``. Must be non-empty. @@ -424,7 +236,7 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_label(self, sample: TypedSample) -> str: + def _find_label(self, sample: Sample) -> str: """Resolve the KEY of the ``Label`` field to encode (``self.field`` or the first ``Label``).""" if self.field: if self.field not in sample.keys(): @@ -437,13 +249,12 @@ def _find_label(self, sample: TypedSample) -> str: return key raise ValueError(f"EncodeTarget: no Label field in sample (fields: {list(sample.keys())})") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: + if not self.mapping: + raise ValueError("EncodeTarget: mapping must contain at least one entry.") key = self._find_label(sample) label = sample[key] - # Reuse the legacy op VERBATIM (non-empty validation + shared _lookup) for byte-parity. - encoded = EncodeTargetOp(self.mapping, self.ignore_unknown, self.default)( - Sample(input=None, target=label.value, metadata={}) - ).target + encoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") out_key = self.output or key out = sample.replace_field(out_key, Label(encoded, classes=label.classes)) return out.set_role(out_key, "target") @@ -451,14 +262,12 @@ def __call__(self, sample: TypedSample) -> TypedSample: @configurable(category="op", group="structure") class DecodeTarget(Transform): - """Typed twin of :class:`DecodeTargetOp` — a class-ID ``Label`` → a class-NAME ``Label`` (inverse of encode). + """A class-ID ``Label`` → a class-NAME ``Label`` (inverse of :class:`EncodeTarget`). - The typed-bag counterpart of :class:`DecodeTargetOp`: it reads a :class:`~sampleflux.Label` field - (``field``; blank picks the first ``Label``) whose ``.value`` is an encoded class id and maps it - back to its label name through ``mapping`` — the readback half used in prediction / reporting. - This twin REUSES the legacy ``DecodeTargetOp`` verbatim, so the decoded value is byte-identical. - The result is a new :class:`~sampleflux.Label` (carrying the source label's ``classes``) written - under ``output`` — blank (default) replaces the source field in place — tagged ``target``. + Reads a :class:`~sampleflux.Label` field (``field``; blank picks the first ``Label``) + whose ``.value`` is an encoded class id and maps it back to its label name through + ``mapping`` — the readback half used in prediction / reporting. The result is a new + :class:`~sampleflux.Label` written under ``output`` (blank replaces in place) tagged ``target``. Args: mapping: Lookup from class id → label name, e.g. ``{2: "DJI AVATA2", ...}``. Must be non-empty. @@ -490,7 +299,7 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_label(self, sample: TypedSample) -> str: + def _find_label(self, sample: Sample) -> str: """Resolve the KEY of the ``Label`` field to decode (``self.field`` or the first ``Label``).""" if self.field: if self.field not in sample.keys(): @@ -503,13 +312,12 @@ def _find_label(self, sample: TypedSample) -> str: return key raise ValueError(f"DecodeTarget: no Label field in sample (fields: {list(sample.keys())})") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: + if not self.mapping: + raise ValueError("DecodeTarget: mapping must contain at least one entry.") key = self._find_label(sample) label = sample[key] - # Reuse the legacy op VERBATIM (non-empty validation + shared _lookup) for byte-parity. - decoded = DecodeTargetOp(self.mapping, self.ignore_unknown, self.default)( - Sample(input=None, target=label.value, metadata={}) - ).target + decoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") out_key = self.output or key out = sample.replace_field(out_key, Label(decoded, classes=label.classes)) return out.set_role(out_key, "target") @@ -517,24 +325,14 @@ def __call__(self, sample: TypedSample) -> TypedSample: @configurable(category="op", group="structure") class CocoToTorchVisionDetection(Transform): - """Typed twin of :class:`CocoToTorchVisionDetectionOp` — a COCO / HF ``objects`` annotation → a target ``Regions``. - - The typed-bag counterpart of :class:`CocoToTorchVisionDetectionOp`: it reads a source field - (``field``; blank picks the first :class:`~sampleflux.Label` field, else the first field) - carrying a HuggingFace / COCO ``objects`` mapping — ``{"bbox": [[...], ...], "category": [...]}``, - each box ``[x, y, w, h]`` in absolute pixels — either the field's natural value (a ``Label``'s - ``.value``, else the item's payload) and rewrites it to the torchvision detection target. This - twin REUSES the legacy ``CocoToTorchVisionDetectionOp`` VERBATIM (its objects-shape validation - AND its bbox/category conversion math on a shim :class:`~sampleflux.sample.Sample`), so the - ``boxes`` / ``labels`` tensors are byte-identical. - - The target rides as a :class:`~sampleflux.Regions` item under ``output`` (``boxes`` = the - ``[N, 4]`` float32 xyxy-pixel tensor, ``labels`` = the ``[N]`` int64 class-id tensor) tagged - ``target`` — the natural typed home for a bounding-box set, and the batch-friendly one (the - typed collate gathers per-sample ``Regions`` into a list of targets, the variable-N detection - batch convention, exactly as the classification :class:`EncodeTarget` twin gathers a target - ``Label``). An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example - contract torchvision detectors accept). + """A COCO / HF ``objects`` annotation → a target ``Regions``. + + Reads a source field (``field``; blank picks the first :class:`~sampleflux.Label`, else the + first field) carrying a HuggingFace / COCO ``objects`` mapping and rewrites it to the + torchvision detection target, riding as a :class:`~sampleflux.Regions` item under + ``output`` (``boxes`` = the ``[N, 4]`` float32 xyxy tensor, ``labels`` = the ``[N]`` int64 + class-id tensor) tagged ``target``. An empty annotation yields empty ``[0,4]`` / ``[0]`` + tensors (the negative-example contract). Args: bbox_key: Key in the objects mapping holding per-box coordinates (default ``"bbox"``). @@ -566,7 +364,7 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_source(self, sample: TypedSample) -> str: + def _find_source(self, sample: Sample) -> str: """Resolve the KEY of the source field (``self.field``, else the first ``Label``, else the first field).""" if self.field: if self.field not in sample.keys(): @@ -580,37 +378,23 @@ def _find_source(self, sample: TypedSample) -> str: return key raise ValueError("CocoToTorchVisionDetection: sample is empty — no source field to read") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: key = self._find_source(sample) item = sample[key] objects = item.value if isinstance(item, Label) else item_data(item) - # Reuse the legacy op VERBATIM (objects-shape validation + bbox/category math) on a shim - # Sample so the boxes / labels tensors are byte-identical. - target = CocoToTorchVisionDetectionOp(self.bbox_key, self.category_key, self.bbox_format, self.label_offset)( - Sample(input=None, target=objects, metadata={}) - ).target + target = coco_to_detection(objects, self.bbox_key, self.category_key, self.bbox_format, self.label_offset) out = sample.replace_field(self.output, Regions(boxes=target["boxes"], labels=target["labels"])) return out.set_role(self.output, "target") @configurable(category="op", group="structure") class MasksToDetectionBoxes(Transform): - """Typed twin of :class:`MasksToDetectionBoxesOp` — a segmentation ``Mask`` → a target ``Regions``. - - The typed-bag counterpart of :class:`MasksToDetectionBoxesOp`: it reads the - :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, else the first - array-bearing item) as a 2-D integer mask and derives one tight ``[x0,y0,x1,y1]`` box per object - — either one box per distinct non-zero pixel value (``connected=False``, an instance mask) or - one box per connected component of the binarized mask (``connected=True``, via the shared - :func:`sampleflux.ops.numpy.connected_component_bboxes` helper). This twin REUSES the legacy - ``MasksToDetectionBoxesOp`` VERBATIM on a shim :class:`~sampleflux.sample.Sample`, so the - ``boxes`` / ``labels`` tensors are byte-identical. - - The target rides as a :class:`~sampleflux.Regions` item under ``output`` (``boxes`` = the - ``[N, 4]`` float32 xyxy-pixel tensor, every box's ``labels`` id = ``label``) tagged ``target`` — - the same batch-friendly representation the sibling :class:`CocoToTorchVisionDetection` twin - writes. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract - torchvision detectors accept). + """A segmentation ``Mask`` → a target ``Regions``. + + Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, + else the first array-bearing item) as a 2-D integer mask and derives one tight + ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~sampleflux.Regions` item + under ``output`` tagged ``target``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. Args: label: Foreground class id assigned to every derived box (default ``1``; class 0 = background). @@ -642,7 +426,7 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_mask(self, sample: TypedSample) -> np.ndarray: + def _find_mask(self, sample: Sample) -> np.ndarray: """Resolve the mask array (``self.field``, else the first ``Mask``, else the first array-bearing item).""" if self.field: if self.field not in sample.keys(): @@ -669,26 +453,19 @@ def _find_mask(self, sample: TypedSample) -> np.ndarray: raise TypeError(f"MasksToDetectionBoxes: expected an np.ndarray mask, got {type(data).__name__}") return data - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: mask = self._find_mask(sample) - # Reuse the legacy op VERBATIM (instance / connected-component derivation) on a shim Sample - # so the boxes / labels tensors are byte-identical. - target = MasksToDetectionBoxesOp(self.label, self.connected, self.min_area, self.connectivity)( - Sample(input=None, target=mask, metadata={}) - ).target + target = masks_to_detection(mask, self.label, self.connected, self.min_area, self.connectivity) out = sample.replace_field(self.output, Regions(boxes=target["boxes"], labels=target["labels"])) return out.set_role(self.output, "target") __all__ = [ - "MetadataToTargetOp", - "EncodeTargetOp", - "DecodeTargetOp", - "CocoToTorchVisionDetectionOp", - "MasksToDetectionBoxesOp", "MetadataToTarget", "EncodeTarget", "DecodeTarget", "CocoToTorchVisionDetection", "MasksToDetectionBoxes", + "coco_to_detection", + "masks_to_detection", ] diff --git a/sampleflux/ops/torch.py b/sampleflux/ops/torch.py index e059c5e..ab54834 100644 --- a/sampleflux/ops/torch.py +++ b/sampleflux/ops/torch.py @@ -1,4 +1,4 @@ -from typing import Optional, Sequence, Union +from typing import Any, Optional import numpy as np import torch @@ -6,237 +6,53 @@ from sampleflux.bag.items import Image as ImageItem from sampleflux.bag.items import NDArrayItem, item_data -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.bag.transform import Transform -from sampleflux.sample import Sample -from sampleflux.typespec import ArrayType, PythonType, SampleType, UnionType -_TORCH = ArrayType(frameworks={"torch"}) -_TORCH_FLOAT = ArrayType(dtype="floating", frameworks={"torch"}) +def to_tensor(img: Any, normalize: bool = True, mode: Optional[str] = None) -> torch.Tensor: + """Convert a PIL image / NumPy array to a CHW ``torch.Tensor``. -@configurable(category="op", group="torch") -class ToTensorOp: - """ - Converts input (PIL Image, NumPy array, etc.) to a Torch Tensor. - - Args: - normalize: When ``True``, scale integer pixel inputs into the ``[0, 1]`` float range during conversion. - mode: Optional PIL mode to convert to (e.g. "RGB" forces 3 channels); None (default) arrays as-is. - """ - - ACCEPTS = SampleType(input=UnionType((PythonType("PIL.Image.Image"), ArrayType(frameworks={"numpy"})))) - PRODUCES = SampleType(input=_TORCH) - - def __init__(self, normalize: bool = True, mode: Optional[str] = None): - self.normalize = normalize - self.mode = mode - - def __call__(self, sample: Sample) -> Sample: - img = sample.input - - # Handle PIL / PngImageFile - if hasattr(img, "convert"): - # Optionally coerce the PIL mode (e.g. "RGB") so a mixed-mode dataset - # (RGBA / grayscale / palette samples) yields a uniform channel count. - if self.mode is not None: - img = img.convert(self.mode) - img = np.array(img) - - # Convert to Tensor - if isinstance(img, np.ndarray): - # Standard Vision format: [H, W, C] -> [C, H, W] - if img.ndim == 3: - img = img.transpose(2, 0, 1) - elif img.ndim == 2: - img = img[np.newaxis, :] - - tensor = torch.from_numpy(img) - else: - tensor = torch.as_tensor(img) - - # Normalize 0-255 to 0-1 - if self.normalize and tensor.dtype == torch.uint8: - tensor = tensor.float() / 255.0 - elif self.normalize and tensor.max() > 1.0: - # Fallback for floats that are still in 0-255 range - tensor = tensor / 255.0 - - return sample._replace(input=tensor) - - -@configurable(category="op", group="torch") -class RescaleOp: - """Affine rescale a torch.Tensor from ``[in_min, in_max]`` to ``[out_min, out_max]``. - - The default ``out_min=0.0`` / ``out_max=1.0`` covers the common - ``[0, 255] -> [0, 1]`` image-normalization case. Integer dtypes are - promoted to ``float32`` (``float64`` is preserved). - - Args: - in_min: Lower edge of the input range. Default ``0.0``. - in_max: Upper edge of the input range, must be ``> in_min``. Default ``1.0``. - out_min: Lower edge of the output range. Default ``0.0``. - out_max: Upper edge of the output range, must be ``> out_min``. Default ``1.0``. - clip: When True (default), clamp values outside ``[in_min, in_max]`` - before rescaling. When False, extrapolate linearly. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH_FLOAT) - - def __init__( - self, - in_min: float = 0.0, - in_max: float = 1.0, - out_min: float = 0.0, - out_max: float = 1.0, - clip: bool = True, - ) -> None: - # Lazy / zero-arg: store config only; the bound relationships are validated lazily in __call__. - self.in_min = float(in_min) - self.in_max = float(in_max) - self.out_min = float(out_min) - self.out_max = float(out_max) - self.clip = bool(clip) - - def __call__(self, sample: Sample) -> Sample: - if not (self.in_min < self.in_max): - raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={self.in_min}, in_max={self.in_max}") - if not (self.out_min < self.out_max): - raise ValueError( - f"RescaleOp: require out_min < out_max; got out_min={self.out_min}, out_max={self.out_max}" - ) - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"RescaleOp expects a torch.Tensor, got {type(tensor).__name__}") - if tensor.dtype != torch.float32 and tensor.dtype != torch.float64: - tensor = tensor.float() - src = tensor.clamp(self.in_min, self.in_max) if self.clip else tensor - scaled = (src - self.in_min) / (self.in_max - self.in_min) - out = scaled * (self.out_max - self.out_min) + self.out_min - return sample._replace(input=out) - - -@configurable(category="op", group="torch") -class SqueezeOp: - """Remove size-1 dimensions from a ``torch.Tensor``. - - Args: - dim: Axis index to remove. When ``None`` (default), all size-1 dimensions are removed. - When specified, the dimension must have size 1; otherwise the tensor is returned unchanged - (matching ``torch.squeeze`` semantics). - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH) - - def __init__(self, dim: Optional[int] = None) -> None: - self.dim = dim - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"SqueezeOp expects a torch.Tensor, got {type(tensor).__name__}") - out = torch.squeeze(tensor) if self.dim is None else torch.squeeze(tensor, self.dim) - return sample._replace(input=out) - - -@configurable(category="op", group="torch") -class UnsqueezeOp: - """Insert a size-1 dimension at the specified position in a ``torch.Tensor``. - - Args: - dim: Axis index at which the new dimension is inserted. Default ``0``. + A PIL image is optionally mode-coerced (``mode="RGB"`` forces 3 channels) then arrayed; + an ``[H, W, C]`` array is transposed to ``[C, H, W]`` (a 2-D array gets a leading channel + axis). With ``normalize`` an integer / 0-255-float payload is scaled into ``[0, 1]``. """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH) - - def __init__(self, dim: int = 0) -> None: - self.dim = dim - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"UnsqueezeOp expects a torch.Tensor, got {type(tensor).__name__}") - return sample._replace(input=torch.unsqueeze(tensor, self.dim)) - - -@configurable(category="op", group="torch") -class StandardizeOp: - """ - Standardizes tensor values with given mean and standard deviation. - - Formula: output = (input - mean) / std - - mean/std can be a single float (applied uniformly) or a sequence of - per-channel values that broadcasts over [C, H, W] format. - - Args: - mean: Mean to subtract — a single float (uniform) or a per-channel sequence broadcasting over [C, H, W]. - std: Standard deviation to divide by — a single float (uniform) or a per-channel sequence. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH_FLOAT) - - def __init__(self, mean: Union[float, Sequence[float]] = 0.0, std: Union[float, Sequence[float]] = 1.0): - # Lazy / zero-arg: store config only. The defaults (mean 0, std 1) are an identity standardize. - self.mean = mean - self.std = std - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"StandardizeOp expects a torch.Tensor, got {type(tensor).__name__}") - - if tensor.dtype != torch.float32 and tensor.dtype != torch.float64: - tensor = tensor.float() - - mean_t = torch.tensor( - [self.mean] if isinstance(self.mean, (int, float)) else self.mean, - dtype=tensor.dtype, - device=tensor.device, - ) - std_t = torch.tensor( - [self.std] if isinstance(self.std, (int, float)) else self.std, - dtype=tensor.dtype, - device=tensor.device, - ) - - # Reshape to [C, 1, 1, ...] for broadcasting over [C, H, W] - mean_t = mean_t.view(-1, *([1] * (tensor.ndim - 1))) - std_t = std_t.view(-1, *([1] * (tensor.ndim - 1))) - - tensor = (tensor - mean_t) / std_t - - return sample._replace(input=tensor) + if hasattr(img, "convert"): + if mode is not None: + img = img.convert(mode) + img = np.array(img) + + if isinstance(img, np.ndarray): + if img.ndim == 3: + img = img.transpose(2, 0, 1) + elif img.ndim == 2: + img = img[np.newaxis, :] + tensor = torch.from_numpy(img) + else: + tensor = torch.as_tensor(img) + + if normalize and tensor.dtype == torch.uint8: + tensor = tensor.float() / 255.0 + elif normalize and tensor.max() > 1.0: + tensor = tensor / 255.0 + return tensor @configurable(category="op", group="torch") class ToTensor(Transform): - """Typed twin of :class:`ToTensorOp` — an array-bearing field → a CHW-float ``Image`` item. + """An array-bearing field → a CHW-float ``Image`` item. - The typed-bag counterpart of :class:`ToTensorOp`: it reads the payload of an array-bearing - field (blank ``field`` picks the first array/PIL-bearing item — typically the - :class:`~sampleflux.Image` a :class:`~sampleflux.ops.image.ConvertToImage` produced), runs the - SAME HWC→CHW transpose + ``normalize`` conversion (this twin REUSES the legacy op verbatim on a - shim ``Sample``, so the numbers are identical), and writes a CHW-layout :class:`~sampleflux.Image` - back. By default it REPLACES the resolved field in place (``output`` blank), so the field's role - is preserved — the model's working image tensor stays the ``input`` it already was; set - ``output`` to write a NEW field (tagged ``input``) instead. Any other field passes through - untouched. + Reads the payload of an array-bearing field (blank ``field`` picks the first array/PIL-bearing + item — typically the :class:`~sampleflux.Image` a :class:`~sampleflux.ops.image.ConvertToImage` + produced), runs the HWC→CHW transpose + ``normalize`` conversion (:func:`to_tensor`), and writes + a CHW-layout :class:`~sampleflux.Image` back. By default it REPLACES the resolved field in place + (``output`` blank), so the field's role is preserved; set ``output`` to write a NEW field + (tagged ``input``) instead. Any other field passes through untouched. IMPORTANT — payload dtype. A :class:`~sampleflux.NDArrayItem` (which ``Image`` is) coerces its payload through ``np.asarray`` on construction, so it CANNOT hold a live ``torch.Tensor``: the - stored payload is a CHW ``float32`` **numpy** array whose values are byte-identical to the legacy - ``ToTensorOp`` tensor (``legacy.input.numpy()``). The typed collate (``typed_collate``) stacks - these field payloads with ``np.stack`` into a batched CHW-float array; the numpy→``torch.Tensor`` - conversion happens at the collate / model boundary (exactly as for any numpy-backed dataset). A - torch-``Tensor``-subclass item that would let a field carry a live tensor is the documented - follow-up (see ``sampleflux.bag.items`` — "torch payloads ride in wrapper items in the PoC"). + stored payload is a CHW ``float32`` **numpy** array. The typed collate stacks these payloads with + ``np.stack``; the numpy→``torch.Tensor`` conversion happens at the collate / model boundary. Args: normalize: When ``True`` (default), scale integer pixel inputs into the ``[0, 1]`` float range. @@ -263,7 +79,7 @@ def __init__( self.field = field self.output = output - def _find_field(self, sample: TypedSample) -> str: + def _find_field(self, sample: Sample) -> str: """Resolve the KEY of the field to tensorize (``self.field`` or the first array/PIL item).""" if self.field: if self.field not in sample.keys(): @@ -275,15 +91,16 @@ def _find_field(self, sample: TypedSample) -> str: return key raise ValueError(f"ToTensor: no array-bearing field in sample (fields: {list(sample.keys())})") - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: key = self._find_field(sample) data = item_data(sample[key]) - # Reuse the legacy op's conversion VERBATIM on a shim Sample so the CHW / normalization - # values are identical; NDArrayItem then coerces the tensor to a CHW float32 numpy payload. - tensor = ToTensorOp(self.normalize, self.mode)(Sample(input=data, target=None, metadata={})).input + tensor = to_tensor(data, self.normalize, self.mode) arr = tensor.detach().cpu().numpy() out_key = self.output or key out = sample.replace_field(out_key, ImageItem(arr, layout="CHW")) if self.output: out = out.set_role(out_key, "input") return out + + +__all__ = ["ToTensor", "to_tensor"] diff --git a/sampleflux/ops/torchvision.py b/sampleflux/ops/torchvision.py index 526a560..a8e22fb 100644 --- a/sampleflux/ops/torchvision.py +++ b/sampleflux/ops/torchvision.py @@ -32,8 +32,9 @@ from confluid import configurable from loggair import get_logger +from sampleflux.bag.items import Mask, item_data, with_data +from sampleflux.bag.sample import Sample, primary from sampleflux.ops.albumentations import TargetMode, _resolve_transform -from sampleflux.sample import Sample logger = get_logger(__name__) @@ -133,31 +134,27 @@ def __call__(self, sample: Sample) -> Sample: from torchvision import tv_tensors pipeline = self.pipeline - image = self._wrap_image(sample.input, tv_tensors, torch) + key, item = primary(sample, "input") + image = self._wrap_image(item_data(item), tv_tensors, torch) if self.target == "mask": - mask = self._wrap_mask(sample.target, tv_tensors, torch) + mask_field = next(iter(sample.items_of_type(Mask)), None) + if mask_field is None: + raise ValueError( + "TorchvisionTransformOp(target='mask'): no Mask field in the sample to transform jointly." + ) + mkey, mitem = mask_field + mask = self._wrap_mask(item_data(mitem), tv_tensors, torch) out_image, out_mask = pipeline(image, mask) - return sample._replace(input=self._unwrap(out_image, torch), target=self._unwrap(out_mask, torch)) + result = sample.replace_field(key, with_data(item, self._unwrap(out_image, torch).detach().cpu().numpy())) + return result.replace_field(mkey, with_data(mitem, self._unwrap(out_mask, torch).detach().cpu().numpy())) if self.target == "boxes": - target = sample.target - if not isinstance(target, dict) or "boxes" not in target or "labels" not in target: - raise TypeError( - f"TorchvisionTransformOp(target='boxes'): sample.target must be the torchvision " - f"detection dict {{'boxes': [N,4] xyxy, 'labels': [N]}}; got {type(target).__name__}. " - "Wire CocoToTorchVisionDetectionOp / MasksToDetectionBoxesOp upstream." - ) - boxes = tv_tensors.BoundingBoxes( - torch.as_tensor(np.asarray(target["boxes"], dtype=np.float32).reshape(-1, 4)), - format="XYXY", - canvas_size=self._canvas_size(image), + raise NotImplementedError( + "TorchvisionTransformOp(target='boxes') is not yet ported to the typed-bag Regions target " + "(migration follow-up); use target='none' or 'mask'." ) - labels = torch.as_tensor(np.asarray(target["labels"]).reshape(-1), dtype=torch.int64) - out_image, out_target = pipeline(image, {**target, "boxes": boxes, "labels": labels}) - out_target["boxes"] = self._unwrap(out_target["boxes"], torch).to(torch.float32) - out_target["labels"] = self._unwrap(out_target["labels"], torch) - return sample._replace(input=self._unwrap(out_image, torch), target=out_target) - return sample._replace(input=self._unwrap(pipeline(image), torch)) + out_image = self._unwrap(pipeline(image), torch) + return sample.replace_field(key, with_data(item, out_image.detach().cpu().numpy())) @staticmethod def _wrap_image(value: Any, tv_tensors: Any, torch: Any) -> Any: diff --git a/sampleflux/ops/transform_chain.py b/sampleflux/ops/transform_chain.py index e6dbc09..c3b328c 100644 --- a/sampleflux/ops/transform_chain.py +++ b/sampleflux/ops/transform_chain.py @@ -17,7 +17,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample logger = get_logger(__name__) diff --git a/sampleflux/projection.py b/sampleflux/projection.py index 41612e8..c4250ef 100644 --- a/sampleflux/projection.py +++ b/sampleflux/projection.py @@ -12,9 +12,8 @@ Design notes ------------ -* :class:`SupportsProjection` is a ``Protocol`` (never a base class), so it - composes with the SampleFlux **Functional Purity** mandate — a source opts in by - *defining* ``project``, not by inheriting. +* :class:`SupportsProjection` is a ``Protocol`` (never a base class), so a source + opts in by *defining* ``project``, not by inheriting. * Every public function is a lazy generator (**Lazy Evaluation** mandate) — nothing materializes the whole source. * :func:`num_classes` (integer class-id semantics) is a free function, *not* a @@ -23,73 +22,59 @@ make every ``Flux`` look classification-capable to duck-typed consumers. """ -from typing import Any, Collection, Iterator, Literal, Protocol, Tuple, get_args, runtime_checkable +from typing import Any, Collection, Dict, Iterator, Literal, Protocol, Tuple, get_args, runtime_checkable from sampleflux.bag.items import Label, item_data -from sampleflux.bag.sample import Role, TypedSample, primary -from sampleflux.sample import Sample - -#: The projectable :class:`~sampleflux.sample.Sample` fields, as a *closed* -#: ``Literal`` rather than a bare ``str``. Typing the field set this way lets -#: UIs, form-spec builders, and MCP tool schemas enumerate the allowed values -#: straight from the annotation (``typing.get_args(ProjectionField)``) and lets -#: a type checker reject a typo at the call site — the Literal-over-strings -#: discipline the workspace mandate calls for, applied because the set is fixed -#: and short. +from sampleflux.bag.sample import Role, Sample, primary + +#: The projectable :class:`~sampleflux.bag.sample.Sample` roles, as a *closed* +#: ``Literal`` rather than a bare ``str``. Typing the field set this way lets UIs, +#: form-spec builders, and MCP tool schemas enumerate the allowed values straight +#: from the annotation (``typing.get_args(ProjectionField)``) and lets a type +#: checker reject a typo at the call site. ``metadata`` maps onto the bag's ``aux`` role. ProjectionField = Literal["input", "target", "metadata"] INPUT: ProjectionField = "input" TARGET: ProjectionField = "target" METADATA: ProjectionField = "metadata" _FIELDS: Tuple[ProjectionField, ...] = get_args(ProjectionField) +_FIELD_ROLES: Dict[str, str] = {"input": "input", "target": "target", "metadata": "aux"} @runtime_checkable class SupportsProjection(Protocol): - """A source that can yield partial :class:`~sampleflux.sample.Sample` records. + """A source that can yield partial :class:`~sampleflux.bag.sample.Sample` records. Implementers SHOULD avoid building unrequested fields — e.g. skip decoding the input image when only ``target`` is asked for; that efficiency is the whole - point of the protocol. ``fields`` is a subset of - ``{"input", "target", "metadata"}``; unrequested fields come back as ``None`` - (``{}`` for ``metadata``). + point of the protocol. ``fields`` is a subset of ``{"input", "target", "metadata"}``. """ def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: ... -def _carrier_field(carrier: Any, field: ProjectionField) -> Any: - """Read one field's VALUE from either a legacy :class:`Sample` or a typed :class:`TypedSample`. +def _carrier_field(sample: Sample, field: ProjectionField) -> Any: + """Read one field's VALUE from a typed :class:`Sample`. - For a ``TypedSample`` (the typed-bag carrier the migrated sources yield) the value of the - ``input`` / ``target`` role is the FIRST field of that role — a ``Label``'s ``.value`` (the class - id / scalar), else the item's raw payload (:func:`item_data`). A missing role yields ``None``, so - a target-only walk over a typed source feeds :func:`num_classes` exactly as the legacy carrier did. + The value of the ``input`` / ``target`` role is the FIRST field of that role — a + ``Label``'s ``.value`` (the class id / scalar), else the item's raw payload + (:func:`item_data`). A missing role yields ``None``, so a target-only walk feeds + :func:`num_classes`. """ - if isinstance(carrier, TypedSample): - role: Role = "input" if field == INPUT else "target" - try: - _key, item = primary(carrier, role) - except KeyError: - return None - return item.value if isinstance(item, Label) else item_data(item) - s = Sample.from_any(carrier) - if field == INPUT: - return s.input - if field == TARGET: - return s.target - return s.meta - - -def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Any]: + role: Role = "input" if field == INPUT else "target" + try: + _key, item = primary(sample, role) + except KeyError: + return None + return item.value if isinstance(item, Label) else item_data(item) + + +def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample]: """Yield partial records from ``source`` carrying only ``fields``. - Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the efficient - path that skips building unrequested fields); otherwise falls back to a full iteration that builds - every field and nulls the unrequested ones — always correct, just not faster. A typed-bag - :class:`TypedSample` is passed through VERBATIM (never coerced into a legacy ``Sample``); the walk - helpers below extract the requested field from whichever carrier flows. Lazy: a generator that - never materializes the source. + Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the + efficient path that skips building unrequested fields); otherwise falls back to a full + iteration that keeps only the fields whose role matches the request. Lazy: a generator. """ want = frozenset(fields) unknown = want - frozenset(_FIELDS) @@ -98,16 +83,10 @@ def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Any]: if isinstance(source, SupportsProjection): yield from source.project(want) return - for raw in source: - if isinstance(raw, TypedSample): - yield raw - continue - s = Sample.from_any(raw) - yield Sample( - input=s.input if INPUT in want else None, - target=s.target if TARGET in want else None, - metadata=s.meta if METADATA in want else {}, - ) + want_roles = {_FIELD_ROLES[f] for f in want} + for sample in source: + keep = [k for k in sample.keys() if sample.role_of(k) in want_roles] + yield Sample({k: sample[k] for k in keep}, {k: sample.role_of(k) for k in keep}) def iter_inputs(source: Any) -> Iterator[Any]: diff --git a/sampleflux/sample.py b/sampleflux/sample.py deleted file mode 100644 index 1d00447..0000000 --- a/sampleflux/sample.py +++ /dev/null @@ -1,175 +0,0 @@ -import json -from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Tuple, Union, cast - -if TYPE_CHECKING: # pragma: no cover - typing only - from sampleflux.typespec import SampleType - -# Reserved metadata keys carrying a sample's stored type description (JSON strings so they survive -# every storage backend's metadata round-trip — HDF5 attrs / Zarr attrs / Directory YAML / HF / Confluid). -# ``__features__`` holds a ``datasets.Features`` dict (the standard, concrete structural description); -# ``__spec__`` holds the sidecar refinements Features can't express (framework / ranges / Any / Union). -FEATURES_KEY = "__features__" -SPEC_KEY = "__spec__" -TYPE_KEYS = (FEATURES_KEY, SPEC_KEY) - -# A Sample's metadata is EITHER a single ``dict`` (one item — the normal pipeline form every op -# produces/consumes) OR a ``list`` of per-item dicts (a BATCH — produced by the collate functions when -# stacking N samples into one). The two forms are how a Sample distinguishes a single item from a batch: -# per-sample ops always see (and require) the dict form; the list form appears only AFTER collate, in the -# batched Sample fed to the model / loss / predictions sinks, and never flows back through a per-sample op. -Metadata = Union[Dict[str, Any], List[Dict[str, Any]]] - - -# The named field VIEWS of a Sample — the closed vocabulary of what a transform can -# process (the taxonomy `sampleflux.kinds` introspects and a visual editor can surface as -# socket types). A view is a real runtime NamedTuple, so an op annotated with one -# receives an object with named fields; the engine binds the view from the flowing -# carrier and merges the result back (untouched fields preserved). -class Pair(NamedTuple): - """The classic metadata-free AI pair ``(input, target)`` — a named 2-tuple view.""" - - input: Any - target: Any = None - - -class InputMeta(NamedTuple): - """The ``(input, metadata)`` view — a transform that reads/writes the input WITH its metadata. - - ``metadata`` follows the same single-vs-batch duality as ``Sample.metadata``: one dict - per item, a list of dicts after collation. - """ - - input: Any - metadata: Metadata = {} - - -class TargetMeta(NamedTuple): - """The ``(target, metadata)`` view — a transform that reads/writes the target WITH its metadata. - - ``metadata`` follows the same single-vs-batch duality as ``Sample.metadata``. - """ - - target: Any - metadata: Metadata = {} - - -# Standardized Sample: (input, target, metadata) -# This allows SampleFlux to handle complex pipelines while remaining -# compatible with simple PyTorch/HF (input, target) pairs. -class Sample(NamedTuple): - input: Any - target: Any = None - metadata: Metadata = {} - - def to_tuple(self) -> Tuple[Any, Any, Metadata]: - return (self.input, self.target, self.metadata) - - def to_pair(self) -> Tuple[Any, Any]: - """The metadata-free ``(input, target)`` view (the native-engine pair carrier).""" - return (self.input, self.target) - - def input_meta(self) -> "InputMeta": - """The ``(input, metadata)`` view — the SAME metadata dict (mutation propagates).""" - return InputMeta(self.input, self.meta) - - def target_meta(self) -> "TargetMeta": - """The ``(target, metadata)`` view — the SAME metadata dict (mutation propagates).""" - return TargetMeta(self.target, self.meta) - - @property - def is_batched(self) -> bool: - """True if this Sample holds a BATCH — ``metadata`` is a ``list`` of per-item dicts (one per - stacked item, as the collate functions produce); False for a single item (``metadata`` is a - ``dict``). The single source of truth for telling batch from single.""" - return isinstance(self.metadata, list) - - @property - def meta(self) -> Dict[str, Any]: - """The single-item metadata **dict** — the narrowing accessor per-sample ops/sources use to - read or mutate ``metadata`` (``sample.meta[key]`` / ``sample.meta[key] = v``). Returns the same - underlying dict (mutation propagates). Raises ``TypeError`` on a batched Sample, where there is - no single dict — use :attr:`batch_meta` instead.""" - if isinstance(self.metadata, list): - raise TypeError( - "Sample.meta is the single-item metadata dict, but this Sample is batched (metadata is a " - "list of per-item dicts) — use Sample.batch_meta." - ) - return self.metadata - - @property - def batch_meta(self) -> List[Dict[str, Any]]: - """The per-item metadata **list** of a batched Sample (one dict per stacked item) — the - narrowing accessor batch consumers (collate-fed losses / predictions sinks) use. Raises - ``TypeError`` on a single Sample, whose metadata is one dict — use :attr:`meta` instead.""" - if not isinstance(self.metadata, list): - raise TypeError( - "Sample.batch_meta is the per-item metadata list of a batch, but this Sample is single " - "(metadata is one dict) — use Sample.meta." - ) - return self.metadata - - def describe(self) -> "SampleType": - """Return this sample's :class:`~sampleflux.typespec.SampleType`. - - Prefers the stored type (the reserved metadata keys, set explicitly via :meth:`with_type` or - carried by a serialized dataset); otherwise infers it from the live ``input`` / ``target``. A - batched sample carries no per-reserved-key type, so it always infers from the live data. - """ - from sampleflux.typespec import SampleType, infer_sample_type - - meta = self.metadata - if isinstance(meta, dict): - raw_features = meta.get(FEATURES_KEY) - raw_extras = meta.get(SPEC_KEY) - if raw_features is not None or raw_extras is not None: - features = json.loads(raw_features) if isinstance(raw_features, str) else (raw_features or {}) - extras = json.loads(raw_extras) if isinstance(raw_extras, str) else raw_extras - return SampleType.from_hf_features(features, extras) - return infer_sample_type(self) - - def with_type(self, sample_type: "SampleType") -> "Sample": - """Return a copy carrying ``sample_type`` in the reserved metadata keys (copy-on-write, so the - original sample's metadata is not mutated). Only defined for a single (non-batched) sample — - a batch carries no single stored type.""" - if self.is_batched: - raise TypeError( - "Sample.with_type is only defined for a single (non-batched) sample; this Sample carries " - "list (batched) metadata." - ) - base = cast(Dict[str, Any], self.metadata) - features, extras = sample_type.to_hf_features() - metadata = {**base, FEATURES_KEY: json.dumps(features.to_dict()), SPEC_KEY: json.dumps(extras)} - return self._replace(metadata=metadata) - - @classmethod - def from_any(cls, obj: Any) -> "Sample": - """Coerce raw data from various sources into a Sample. - - The named field VIEWS are recognised BEFORE the generic tuple rule — an - ``InputMeta``/``TargetMeta``/``Pair`` IS a tuple, and positional coercion would - silently misread ``(input, metadata)`` as ``(input, target)``. - """ - if isinstance(obj, cls): - return obj - if isinstance(obj, InputMeta): - return cls(obj.input, None, obj.metadata) - if isinstance(obj, TargetMeta): - return cls(None, obj.target, obj.metadata) - if isinstance(obj, Pair): - return cls(obj.input, obj.target, {}) - if isinstance(obj, tuple): - if len(obj) >= 3: - return cls(obj[0], obj[1], obj[2] or {}) - if len(obj) == 2: - return cls(obj[0], obj[1], {}) - if len(obj) == 1: - return cls(obj[0], None, {}) - # Empty tuple - return cls(None, None, {}) - if isinstance(obj, dict): - return cls( - input=obj.get("input"), - target=obj.get("target"), - metadata=obj.get("metadata", {}), - ) - return cls(obj, None, {}) diff --git a/sampleflux/sources.py b/sampleflux/sources.py index edb6460..d281952 100644 --- a/sampleflux/sources.py +++ b/sampleflux/sources.py @@ -5,24 +5,20 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag import Image, Label, TypedSample +from sampleflux.bag import Image, Label, Sample from sampleflux.projection import ProjectionField -from sampleflux.sample import Sample logger = get_logger(__name__) def _pass_through(item: Any) -> Any: - """Coerce a wrapped source's item to a carrier the engine accepts. + """Pass a wrapped source's item through verbatim. - A typed-bag :class:`~sampleflux.TypedSample` is passed through VERBATIM — the view sources - (:class:`DatasetSplit` / :class:`RangeSource` / :class:`ConcatSource`) only slice/index, they - never inspect payloads, so a typed source flows through them unchanged. Any legacy carrier is - normalized to a :class:`~sampleflux.sample.Sample` via ``Sample.from_any``. + Every carrier is a typed-bag :class:`~sampleflux.Sample`; the view sources + (:class:`DatasetSplit` / :class:`RangeSource` / :class:`ConcatSource`) only slice/index, + they never inspect payloads, so a source's samples flow through them unchanged. """ - if isinstance(item, TypedSample): - return item - return Sample.from_any(item) + return item # Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer @@ -68,7 +64,7 @@ def _resolve_metadata_features( @configurable(category="source") class HuggingFaceSource: """ - SampleFlux Source for Hugging Face Datasets, yielding typed-bag :class:`~sampleflux.TypedSample`\\ s. + SampleFlux Source for Hugging Face Datasets, yielding typed-bag :class:`~sampleflux.Sample`\\ s. Field mapping (the typed-bag layout that replaces the ``Sample(input, target, metadata)`` triple): @@ -162,8 +158,8 @@ def _to_typed_sample( want_input: bool = True, want_target: bool = True, want_meta: bool = True, - ) -> TypedSample: - """Assemble one :class:`~sampleflux.TypedSample` from a raw HF row dict (see the class docstring + ) -> Sample: + """Assemble one :class:`~sampleflux.Sample` from a raw HF row dict (see the class docstring for the field mapping). ``want_input`` / ``want_target`` / ``want_meta`` gate which roles are built — the projection @@ -189,9 +185,9 @@ def _to_typed_sample( fields["hf_split"] = Label(self.split) roles["hf_path"] = "aux" roles["hf_split"] = "aux" - return TypedSample(fields, roles) + return Sample(fields, roles) - def __iter__(self) -> Iterator[TypedSample]: + def __iter__(self) -> Iterator[Sample]: dataset = self.dataset metadata_features = self.resolved_metadata_features limit = self.count or len(dataset) @@ -201,11 +197,11 @@ def __iter__(self) -> Iterator[TypedSample]: break yield self._to_typed_sample(item, metadata_features) - def __getitem__(self, index: int) -> TypedSample: + def __getitem__(self, index: int) -> Sample: return self._to_typed_sample(self.dataset[index], self.resolved_metadata_features) - def project(self, fields: Collection[ProjectionField]) -> Iterator[TypedSample]: - """Yield role-restricted ``TypedSample``\\ s — the ``SupportsProjection`` efficient path. + def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: + """Yield role-restricted ``Sample``\\ s — the ``SupportsProjection`` efficient path. Only the requested roles are built, so a target-only walk (e.g. :func:`~sampleflux.num_classes`) skips decoding the image entirely: ``"input"`` -> the ``"image"`` field, ``"target"`` -> the diff --git a/sampleflux/storage/base.py b/sampleflux/storage/base.py index 677b5f2..974bd8a 100644 --- a/sampleflux/storage/base.py +++ b/sampleflux/storage/base.py @@ -4,8 +4,7 @@ import numpy as np import torch -from sampleflux.bag.sample import TypedSample -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample #: Root-attribute format tag stamped on stores written in the typed field-group layout. TYPED_FORMAT = "typedsample-v1" @@ -55,7 +54,7 @@ def flush(self) -> None: class TypedDataSource(Protocol): """Minimum contract for a typed-bag data source.""" - def __iter__(self) -> Iterator[TypedSample]: + def __iter__(self) -> Iterator[Sample]: """Iterate over typed samples in the source.""" ... @@ -68,7 +67,7 @@ def __len__(self) -> int: class TypedDataSink(Protocol): """Minimum contract for a typed-bag data sink.""" - def write(self, sample: TypedSample) -> None: + def write(self, sample: Sample) -> None: """Write a single typed sample to the sink.""" ... diff --git a/sampleflux/storage/directory.py b/sampleflux/storage/directory.py index d395a49..352226d 100644 --- a/sampleflux/storage/directory.py +++ b/sampleflux/storage/directory.py @@ -6,7 +6,7 @@ import numpy as np from sampleflux.bag.io import EncodedItem, decode_item, encode_item -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.storage.base import DataSink, Storage, restore_attrs, split_attrs, to_numpy #: Typed-layout filenames inside each per-sample directory. @@ -38,37 +38,12 @@ def open(self) -> "DirectorySink": def write(self, sample: Any) -> None: """Write a sample to its own subdirectory.""" - if isinstance(sample, TypedSample): - self.open() - self._write_typed(sample) - return + if not isinstance(sample, Sample): + raise TypeError(f"DirectorySink: expected a Sample bag, got {type(sample).__name__}") + self.open() + self._write_typed(sample) - # Use a zero-padded index for sorting - sample_dir = self.path / f"{self._counter:06d}" - sample_dir.mkdir(parents=True, exist_ok=True) - - # 1. Save Metadata (YAML via Confluid) - if sample.meta: - meta_path = sample_dir / "metadata.yaml" - meta_path.write_text(confluid.dump(sample.meta)) - - # 2. Save Input and Target (Numpy) - if self.use_npz: - # Combined file - np.savez( - sample_dir / "sample.npz", - data=sample.input, - target=sample.target if sample.target is not None else np.array([]), - ) - else: - # Separate files - np.save(sample_dir / "data.npy", sample.input) - if sample.target is not None: - np.save(sample_dir / "target.npy", sample.target) - - self._counter += 1 - - def _write_typed(self, sample: TypedSample) -> None: + def _write_typed(self, sample: Sample) -> None: """One sample in the typed field-group layout: ``fields.json`` + ``fields.npz``. ``fields.json`` describes every field (order, item type, role, plain attrs); @@ -129,7 +104,7 @@ def _sample_dirs(self) -> list: raise FileNotFoundError(f"DirectorySource: {self.path} does not exist") return sorted(p for p in self.path.iterdir() if p.is_dir() and (p / _FIELDS_JSON).exists()) - def __iter__(self) -> Iterator[TypedSample]: + def __iter__(self) -> Iterator[Sample]: for sample_dir in self._sample_dirs(): yield self._read(sample_dir) @@ -137,7 +112,7 @@ def __len__(self) -> int: return len(self._sample_dirs()) @staticmethod - def _read(sample_dir: Path) -> TypedSample: + def _read(sample_dir: Path) -> Sample: spec = json.loads((sample_dir / _FIELDS_JSON).read_text()) npz_path = sample_dir / _FIELDS_NPZ payloads = dict(np.load(npz_path, allow_pickle=False)) if npz_path.exists() else {} @@ -150,4 +125,4 @@ def _read(sample_dir: Path) -> TypedSample: payload = payloads[key] if entry["has_payload"] else None fields[key] = decode_item(EncodedItem(type_name=entry["type"], payload=payload, attrs=attrs)) roles[key] = entry["role"] - return TypedSample(fields, roles) + return Sample(fields, roles) diff --git a/sampleflux/storage/hdf5.py b/sampleflux/storage/hdf5.py index 84d59aa..85e4805 100644 --- a/sampleflux/storage/hdf5.py +++ b/sampleflux/storage/hdf5.py @@ -4,13 +4,11 @@ import h5py import numpy as np -import torch from confluid import configurable from loggair import get_logger from sampleflux.bag.io import EncodedItem, decode_item, encode_item -from sampleflux.bag.sample import TypedSample -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample from sampleflux.storage.base import TYPED_FORMAT, DataSink, DataSource, Storage, restore_attrs, split_attrs, to_numpy logger = get_logger("sampleflux.storage.hdf5") @@ -21,7 +19,7 @@ _ORDER_ATTR = "__field_order__" -def _read_typed_sample(group: h5py.Group) -> TypedSample: +def _read_typed_sample(group: h5py.Group) -> Sample: """Decode one ``sNNNNNN`` sample group of the typed field-group layout.""" order = json.loads(group.attrs[_ORDER_ATTR]) fields: Dict[str, Any] = {} @@ -38,7 +36,7 @@ def _read_typed_sample(group: h5py.Group) -> TypedSample: attrs = restore_attrs(dict(plain), arrays) fields[name] = decode_item(EncodedItem(type_name=str(fgrp.attrs[_TYPE_ATTR]), payload=payload, attrs=attrs)) roles[name] = str(fgrp.attrs[_ROLE_ATTR]) - return TypedSample(fields, roles) + return Sample(fields, roles) @configurable @@ -78,34 +76,14 @@ def __iter__(self) -> Iterator[Any]: self.open() if self._file is None: return - - if self.is_typed: - for name in sorted(k for k in self._file.keys() if k.startswith("s")): - yield _read_typed_sample(self._file[name]) - return - - prefixes = sorted([k.split("_data")[0] for k in self._file.keys() if k.endswith("_data")]) - - for pref in prefixes: - data = self._file[f"{pref}_data"][()] - target = self._file[f"{pref}_target"][()] if f"{pref}_target" in self._file else None - metadata = dict(self._file[f"{pref}_data"].attrs) - # Merge array-valued metadata written as datasets under the per-sample meta group - # (see HDF5Sink.write). Absent on files written before this layout — old files read unchanged. - meta_grp = self._file.get(f"{pref}_meta") - if isinstance(meta_grp, h5py.Group): - for key, dset in meta_grp.items(): - metadata[key] = dset[()] - # Source returns Tensors to match schema - yield Sample(input=torch.from_numpy(data), target=target, metadata=metadata) + for name in sorted(k for k in self._file.keys() if k.startswith("s")): + yield _read_typed_sample(self._file[name]) def __len__(self) -> int: self.open() if self._file is None: return 0 - if self.is_typed: - return len([k for k in self._file.keys() if k.startswith("s")]) - return len([k for k in self._file.keys() if k.endswith("_data")]) + return len([k for k in self._file.keys() if k.startswith("s")]) def iter_metadata(self) -> "Iterator[tuple[str, dict]]": """(prefix, metadata) per sample WITHOUT loading data arrays (SupportsMetadataScan). @@ -153,59 +131,11 @@ def write(self, sample: Any) -> None: self.open() if self._file is None: return + if not isinstance(sample, Sample): + raise TypeError(f"HDF5Sink: expected a Sample bag, got {type(sample).__name__}") + self._write_typed(sample) - if isinstance(sample, TypedSample): - self._write_typed(sample) - return - if self._file.attrs.get("sampleflux_format") == TYPED_FORMAT: - raise TypeError( - "HDF5Sink: this file carries the typed field-group layout — cannot append a legacy " - "Sample to it (one carrier per file)." - ) - - prefix = f"{self._counter:05d}" - - # Convert tensors to numpy for h5py - input_data = to_numpy(sample.input) - target_data = to_numpy(sample.target) - - # 1. Write Data - kwargs = {} - if self.compression and hasattr(input_data, "shape") and len(input_data.shape) > 0: - kwargs["compression"] = self.compression - - ds = self._file.create_dataset(f"{prefix}_data", data=input_data, **kwargs) - - # 2. Write Metadata. Scalars/strings go on the data dataset's HDF5 attributes (compact, - # round-trips for the common case). Array-valued metadata (e.g. a segmentation mask) CANNOT - # be stored as an attribute — HDF5 caps attribute size ("object header message is too large") - # and the str() fallback would silently truncate the array — so it is written as its own - # dataset under a per-sample group ``{prefix}_meta/`` (the "/" makes h5py auto-create the - # group; arbitrary metadata keys are safe as dataset names). HDF5Source merges both back. - for k, v in sample.meta.items(): - if isinstance(v, (np.ndarray, torch.Tensor)): - arr = to_numpy(v) - m_kwargs = {} - if self.compression and getattr(arr, "ndim", 0) > 0: - m_kwargs["compression"] = self.compression - self._file.create_dataset(f"{prefix}_meta/{k}", data=arr, **m_kwargs) - else: - try: - ds.attrs[k] = v - except Exception: - ds.attrs[k] = str(v) - - # 3. Write Target - if target_data is not None: - t_kwargs = {} - if self.compression and hasattr(target_data, "shape") and len(target_data.shape) > 0: - t_kwargs["compression"] = self.compression - - self._file.create_dataset(f"{prefix}_target", data=target_data, **t_kwargs) - - self._counter += 1 - - def _write_typed(self, sample: TypedSample) -> None: + def _write_typed(self, sample: Sample) -> None: """One sample in the typed field-group layout — see ``docs/typed-model.md`` (storage). Layout: root attr ``sampleflux_format = "typedsample-v1"``; per sample a group @@ -220,13 +150,13 @@ def _write_typed(self, sample: TypedSample) -> None: if any(k.endswith("_data") for k in self._file.keys()): raise TypeError( "HDF5Sink: this file carries the legacy Sample layout — cannot append a " - "TypedSample to it (one carrier per file)." + "Sample to it (one carrier per file)." ) self._file.attrs["sampleflux_format"] = TYPED_FORMAT elif self._file.attrs.get("sampleflux_format") != TYPED_FORMAT: raise TypeError( "HDF5Sink: this file carries the legacy Sample layout — cannot append a " - "TypedSample to it (one carrier per file)." + "Sample to it (one carrier per file)." ) group = self._file.create_group(f"s{self._counter:06d}") diff --git a/sampleflux/storage/query.py b/sampleflux/storage/query.py index 3af333f..2eab8bf 100644 --- a/sampleflux/storage/query.py +++ b/sampleflux/storage/query.py @@ -28,9 +28,8 @@ from loggair import get_logger from sampleflux.bag.io import encode_item -from sampleflux.bag.sample import TypedSample +from sampleflux.bag.sample import Sample from sampleflux.ops.formula import _FORMULA_NAMESPACE -from sampleflux.sample import Sample from sampleflux.storage.base import TYPED_FORMAT, restore_attrs logger = get_logger("sampleflux.storage.query") @@ -61,7 +60,7 @@ def _viewed(metadata: Dict[str, Any]) -> Dict[str, Any]: return {k: _AttrView(v) if isinstance(v, dict) else v for k, v in metadata.items()} -def typed_sample_metadata(sample: TypedSample) -> Dict[str, Dict[str, Any]]: +def typed_sample_metadata(sample: Sample) -> Dict[str, Dict[str, Any]]: """A live sample's queryable metadata: ``{field: {attr: value}}`` (attrs via the io codec, payloads untouched) — the same nested shape the typed storage scans yield.""" return {key: dict(encode_item(item).attrs) for key, item in sample.items()} @@ -219,11 +218,7 @@ def matches(self) -> List[int]: "falling back to full-iteration filtering (arrays load for every sample)." ) self._matches = [ - i - for i, sample in enumerate(self.source) - if self._match( - typed_sample_metadata(sample) if isinstance(sample, TypedSample) else dict(sample.meta) - ) + i for i, sample in enumerate(self.source) if self._match(typed_sample_metadata(sample)) ] return self._matches diff --git a/sampleflux/storage/zarr.py b/sampleflux/storage/zarr.py index d7151cf..b596b13 100644 --- a/sampleflux/storage/zarr.py +++ b/sampleflux/storage/zarr.py @@ -4,12 +4,10 @@ import confluid import numpy as np -import torch import zarr from sampleflux.bag.io import EncodedItem, decode_item, encode_item -from sampleflux.bag.sample import TypedSample -from sampleflux.sample import Sample +from sampleflux.bag.sample import Sample, primary from sampleflux.storage.base import TYPED_FORMAT, DataSink, DataSource, Storage, restore_attrs, split_attrs, to_numpy #: Reserved field-group attr names in the typed layout (never item attrs). @@ -18,7 +16,7 @@ _ORDER_ATTR = "__field_order__" -def _read_typed_group(grp: "zarr.Group") -> TypedSample: +def _read_typed_group(grp: "zarr.Group") -> Sample: """Decode one ``sample_NNNNNN`` group of the typed field-group layout.""" order = json.loads(str(grp.attrs[_ORDER_ATTR])) fields: Dict[str, Any] = {} @@ -36,7 +34,7 @@ def _read_typed_group(grp: "zarr.Group") -> TypedSample: attrs = restore_attrs(plain, arrays) fields[name] = decode_item(EncodedItem(type_name=str(fattrs[_TYPE_ATTR]), payload=payload, attrs=attrs)) roles[name] = str(fattrs[_ROLE_ATTR]) - return TypedSample(fields, roles) + return Sample(fields, roles) # category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @@ -66,35 +64,11 @@ def write(self, sample: Any) -> None: self.open() if self._root is None: raise RuntimeError("Zarr group not open") + if not isinstance(sample, Sample): + raise TypeError(f"ZarrGroupSink: expected a Sample bag, got {type(sample).__name__}") + self._write_typed(sample) - if isinstance(sample, TypedSample): - self._write_typed(sample) - return - if self._root.attrs.get("sampleflux_format") == TYPED_FORMAT: - raise TypeError( - "ZarrGroupSink: this store carries the typed field-group layout — cannot append a " - "legacy Sample to it (one carrier per store)." - ) - - # Use require_group to handle existing nodes safely - name = f"sample_{self._counter:06d}" - grp = self._root.require_group(name) - - # 1. Save data and target. create_array needs a numpy array (it can't read a - # torch tensor's dtype); to_numpy detaches/moves to CPU. overwrite=True replaces - # an existing node, so no manual delete is needed on re-write. - grp.create_array("data", data=to_numpy(sample.input), overwrite=True) - - if sample.target is not None: - grp.create_array("target", data=to_numpy(sample.target), overwrite=True) - - # 2. Save metadata as Zarr attributes (.zattrs) - if sample.meta: - grp.attrs.update(sample.meta) - - self._counter += 1 - - def _write_typed(self, sample: TypedSample) -> None: + def _write_typed(self, sample: Sample) -> None: """One sample in the typed field-group layout (the Zarr twin of HDF5Sink._write_typed).""" assert self._root is not None existing_format = self._root.attrs.get("sampleflux_format") @@ -102,7 +76,7 @@ def _write_typed(self, sample: TypedSample) -> None: if any(True for _ in self._root.group_keys()) and self._counter == 0: raise TypeError( "ZarrGroupSink: this store carries the legacy Sample layout — cannot append a " - "TypedSample to it (one carrier per store)." + "Sample to it (one carrier per store)." ) self._root.attrs["sampleflux_format"] = TYPED_FORMAT elif existing_format != TYPED_FORMAT: @@ -172,15 +146,8 @@ def __iter__(self) -> Iterator[Any]: self.open() if self._root is None: return - if self.is_typed: - for name in sorted(self._root.group_keys()): - yield _read_typed_group(cast(zarr.Group, self._root[name])) - return for name in sorted(self._root.group_keys()): - grp = cast(zarr.Group, self._root[name]) - data = cast(zarr.Array, grp[self.sample_key])[:] - target = cast(zarr.Array, grp[self.target_key])[:] if self.target_key in grp else None - yield Sample(input=torch.from_numpy(np.asarray(data)), target=target, metadata=dict(grp.attrs)) + yield _read_typed_group(cast(zarr.Group, self._root[name])) def __len__(self) -> int: self.open() @@ -237,42 +204,26 @@ def write(self, sample: Any) -> None: self.open() if self._data_arr is None: raise RuntimeError("Zarr array not open") - - if isinstance(sample, TypedSample): - # The batch sink stores ONE uniform array: the PRIMARY input field's payload per - # row, plus a one-time item template (type/field/attrs of the FIRST sample) so the - # source can rebuild typed rows. Uniform-batch by design — per-sample attr - # variation does not fit a single stacked array; use ZarrGroupSink for that. - from sampleflux.bag.sample import primary - - key, item = primary(sample) - encoded = encode_item(item) - if "sampleflux_format" not in self._data_arr.attrs: - plain, arrays = split_attrs(encoded.attrs) - if arrays: - raise TypeError( - "ZarrBatchSink: array-valued item attrs do not fit the single-array batch " - "layout — use ZarrGroupSink." - ) - self._data_arr.attrs.update( - {"sampleflux_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} + if not isinstance(sample, Sample): + raise TypeError(f"ZarrBatchSink: expected a Sample bag, got {type(sample).__name__}") + + # The batch sink stores ONE uniform array: the PRIMARY input field's payload per row, + # plus a one-time item template (type/field/attrs of the FIRST sample) so the source can + # rebuild typed rows. Uniform-batch by design — per-sample attr variation does not fit a + # single stacked array; use ZarrGroupSink for that. + key, item = primary(sample) + encoded = encode_item(item) + if "sampleflux_format" not in self._data_arr.attrs: + plain, arrays = split_attrs(encoded.attrs) + if arrays: + raise TypeError( + "ZarrBatchSink: array-valued item attrs do not fit the single-array batch " + "layout — use ZarrGroupSink." ) - self._data_arr.append([np.asarray(to_numpy(encoded.payload))], axis=0) - self._counter += 1 - return - if self._data_arr.attrs.get("sampleflux_format") == TYPED_FORMAT: - raise TypeError( - "ZarrBatchSink: this store carries the typed layout — cannot append a legacy " - "Sample to it (one carrier per store)." + self._data_arr.attrs.update( + {"sampleflux_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} ) - - # Append to the primary array - # Zarr handles the resizing and chunking internally - self._data_arr.append([sample.input], axis=0) - - # Note: Handling metadata in a single-array sink requires - # a separate attribute list or sidecar file. - # For simplicity, we attach to the array attributes. + self._data_arr.append([np.asarray(to_numpy(encoded.payload))], axis=0) self._counter += 1 def flush(self) -> None: @@ -310,21 +261,17 @@ def __iter__(self) -> Iterator[Any]: if self._data_arr is None: return attrs = dict(self._data_arr.attrs) - if attrs.get("sampleflux_format") == TYPED_FORMAT: - # Typed batch rows: rebuild each row as the stored item type under the stored - # field key (uniform template — see ZarrBatchSink.write). - field = str(attrs["__field__"]) - type_name = str(attrs[_TYPE_ATTR]) - item_attrs = restore_attrs( - {k: v for k, v in attrs.items() if k not in ("sampleflux_format", _TYPE_ATTR, "__field__")}, {} - ) - for i in range(self._data_arr.shape[0]): - payload = np.asarray(self._data_arr[i]) - item = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=item_attrs)) - yield TypedSample({field: item}) - return + # Typed batch rows: rebuild each row as the stored item type under the stored field key + # (uniform template — see ZarrBatchSink.write). + field = str(attrs["__field__"]) + type_name = str(attrs[_TYPE_ATTR]) + item_attrs = restore_attrs( + {k: v for k, v in attrs.items() if k not in ("sampleflux_format", _TYPE_ATTR, "__field__")}, {} + ) for i in range(self._data_arr.shape[0]): - yield Sample(input=torch.from_numpy(np.asarray(self._data_arr[i]))) + payload = np.asarray(self._data_arr[i]) + item = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=item_attrs)) + yield Sample({field: item}) def __len__(self) -> int: self.open() diff --git a/sampleflux/typespec.py b/sampleflux/typespec.py deleted file mode 100644 index b4cdd4d..0000000 --- a/sampleflux/typespec.py +++ /dev/null @@ -1,937 +0,0 @@ -"""Type-spec system: describe and match the types flowing through a :class:`~sampleflux.sample.Sample`. - -Two gaps this fills: - -1. A ``Sample`` carries no description of *what kind of data* sits in its ``input`` / ``target``. -2. Ops and sources don't declare which input/target types they accept or produce. - -The model is a small set of frozen value objects (no base class — see SampleFlux "Functional Purity" -mandate; these are values, not data ops) describing one slot: - -* :class:`AnyType` — matches everything (the default when nothing is declared). -* :class:`ArrayType` — an N-D array/tensor across frameworks (numpy / torch / tensorflow): optional - rank, per-axis :class:`Dim` constraints (exact / bounded-range / unbounded), optional dtype - (concrete ``"float32"`` or a *family* ``"floating"`` / ``"integer"`` / ``"numeric"`` …), optional - framework set, optional ``semantic`` tag. ``ArrayType.image(...)`` is a convenience for images. -* :class:`PythonType` — a non-array Python value by qualname (``"PIL.Image.Image"``, ``"dict"`` …). -* :class:`UnionType` — any-of. -* :class:`MappingType` / :class:`ListType` — mirror ``datasets`` struct / ``Sequence`` so structured - targets (e.g. detection ``{boxes, labels}``) get a real type and bridge 1:1 to HF ``Features``. -* :class:`SampleType` — the ``(input, target)`` pair an op/source declares or a sample reports. - -**Matching** is asymmetric (covariant): ``consumer.accepts(producer)`` is True iff every concrete -value the producer can emit is acceptable to the consumer. Two flavours share the leaf logic: - -* ``accepts`` — *strict*; used by the runtime check where the producer is a concrete inferred type. -* ``compatible`` — *permissive*; used at edit-time (a visual canvas) and for discovery filtering: - ``Any``/unknown on **either** side ⇒ compatible (honours "if not defined, assume Any"), and an - unbounded producer axis against a bounded consumer axis is a soft-pass (the runtime check still - catches an actual out-of-range value). - -Prior art borrowed rather than reinvented: the ``None``-dim base case mirrors ``tf.TensorShape`` -(``Dim`` adds the bounded-range extension tf/HF/jaxtyping lack); dtype family names align with -``numpy.isdtype`` / the array-api taxonomy; ``ArrayType.parse`` accepts a jaxtyping-style shape -string; and :meth:`SampleType.from_hf_features` / :meth:`SampleType.to_hf_features` bridge to the -``datasets.Features`` we already depend on (used for the concrete per-sample stored type). - -Everything is JSON round-trippable (``to_dict`` / :func:`type_from_dict` / :func:`sampletype_from_dict`) -so specs ride the discovery manifest and can be re-implemented by a GUI connection-validator. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import ( - TYPE_CHECKING, - AbstractSet, - Any, - Callable, - Dict, - FrozenSet, - List, - Literal, - Optional, - Tuple, - TypeVar, - Union, - cast, -) - -import numpy as np - -if TYPE_CHECKING: # pragma: no cover - typing only - from sampleflux.sample import Sample - -# A single-slot type spec. Defined as a forward-ref union so leaf classes can annotate it before the -# alias is bound at runtime (annotations are strings under ``from __future__ import annotations``). -TypeSpec = Union["AnyType", "ArrayType", "PythonType", "UnionType", "MappingType", "ListType"] - -# Closed enumerations for the small, fixed string sets the type system uses — declared as ``Literal`` -# rather than bare ``str`` so authors get a typo-checked value and UIs / a GUI connection- -# validator enumerate the choices straight from the annotation (``typing.get_args(...)``); the -# workspace "prefer closed ``Literal``s over bare strings" mandate. Both are *deliberately closed* — -# extend the Literal when adding real support (e.g. a ``"jax"`` framework), don't widen to ``str``. -# (The dtype enumerations — ``Dtype`` / ``DtypeFamily`` / ``DtypeSpec`` — are defined next to -# ``_DTYPE_FAMILIES`` below, since they share that block's set membership as their source of truth.) -Framework = Literal["numpy", "torch", "tensorflow"] -ImageLayout = Literal["CHW", "HWC"] - -C = TypeVar("C") - -__all__ = [ - "Dim", - "AnyType", - "ArrayType", - "PythonType", - "UnionType", - "MappingType", - "ListType", - "SampleType", - "TypeSpec", - "Framework", - "ImageLayout", - "Dtype", - "DtypeFamily", - "DtypeSpec", - "typed", - "accepts", - "compatible", - "infer_field_types", - "infer_type", - "infer_sample_type", - "canonical_dtype", - "type_to_dict", - "type_from_dict", - "sampletype_from_dict", -] - - -# -------------------------------------------------------------------------------------------------- -# dtype canonicalization + families (array-api-aligned names) -# -------------------------------------------------------------------------------------------------- - -_DTYPE_ALIASES: Dict[str, str] = { - "double": "float64", - "single": "float32", - "half": "float16", - "bool_": "bool", - "boolean": "bool", -} - -_FLOATING = {"float16", "bfloat16", "float32", "float64"} -_INTEGER = {"int8", "int16", "int32", "int64"} -_UNSIGNED = {"uint8", "uint16", "uint32", "uint64"} -_COMPLEX = {"complex64", "complex128"} -_DTYPE_FAMILIES: Dict[str, FrozenSet[str]] = { - "floating": frozenset(_FLOATING), - "integer": frozenset(_INTEGER), - "unsigned": frozenset(_UNSIGNED), - "bool": frozenset({"bool"}), - "complex": frozenset(_COMPLEX), - "numeric": frozenset(_FLOATING | _INTEGER | _UNSIGNED), -} - -#: A concrete dtype name — a closed ``Literal`` (not bare ``str``) so an authored ``ACCEPTS`` / -#: ``PRODUCES`` dtype is typo-checked and UIs / a GUI connection-validator enumerate the -#: choices via ``typing.get_args(Dtype)``. These ARE the union of the family members above (pinned -#: equal in ``tests/test_typespec.py`` so the two can't drift). Authoring uses canonical lowercase -#: names; aliases / casing (``"double"``, ``"FLOAT32"``) and genuinely exotic, platform-dependent -#: dtypes (``float128``, ``complex256``) are *runtime-only* — they reach the field via -#: :func:`canonical_dtype`, the single boundary that normalizes arbitrary input into this domain, and -#: an unmodeled one keeps its own name and simply matches no family. -Dtype = Literal[ - "bool", - "int8", - "int16", - "int32", - "int64", - "uint8", - "uint16", - "uint32", - "uint64", - "float16", - "bfloat16", - "float32", - "float64", - "complex64", - "complex128", -] -#: A relaxed dtype *family* constraint (matches any concrete member). The names are the keys of -#: ``_DTYPE_FAMILIES`` (pinned equal in tests); kept as a ``Literal`` for the same author/UI reasons. -DtypeFamily = Literal["floating", "integer", "unsigned", "bool", "complex", "numeric"] -#: What :attr:`ArrayType.dtype` accepts: a concrete :data:`Dtype` or a relaxed :data:`DtypeFamily`. -DtypeSpec = Union[Dtype, DtypeFamily] - - -def canonical_dtype(x: Any) -> DtypeSpec: - """Normalize a dtype (str, numpy dtype/scalar-type, or torch dtype) to a canonical lowercase name. - - Family names (``"floating"``, ``"integer"``, ``"numeric"`` …) pass through unchanged so they can - be used as relaxed dtype constraints on an :class:`ArrayType`. This is the single boundary where - arbitrary input (aliases, casing, framework dtype objects, exotic dtypes) crosses into the typed - :data:`DtypeSpec` domain — hence the closing ``cast``: a genuinely unmodeled dtype keeps its own - name (and simply matches no family), which is correct even though it lies outside the Literal. - """ - if isinstance(x, str): - s = x.lower() - name = _DTYPE_ALIASES.get(s, s) - elif isinstance(x, np.dtype): - name = str(x.name) - elif isinstance(x, type) and issubclass(x, np.generic): - name = str(np.dtype(x).name) - else: - # Default for any non-numpy value; refined to the bare name for a torch dtype (lazy import — - # torch is a hard dep but we avoid importing it at module load). - name = str(x).lower() - try: - import torch - - if isinstance(x, torch.dtype): - name = str(x).replace("torch.", "") - except ImportError: # pragma: no cover - torch is a hard dep - pass - return cast(DtypeSpec, name) - - -def _dtype_accepts(consumer: str, producer: str) -> bool: - """True iff a ``producer`` concrete/family dtype satisfies a ``consumer`` dtype constraint.""" - c = canonical_dtype(consumer) - p = canonical_dtype(producer) - if c == p: - return True - fam = _DTYPE_FAMILIES.get(c) - if fam is not None: - # producer may itself be a (sub)family name or a concrete member - if p in fam: - return True - pfam = _DTYPE_FAMILIES.get(p) - return pfam is not None and pfam <= fam - return False - - -# -------------------------------------------------------------------------------------------------- -# Spec value objects -# -------------------------------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class Dim: - """A single axis-size constraint: the closed interval ``[min, max]`` (``None`` = unbounded). - - ``name`` is informational only (never matched). ``Dim.any()`` is exactly tf's ``None`` dim. - """ - - min: Optional[int] = None - max: Optional[int] = None - name: Optional[str] = None - - @classmethod - def exact(cls, n: int, name: Optional[str] = None) -> "Dim": - return cls(n, n, name) - - @classmethod - def any(cls, name: Optional[str] = None) -> "Dim": - return cls(None, None, name) - - @classmethod - def range(cls, lo: Optional[int], hi: Optional[int], name: Optional[str] = None) -> "Dim": - return cls(lo, hi, name) - - def accepts(self, other: "Dim") -> bool: - """Strict: ``other``'s whole possible interval lies within this interval.""" - lo = self.min if self.min is not None else 0 - o_lo = other.min if other.min is not None else 0 - if o_lo < lo: - return False - if self.max is not None and (other.max is None or other.max > self.max): - return False - return True - - def compatible(self, other: "Dim") -> bool: - """Permissive: the two intervals could overlap (an unbounded ``other`` is a soft-pass).""" - c_lo = self.min if self.min is not None else 0 - o_lo = other.min if other.min is not None else 0 - if self.max is not None and o_lo > self.max: - return False - if other.max is not None and other.max < c_lo: - return False - return True - - def __str__(self) -> str: - if self.min is None and self.max is None: - return "any" - if self.min == self.max: - return str(self.min) - lo = str(self.min) if self.min is not None else "0" - hi = str(self.max) if self.max is not None else "∞" - return f"{lo}–{hi}" - - def to_dict(self) -> Dict[str, Any]: - return {"min": self.min, "max": self.max, "name": self.name} - - @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Dim": - return cls(d.get("min"), d.get("max"), d.get("name")) - - -@dataclass(frozen=True) -class AnyType: - """Matches everything; the default when an op/source/sample declares no type.""" - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "any"} - - -@dataclass(frozen=True) -class ArrayType: - """An N-D array/tensor constraint across numpy / torch / tensorflow. - - All fields are optional and each ``None`` means "unconstrained": - - * ``ndim`` — required rank (derived from ``shape`` when that is given). - * ``shape`` — per-axis :class:`Dim` tuple. - * ``dtype`` — a :data:`DtypeSpec`: a concrete :data:`Dtype` (``"float32"``) or a relaxed - :data:`DtypeFamily` (``"floating"``/``"numeric"`` …). Stored canonicalized via - :func:`canonical_dtype`, which also accepts aliases / casing / framework dtype objects at runtime. - * ``frameworks`` — allowed framework set, each a :data:`Framework` (``{"numpy", "torch"}`` …). - * ``semantic`` — free-form tag (e.g. ``"image"``); display/inference hint, never matched. - """ - - ndim: Optional[int] = None - shape: Optional[Tuple[Dim, ...]] = None - dtype: Optional[DtypeSpec] = None - # Accept any set on construction (ergonomic ``frameworks={"torch"}``); ``__post_init__`` freezes it. - frameworks: Optional[AbstractSet[Framework]] = None - semantic: Optional[str] = None - - def __post_init__(self) -> None: - if self.shape is not None: - shape = tuple(self.shape) - object.__setattr__(self, "shape", shape) - if self.ndim is None: - object.__setattr__(self, "ndim", len(shape)) - elif self.ndim != len(shape): - raise ValueError(f"ArrayType: ndim={self.ndim} disagrees with shape length {len(shape)}") - if self.frameworks is not None and not isinstance(self.frameworks, frozenset): - object.__setattr__(self, "frameworks", frozenset(self.frameworks)) - if self.dtype is not None: - object.__setattr__(self, "dtype", canonical_dtype(self.dtype)) - - @classmethod - def image( - cls, - layout: ImageLayout = "CHW", - channels: Union[int, Tuple[int, ...]] = 3, - dtype: Optional[DtypeSpec] = None, - framework: Optional[Framework] = None, - ) -> "ArrayType": - """Rank-3 image convenience. ``channels`` as a tuple is treated as the inclusive range - ``[min, max]`` (a pragmatic approximation — pass an exact :class:`Dim` via the constructor - for anything finer).""" - if isinstance(channels, int): - cdim = Dim.exact(channels, "C") - else: - cdim = Dim(min(channels), max(channels), "C") - h, w = Dim.any("H"), Dim.any("W") - up = layout.upper() - if up == "CHW": - shape: Tuple[Dim, ...] = (cdim, h, w) - elif up == "HWC": - shape = (h, w, cdim) - else: - raise ValueError(f"ArrayType.image: layout must be 'CHW' or 'HWC', got {layout!r}") - fws = frozenset({framework}) if framework else None - return cls(ndim=3, shape=shape, dtype=dtype, frameworks=fws, semantic="image") - - @classmethod - def parse( - cls, - spec: str, - dtype: Optional[DtypeSpec] = None, - framework: Optional[Framework] = None, - semantic: Optional[str] = None, - ) -> "ArrayType": - """Build from a jaxtyping-style shape string: space-separated axes where a bare int is an - exact size, ``lo-hi`` a bounded range (open ends allowed, e.g. ``"1-"``), and any other token - a named free axis. Example: ``ArrayType.parse("3 h w", dtype="float32")``. - """ - dims: List[Dim] = [] - for tok in spec.split(): - if tok.startswith("*"): - raise ValueError("ArrayType.parse: variadic '*' axes are not supported in v1") - if tok.isdigit(): - dims.append(Dim.exact(int(tok))) - elif "-" in tok[1:] and all(part == "" or part.isdigit() for part in tok.split("-", 1)): - lo_s, hi_s = tok.split("-", 1) - dims.append(Dim.range(int(lo_s) if lo_s else None, int(hi_s) if hi_s else None)) - else: - dims.append(Dim.any(tok)) - fws = frozenset({framework}) if framework else None - return cls(ndim=len(dims), shape=tuple(dims), dtype=dtype, frameworks=fws, semantic=semantic) - - def __str__(self) -> str: - parts: List[str] = [] - if self.frameworks: - parts.append("/".join(sorted(self.frameworks))) - if self.dtype: - parts.append(str(self.dtype)) - if self.shape is not None: - parts.append(f"shape=({', '.join(str(d) for d in self.shape)})") - elif self.ndim is not None: - parts.append(f"rank-{self.ndim}") - return f"array[{', '.join(parts)}]" if parts else "array" - - def explain_mismatch(self, producer: "ArrayType") -> List[str]: - """Return human-readable reasons why this consumer does not accept *producer*.""" - reasons: List[str] = [] - if self.frameworks is not None: - if producer.frameworks is None: - reasons.append(f"framework unknown in upstream (op requires {'/'.join(sorted(self.frameworks))})") - elif not (producer.frameworks <= self.frameworks): - exp = "/".join(sorted(self.frameworks)) - got = "/".join(sorted(producer.frameworks)) - reasons.append(f"framework mismatch: op requires {exp}, upstream produces {got}") - if self.ndim is not None and producer.ndim is not None and producer.ndim != self.ndim: - reasons.append(f"rank mismatch: op requires rank {self.ndim}, upstream has rank {producer.ndim}") - if self.dtype is not None and producer.dtype is not None: - if not _dtype_accepts(self.dtype, producer.dtype): - reasons.append(f"dtype mismatch: op requires {self.dtype}, upstream produces {producer.dtype}") - elif self.dtype is not None and producer.dtype is None: - reasons.append(f"dtype unknown in upstream (op requires {self.dtype})") - if self.shape is not None and producer.shape is not None and len(self.shape) == len(producer.shape): - for i, (cdim, pdim) in enumerate(zip(self.shape, producer.shape)): - if not cdim.accepts(pdim): - name = f" ({cdim.name})" if cdim.name else "" - reasons.append(f"axis {i}{name}: op requires size {cdim}, upstream has size {pdim}") - return reasons - - def to_dict(self) -> Dict[str, Any]: - return { - "kind": "array", - "ndim": self.ndim, - "shape": [d.to_dict() for d in self.shape] if self.shape is not None else None, - "dtype": self.dtype, - "frameworks": sorted(self.frameworks) if self.frameworks is not None else None, - "semantic": self.semantic, - } - - -@dataclass(frozen=True) -class PythonType: - """A non-array Python value, matched by exact qualname (e.g. ``"PIL.Image.Image"``, ``"dict"``).""" - - qualname: str - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "python", "qualname": self.qualname} - - -@dataclass(frozen=True) -class UnionType: - """Any-of. Stored as a tuple so the value object stays hashable/immutable.""" - - members: Tuple[TypeSpec, ...] - - def __post_init__(self) -> None: - object.__setattr__(self, "members", tuple(self.members)) - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "union", "members": [type_to_dict(m) for m in self.members]} - - -@dataclass(frozen=True) -class ListType: - """A homogeneous sequence (mirrors ``datasets.Sequence`` / ``List``).""" - - item: TypeSpec - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "list", "item": type_to_dict(self.item)} - - -@dataclass(frozen=True) -class MappingType: - """A struct with named fields (mirrors a nested ``datasets.Features`` / Python ``dict`` schema). - - Fields are stored as a sorted tuple of ``(name, spec)`` pairs so the object stays hashable. - Build via :meth:`of`. - """ - - fields: Tuple[Tuple[str, TypeSpec], ...] - - def __post_init__(self) -> None: - items = self.fields.items() if isinstance(self.fields, dict) else self.fields - object.__setattr__(self, "fields", tuple(sorted(items, key=lambda kv: kv[0]))) - - @classmethod - def of(cls, fields: Dict[str, TypeSpec]) -> "MappingType": - return cls(tuple(sorted(fields.items(), key=lambda kv: kv[0]))) - - def field_map(self) -> Dict[str, TypeSpec]: - return dict(self.fields) - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "mapping", "fields": {k: type_to_dict(v) for k, v in self.fields}} - - -@dataclass(frozen=True) -class SampleType: - """The ``(input, target)`` type pair an op/source declares or a sample reports.""" - - input: TypeSpec = field(default_factory=AnyType) - target: TypeSpec = field(default_factory=AnyType) - - def accepts(self, producer: "SampleType") -> bool: - """Strict: this consumer accepts every value ``producer`` can emit (both slots).""" - return _accepts(self.input, producer.input, permissive=False) and _accepts( - self.target, producer.target, permissive=False - ) - - def compatible(self, producer: "SampleType") -> bool: - """Permissive: edit-time/discovery wiring — ``Any``/unknown on either side passes.""" - return _accepts(self.input, producer.input, permissive=True) and _accepts( - self.target, producer.target, permissive=True - ) - - def explain_mismatch(self, producer: "SampleType") -> str: - """Return a human-readable explanation of why ``self.accepts(producer)`` is False. - - Walks each slot (input / target) and collects all failing conditions — framework, - rank, dtype, and per-axis shape — then returns them as a single comma-separated - sentence so the caller can embed it directly in an error message. - """ - reasons: List[str] = [] - for slot, cspec, pspec in (("input", self.input, producer.input), ("target", self.target, producer.target)): - slot_reasons = _explain_slot_mismatch(cspec, pspec) - reasons.extend(f"{slot} {r}" for r in slot_reasons) - return "; ".join(reasons) if reasons else "incompatible types (no specific reason derived)" - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "sample", "input": type_to_dict(self.input), "target": type_to_dict(self.target)} - - @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "SampleType": - return cls(type_from_dict(d["input"]), type_from_dict(d["target"])) - - # -- datasets.Features bridge (the concrete per-sample stored representation) ------------------ - - @classmethod - def from_hf_features(cls, features: Any, extras: Optional[Dict[str, Any]] = None) -> "SampleType": - """Build from a ``datasets.Features`` (or its ``.to_dict()`` form) keyed by ``"input"`` / - ``"target"``, applying the sidecar ``extras`` that Features can't express (framework, ranges, - or a full ``typespec`` override for Any/Union/family-only slots).""" - from datasets import Features - - if not isinstance(features, Features): - features = Features.from_dict(features) - extras = extras or {} - return cls( - input=_slot_from_features(features.get("input"), extras.get("input")), - target=_slot_from_features(features.get("target"), extras.get("target")), - ) - - def to_hf_features(self) -> Tuple[Any, Dict[str, Any]]: - """Split into a ``datasets.Features`` (the Feature-expressible part) and an ``extras`` dict - (framework/range refinements + a ``typespec`` fallback for slots Features can't represent).""" - from datasets import Features - - feats: Dict[str, Any] = {} - extras: Dict[str, Any] = {} - for slot, spec in (("input", self.input), ("target", self.target)): - feat, slot_extra = _feature_from_typespec(spec) - if feat is not None: - feats[slot] = feat - if slot_extra: - extras[slot] = slot_extra - return Features(feats), extras - - -# -------------------------------------------------------------------------------------------------- -# Matching (module-level dispatch — no base class) -# -------------------------------------------------------------------------------------------------- - - -def accepts(consumer: TypeSpec, producer: TypeSpec) -> bool: - """Strict single-slot match: does ``consumer`` accept every value ``producer`` can emit?""" - return _accepts(consumer, producer, permissive=False) - - -def compatible(consumer: TypeSpec, producer: TypeSpec) -> bool: - """Permissive single-slot match (edit-time / discovery): ``Any``/unknown on either side passes.""" - return _accepts(consumer, producer, permissive=True) - - -def _explain_slot_mismatch(consumer: TypeSpec, producer: TypeSpec) -> List[str]: - """Return human-readable reasons why *consumer* does not strictly accept *producer* for one slot.""" - if isinstance(consumer, AnyType) or isinstance(producer, AnyType): - return [] - if isinstance(consumer, ArrayType) and isinstance(producer, ArrayType): - return consumer.explain_mismatch(producer) - if isinstance(consumer, UnionType): - # None of the union members accepted — collect reasons from the closest member. - all_reasons = [_explain_slot_mismatch(m, producer) for m in consumer.members] - # Pick the member with fewest (most specific) reasons as the most helpful. - best = min(all_reasons, key=lambda r: (len(r) == 0, len(r)), default=[]) - return best if best else ["incompatible union types"] - if isinstance(consumer, PythonType) and isinstance(producer, PythonType): - if consumer.qualname != producer.qualname: - return [f"type mismatch: op requires {consumer.qualname}, upstream produces {producer.qualname}"] - return [] - # Framework/kind-level mismatch (e.g. ArrayType vs PythonType). - return [f"kind mismatch: op requires {type(consumer).__name__}, upstream produces {type(producer).__name__}"] - - -def _accepts(consumer: TypeSpec, producer: TypeSpec, *, permissive: bool) -> bool: - if isinstance(consumer, AnyType): - return True - if isinstance(producer, AnyType): - return permissive - if isinstance(producer, UnionType): - return all(_accepts(consumer, m, permissive=permissive) for m in producer.members) - if isinstance(consumer, UnionType): - return any(_accepts(m, producer, permissive=permissive) for m in consumer.members) - if isinstance(consumer, ArrayType) and isinstance(producer, ArrayType): - return _array_accepts(consumer, producer, permissive) - if isinstance(consumer, PythonType) and isinstance(producer, PythonType): - return consumer.qualname == producer.qualname - if isinstance(consumer, ListType) and isinstance(producer, ListType): - return _accepts(consumer.item, producer.item, permissive=permissive) - if isinstance(consumer, MappingType) and isinstance(producer, MappingType): - pm = producer.field_map() - for key, cval in consumer.fields: - if key not in pm or not _accepts(cval, pm[key], permissive=permissive): - return False - return True - return False - - -def _array_accepts(consumer: ArrayType, producer: ArrayType, permissive: bool) -> bool: - if consumer.ndim is not None: - if producer.ndim is None: - if not permissive: - return False - elif producer.ndim != consumer.ndim: - return False - if consumer.shape is not None: - producer_shape = producer.shape - if producer_shape is None and producer.ndim is not None and producer.ndim == len(consumer.shape): - # A shapeless producer of matching rank carries no per-axis info — equivalent to all - # unbounded dims. This keeps matching symmetric across the Features round-trip, where a - # 1-D ``ArrayType(shape=None)`` stores as a Sequence and reads back as ``shape=(Dim.any(),)``. - producer_shape = tuple(Dim.any() for _ in consumer.shape) - if producer_shape is None: - if not permissive: - return False - elif len(producer_shape) != len(consumer.shape): - return False - else: - for cdim, pdim in zip(consumer.shape, producer_shape): - ok = cdim.compatible(pdim) if permissive else cdim.accepts(pdim) - if not ok: - return False - if consumer.dtype is not None: - if producer.dtype is None: - if not permissive: - return False - elif not _dtype_accepts(consumer.dtype, producer.dtype): - return False - if consumer.frameworks is not None: - if producer.frameworks is None: - if not permissive: - return False - elif permissive: - if not (producer.frameworks & consumer.frameworks): - return False - elif not (producer.frameworks <= consumer.frameworks): - return False - return True - - -# -------------------------------------------------------------------------------------------------- -# Inference from live values -# -------------------------------------------------------------------------------------------------- - - -def infer_type(value: Any) -> TypeSpec: - """Infer a concrete :data:`TypeSpec` from a live value. ``None`` ⇒ :class:`AnyType` (so an unset - target stays permissive). torch/tf/PIL are imported lazily and guarded.""" - if value is None: - return AnyType() - if isinstance(value, np.ndarray): - return ArrayType( - ndim=value.ndim, - shape=tuple(Dim.exact(int(s)) for s in value.shape), - dtype=canonical_dtype(value.dtype), - frameworks=frozenset({"numpy"}), - ) - if isinstance(value, np.generic): - return ArrayType(ndim=0, shape=(), dtype=canonical_dtype(value.dtype), frameworks=frozenset({"numpy"})) - try: - import torch - - if isinstance(value, torch.Tensor): - return ArrayType( - ndim=value.dim(), - shape=tuple(Dim.exact(int(s)) for s in tuple(value.shape)), - dtype=canonical_dtype(value.dtype), - frameworks=frozenset({"torch"}), - ) - except ImportError: # pragma: no cover - torch is a hard dep - pass - try: - import tensorflow as tf - - if isinstance(value, tf.Tensor): # pragma: no cover - tensorflow is optional / not installed here - shape = tuple(Dim.exact(int(s)) if s is not None else Dim.any() for s in value.shape) - return ArrayType( - ndim=len(shape), - shape=shape, - dtype=canonical_dtype(value.dtype.name), - frameworks=frozenset({"tensorflow"}), - ) - except ImportError: - pass - try: - from PIL import Image - - if isinstance(value, Image.Image): - return PythonType("PIL.Image.Image") - except ImportError: # pragma: no cover - Pillow optional - pass - if isinstance(value, bool): - return PythonType("bool") - if isinstance(value, int): - return PythonType("int") - if isinstance(value, float): - return PythonType("float") - if isinstance(value, str): - return PythonType("str") - if isinstance(value, dict): - return PythonType("dict") - if isinstance(value, list): - return PythonType("list") - if isinstance(value, tuple): - return PythonType("tuple") - return PythonType(type(value).__qualname__) - - -def infer_sample_type(sample: "Sample") -> SampleType: - """Infer the :class:`SampleType` of a live sample (duck-typed: reads ``.input`` / ``.target``).""" - return SampleType(input=infer_type(sample.input), target=infer_type(sample.target)) - - -def infer_field_types(sample: Any) -> Dict[str, TypeSpec]: - """Per-field type inference for a typed bag: ``{field key: TypeSpec of the item's payload}``. - - The typed-bag analogue of :func:`infer_sample_type` — one spec per NAMED field instead of - the fixed input/target pair. The spec describes the item's PAYLOAD (via - :func:`~sampleflux.bag.items.item_data`, so an array item and a data-bearing wrapper both - report their array); a payload-less structured item reports its Python type. Visual - editors use this to type per-field sockets and pickers. - """ - from sampleflux.bag.items import item_data - from sampleflux.bag.sample import TypedSample - - if not isinstance(sample, TypedSample): - raise TypeError(f"infer_field_types: expected a TypedSample, got {type(sample).__name__}") - specs: Dict[str, TypeSpec] = {} - for key, item in sample.items(): - payload = item_data(item) - specs[key] = PythonType(type(item).__qualname__) if payload is item else infer_type(payload) - return specs - - -# -------------------------------------------------------------------------------------------------- -# JSON (de)serialization -# -------------------------------------------------------------------------------------------------- - - -def type_to_dict(t: TypeSpec) -> Dict[str, Any]: - return t.to_dict() - - -def type_from_dict(d: Dict[str, Any]) -> TypeSpec: - kind = d["kind"] - if kind == "any": - return AnyType() - if kind == "python": - return PythonType(d["qualname"]) - if kind == "array": - shape_d = d.get("shape") - shape = tuple(Dim.from_dict(x) for x in shape_d) if shape_d is not None else None - fw = d.get("frameworks") - return ArrayType( - ndim=d.get("ndim"), - shape=shape, - dtype=d.get("dtype"), - frameworks=frozenset(fw) if fw is not None else None, - semantic=d.get("semantic"), - ) - if kind == "union": - return UnionType(tuple(type_from_dict(m) for m in d["members"])) - if kind == "list": - return ListType(type_from_dict(d["item"])) - if kind == "mapping": - return MappingType.of({k: type_from_dict(v) for k, v in d["fields"].items()}) - raise ValueError(f"type_from_dict: unknown kind {kind!r}") - - -def sampletype_from_dict(d: Dict[str, Any]) -> SampleType: - return SampleType.from_dict(d) - - -# -------------------------------------------------------------------------------------------------- -# datasets.Features bridge helpers -# -------------------------------------------------------------------------------------------------- - - -def _slot_from_features(feat: Any, extra: Optional[Dict[str, Any]]) -> TypeSpec: - """Reconstruct one slot's :data:`TypeSpec` from its Feature plus sidecar ``extras`` (recursively).""" - extra = extra or {} - if "typespec" in extra: - return type_from_dict(extra["typespec"]) - if feat is None: - return AnyType() - return _apply_extras(_typespec_from_feature(feat), extra) - - -def _apply_extras(spec: TypeSpec, extra: Dict[str, Any]) -> TypeSpec: - """Layer the sidecar refinements Features can't hold back onto a Feature-derived spec, recursing - into mapping fields / list items so per-field framework/range info survives the round-trip.""" - if not extra: - return spec - if "typespec" in extra: - return type_from_dict(extra["typespec"]) - if isinstance(spec, ArrayType): - frameworks = frozenset(extra["frameworks"]) if "frameworks" in extra else spec.frameworks - shape = spec.shape - bounds = extra.get("shape_bounds") - if bounds and shape is not None: - new_shape = list(shape) - for idx_s, (lo, hi) in bounds.items(): - new_shape[int(idx_s)] = Dim.range(lo, hi, new_shape[int(idx_s)].name) - shape = tuple(new_shape) - return ArrayType( - ndim=spec.ndim, - shape=shape, - dtype=spec.dtype, - frameworks=frameworks, - semantic=extra.get("semantic", spec.semantic), - ) - if isinstance(spec, MappingType) and "fields" in extra: - sub = extra["fields"] - return MappingType.of({k: _apply_extras(v, sub.get(k, {})) for k, v in spec.field_map().items()}) - if isinstance(spec, ListType) and "item" in extra: - return ListType(_apply_extras(spec.item, extra["item"])) - return spec - - -def _typespec_from_feature(feat: Any) -> TypeSpec: - from datasets import Features, Image, Value - - if isinstance(feat, (Features, dict)): - return MappingType.of({k: _typespec_from_feature(v) for k, v in feat.items()}) - inner = feat[0] if isinstance(feat, list) else getattr(feat, "feature", None) - is_sequence = isinstance(feat, list) or feat.__class__.__name__ in ("Sequence", "List", "LargeList") - if inner is not None and is_sequence: - # A sequence of a numeric scalar is canonically a 1-D array (that's how HF stores a [N] tensor); - # only a sequence of non-scalars (struct / nested array / string) is a true ListType. - if isinstance(inner, Value) and inner.dtype not in ("string", "large_string"): - return ArrayType(ndim=1, shape=(Dim.any(),), dtype=canonical_dtype(inner.dtype)) - return ListType(_typespec_from_feature(inner)) - if isinstance(feat, Image): - return PythonType("PIL.Image.Image") - shape = getattr(feat, "shape", None) - dtype = getattr(feat, "dtype", None) - if shape is not None and dtype is not None: - dims = tuple(Dim.any() if s is None else Dim.exact(int(s)) for s in shape) - return ArrayType(ndim=len(dims), shape=dims, dtype=canonical_dtype(dtype)) - if isinstance(feat, Value): - if feat.dtype in ("string", "large_string"): - return PythonType("str") - return ArrayType(ndim=0, dtype=canonical_dtype(feat.dtype)) - return AnyType() - - -def _feature_from_typespec(spec: TypeSpec) -> Tuple[Any, Dict[str, Any]]: - """Return ``(feature_or_None, extras)`` for one slot. ``feature`` is ``None`` when the spec can't - be a concrete ``Features`` entry (Any/Union/family-dtype/unknown-rank/exotic python type).""" - from datasets import Array2D, Array3D, Array4D, Array5D, Features, Image, Sequence, Value - - if isinstance(spec, PythonType): - if spec.qualname == "PIL.Image.Image": - return Image(), {} - if spec.qualname == "str": - return Value("string"), {} - return None, {"typespec": spec.to_dict()} - if isinstance(spec, ListType): - inner_feat, inner_extra = _feature_from_typespec(spec.item) - if inner_feat is None: - return None, {"typespec": spec.to_dict()} - return Sequence(inner_feat), ({"item": inner_extra} if inner_extra else {}) - if isinstance(spec, MappingType): - feats: Dict[str, Any] = {} - sub: Dict[str, Any] = {} - for key, val in spec.fields: - feat, extra = _feature_from_typespec(val) - if feat is None: - return None, {"typespec": spec.to_dict()} - feats[key] = feat - if extra: - sub[key] = extra - return Features(feats), ({"fields": sub} if sub else {}) - if isinstance(spec, ArrayType): - if spec.ndim is None or spec.dtype is None or spec.dtype in _DTYPE_FAMILIES: - return None, {"typespec": spec.to_dict()} - extras: Dict[str, Any] = {} - if spec.frameworks is not None: - extras["frameworks"] = sorted(spec.frameworks) - if spec.semantic is not None: - extras["semantic"] = spec.semantic - if spec.ndim == 0: - return Value(spec.dtype), extras - if spec.ndim == 1: - return Sequence(Value(spec.dtype)), extras - arrcls = {2: Array2D, 3: Array3D, 4: Array4D, 5: Array5D}.get(spec.ndim) - if arrcls is None: - return None, {"typespec": spec.to_dict()} - shape_list: List[Optional[int]] = [] - bounds: Dict[str, List[Optional[int]]] = {} - if spec.shape is None: - shape_list = [None] * spec.ndim # rank known but no per-axis info -> all dynamic - else: - for i, d in enumerate(spec.shape): - if d.min is not None and d.min == d.max: - shape_list.append(int(d.min)) - else: - shape_list.append(None) - if d.min is not None or d.max is not None: - bounds[str(i)] = [d.min, d.max] - if bounds: - extras["shape_bounds"] = bounds - return arrcls(shape=tuple(shape_list), dtype=spec.dtype), extras - # AnyType / UnionType - return None, {"typespec": spec.to_dict()} - - -# -------------------------------------------------------------------------------------------------- -# Declaration decorator -# -------------------------------------------------------------------------------------------------- - - -def typed(*, accepts: Optional[SampleType] = None, produces: Optional[SampleType] = None) -> Callable[[C], C]: - """Class decorator that sets ``ACCEPTS`` / ``PRODUCES`` on an op/source (ergonomic alternative to - plain class attributes). Validates the arguments are :class:`SampleType` at decoration time.""" - - def deco(cls: C) -> C: - if accepts is not None: - if not isinstance(accepts, SampleType): - raise TypeError(f"@typed(accepts=...) expects a SampleType, got {type(accepts).__name__}") - setattr(cls, "ACCEPTS", accepts) - if produces is not None: - if not isinstance(produces, SampleType): - raise TypeError(f"@typed(produces=...) expects a SampleType, got {type(produces).__name__}") - setattr(cls, "PRODUCES", produces) - return cls - - return deco diff --git a/tests/_bag_fixtures.py b/tests/_bag_fixtures.py index 4846704..27a3ec3 100644 --- a/tests/_bag_fixtures.py +++ b/tests/_bag_fixtures.py @@ -13,7 +13,7 @@ import numpy as np -from sampleflux import Image, Mask, Regions, Transform, TypedSample, item_data, with_data +from sampleflux import Image, Mask, Regions, Sample, Transform, item_data, with_data class FixtureFlip(Transform): @@ -28,7 +28,7 @@ def __init__(self, p: float = 0.5, only: Optional[List[str]] = None) -> None: super().__init__(only=only) self.p = p - def get_params(self, sample: TypedSample) -> Dict[str, Any]: + def get_params(self, sample: Sample) -> Dict[str, Any]: do = float(np.random.random()) < self.p return {"do": do, "width": _reference_width(sample)} @@ -59,7 +59,7 @@ def _flip_regions(item: Regions, params: Dict[str, Any]) -> Regions: return Regions(boxes=boxes, labels=item.labels, scores=item.scores, canvas=item.canvas) -def _reference_width(sample: TypedSample) -> Optional[int]: +def _reference_width(sample: Sample) -> Optional[int]: """The horizontal extent to flip boxes against — from the first Image/Mask, or a Regions canvas.""" for _, item in sample.items(): if isinstance(item, Image): diff --git a/tests/test_augment_ops.py b/tests/test_augment_ops.py deleted file mode 100644 index 4199654..0000000 --- a/tests/test_augment_ops.py +++ /dev/null @@ -1,312 +0,0 @@ -"""Tests for the augmentation adapters + the generated per-transform op families. - -``AlbumentationsOp`` / ``TorchvisionTransformOp`` are classic sample-scoped ops -(``__call__(sample)`` — the form every engine, composing op, AND visual-canvas node -classifier handles) applying ONE library draw jointly to input and target per the -``target`` mode. Transforms are configured Confluid-natively (nested ``!class:`` nodes or -the generated ``Alb*`` / ``Tv*`` per-transform ops from -:mod:`sampleflux.ops.albumentations_transforms` / -:mod:`sampleflux.ops.torchvision_transforms`). -""" - -import subprocess -import sys -from pathlib import Path - -import albumentations as A -import confluid # type: ignore[import-not-found] -import numpy as np -import pytest -import torch -from PIL import Image - -import sampleflux.ops.albumentations_transforms as albt -from sampleflux.core import Flux -from sampleflux.kinds import op_contract -from sampleflux.ops.albumentations import AlbumentationsOp -from sampleflux.ops.target import MasksToDetectionBoxesOp -from sampleflux.ops.torchvision import TorchvisionTransformOp -from sampleflux.sample import Sample - - -def _image() -> np.ndarray: - return np.arange(4 * 6 * 3, dtype=np.uint8).reshape(4, 6, 3) - - -def _mask() -> np.ndarray: - mask = np.zeros((4, 6), dtype=np.uint8) - mask[1:3, 0:2] = 1 - return mask - - -def _sample(**meta: object) -> Sample: - return Sample(_image(), _mask(), dict(meta)) - - -def _detection_sample() -> Sample: - # MasksToDetectionBoxesOp derives the tight xyxy box from the mask — the exact - # detection-dict contract both adapters consume in target="boxes" mode. - return MasksToDetectionBoxesOp()(Sample(_image(), _mask(), {})) - - -class TestAlbumentationsOp: - def test_zero_arg_construction_and_lazy_validation(self) -> None: - op = AlbumentationsOp() - with pytest.raises(ValueError, match="transform"): - op(_sample()) - - def test_transform_and_transforms_mutually_exclusive(self) -> None: - op = AlbumentationsOp(transform=A.HorizontalFlip(p=1.0), transforms=[A.HorizontalFlip(p=1.0)]) - with pytest.raises(ValueError, match="not both"): - op(_sample()) - - def test_contract_is_sample_scoped(self) -> None: - # Sample scope is what the engine fast-path, every composing op, AND the visual - # canvas classifier handle — guard the signature. - contract = op_contract(AlbumentationsOp()) - assert contract.accepts == "sample" - - def test_single_transform_input_only(self) -> None: - out = list(Flux(source=[_sample(idx=7)], ops=[AlbumentationsOp(A.HorizontalFlip(p=1.0))]))[0] - assert np.array_equal(out.input, _image()[:, ::-1]) - assert np.array_equal(out.target, _mask()) # target NOT flipped in "none" mode - assert out.meta == {"idx": 7} - - def test_transforms_list_mask_mode(self) -> None: - op = AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="mask", seed=0) - out = list(Flux(source=[_sample(idx=7)], ops=[op]))[0] - assert np.array_equal(out.input, _image()[:, ::-1]) - assert np.array_equal(out.target, _mask()[:, ::-1]) - assert out.meta == {"idx": 7} # metadata untouched - - def test_prebuilt_compose_accepted_seed_rejected(self) -> None: - compose = A.Compose([A.HorizontalFlip(p=1.0)]) - out = AlbumentationsOp(compose, target="mask")(_sample()) - assert np.array_equal(out.target, _mask()[:, ::-1]) - with pytest.raises(ValueError, match="seed"): - AlbumentationsOp(compose, target="mask", seed=3)(_sample()) - - def test_pipeline_rebuilds_when_transform_changes(self) -> None: - op = AlbumentationsOp(A.HorizontalFlip(p=1.0)) - first = op.pipeline - op.transform = A.HorizontalFlip(p=0.0) - assert op.pipeline is not first # the lazy cache keys on the configured objects - - def test_pil_input_accepted(self) -> None: - out = AlbumentationsOp(A.HorizontalFlip(p=1.0))(Sample(Image.fromarray(_image()), None, {})) - assert isinstance(out.input, np.ndarray) - assert np.array_equal(out.input, _image()[:, ::-1]) - - def test_boxes_mode_auto_bbox_params(self) -> None: - # When the op builds the Compose itself, bbox_params are added automatically. - op = AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="boxes") - sample = _detection_sample() - out = list(Flux(source=[sample], ops=[op]))[0] - width = _image().shape[1] - x0, _, x1, _ = sample.target["boxes"][0].tolist() - assert isinstance(out.target["boxes"], torch.Tensor) - assert out.target["boxes"].dtype == torch.float32 - assert out.target["labels"].dtype == torch.int64 - assert np.allclose(out.target["boxes"][0].tolist(), [width - x1, 1.0, width - x0, 3.0], atol=1e-4) - - def test_boxes_mode_prebuilt_compose_requires_bbox_params(self) -> None: - op = AlbumentationsOp(A.Compose([A.HorizontalFlip(p=1.0)]), target="boxes") - with pytest.raises(ValueError, match="bbox_params"): - op(_detection_sample()) - - def test_boxes_mode_requires_detection_dict(self) -> None: - op = AlbumentationsOp(transforms=[A.HorizontalFlip(p=1.0)], target="boxes") - with pytest.raises(TypeError, match="detection"): - op(Sample(_image(), "not-a-dict", {})) - - def test_confluid_native_yaml_roundtrip(self) -> None: - # The YAML surface is nested !class: nodes — no library-specific dict formats. - yaml_text = ( - "!class:sampleflux.ops.albumentations.AlbumentationsOp\n" - "target: mask\n" - "transforms:\n" - " - !class:albumentations.HorizontalFlip\n" - " p: 1.0\n" - ) - op = confluid.load(yaml_text) - out = op(_sample()) - assert np.array_equal(out.target, _mask()[:, ::-1]) - # Pipeline Parity: dump → reload → identical output. - reloaded = confluid.load(confluid.dump(op)) - out2 = reloaded(_sample()) - assert np.array_equal(out.input, out2.input) - assert np.array_equal(out.target, out2.target) - - -class TestGeneratedAlbumentationsOps: - def test_family_generated(self) -> None: - assert len(albt.__all__) > 50 - for name in ("AlbHorizontalFlip", "AlbAffine", "AlbRandomBrightnessContrast"): - assert name in albt.__all__ - - def test_flip_parity_with_raw_library(self) -> None: - out = list(Flux(source=[_sample(idx=1)], ops=[albt.AlbHorizontalFlip(p=1.0, target="mask")]))[0] - assert np.array_equal(out.input, _image()[:, ::-1]) - assert np.array_equal(out.target, _mask()[:, ::-1]) - assert out.meta == {"idx": 1} - - def test_marks_and_canvas_classification(self) -> None: - cls = albt.AlbHorizontalFlip - assert cls.__confluid_category__ == "op" - assert cls.__confluid_group__ == "augment/albumentations" - assert cls.__confluid_random__ is True - # The canvas op classifier reads vars(cls) — the base __call__ must be re-stated. - assert "__call__" in vars(cls) - assert cls.LIBRARY_CLS is A.HorizontalFlip - - def test_signature_mirrors_transform_plus_adapter_knobs(self) -> None: - import inspect - - params = list(inspect.signature(albt.AlbHorizontalFlip).parameters) - assert "p" in params - assert params[-2:] == ["target", "seed"] - docs = confluid.parse_param_docs(albt.AlbHorizontalFlip) - assert docs.get("target") # adapter knobs documented in the spliced Args block - - def test_required_param_lazy_error(self) -> None: - crop = albt.AlbRandomCrop() # zero-arg construction always works - with pytest.raises(Exception, match="height"): - crop(_sample()) # the library's own missing-argument error, raised lazily - - def test_post_construction_reconfigure_rebuilds(self) -> None: - op = albt.AlbHorizontalFlip(p=0.0) - assert np.array_equal(op(_sample()).input, _image()) # p=0 → identity - op.p = 1.0 # confluid post-construction paradigm - assert np.array_equal(op(_sample()).input, _image()[:, ::-1]) - - def test_generated_op_unwraps_in_transforms_list(self) -> None: - # A generated op wired into an adapter's transforms slot (the canvas pattern) - # unwraps to its inner library transform via raw_transform. - op = AlbumentationsOp(transforms=[albt.AlbHorizontalFlip(p=1.0)], target="mask") - out = op(_sample()) - assert np.array_equal(out.target, _mask()[:, ::-1]) - - def test_short_name_yaml_roundtrip(self) -> None: - yaml_text = "!class:AlbHorizontalFlip\np: 1.0\ntarget: mask\n" - op = confluid.load(yaml_text) - out = op(_sample()) - assert np.array_equal(out.target, _mask()[:, ::-1]) - dumped = confluid.dump(op) - assert "AlbHorizontalFlip" in dumped and "p: 1.0" in dumped - out2 = confluid.load(dumped)(_sample()) - assert np.array_equal(out.input, out2.input) - - def test_composes_inside_random_apply_and_chain(self) -> None: - # Composing ops route inner ops through core._apply_op, so the generated ops nest - # inside the gate/chain — the canonical "gate an augmentation" pattern. - from sampleflux.ops.random_apply import RandomApply - from sampleflux.ops.transform_chain import TransformChain - - chain = TransformChain( - ops=[RandomApply(op=albt.AlbHorizontalFlip(p=1.0, target="mask"), probability=1.0, random_state=0)] - ) - out = list(Flux(source=[_sample(idx=3)], ops=[chain]))[0] - assert np.array_equal(out.input, _image()[:, ::-1]) - assert np.array_equal(out.target, _mask()[:, ::-1]) - assert out.meta == {"idx": 3} - - def test_composes_inside_enable(self) -> None: - from sampleflux.ops.enable import Enable - - enable = Enable(ops=[albt.AlbHorizontalFlip(p=1.0, target="mask")]) - setattr(enable, "augment", True) # the toggle flag arrives post-construction (Confluid paradigm) - out = list(Flux(source=[_sample()], ops=[enable]))[0] - assert np.array_equal(out.input, _image()[:, ::-1]) - assert np.array_equal(out.target, _mask()[:, ::-1]) - - -class TestTorchvisionTransformOp: - v2 = pytest.importorskip("torchvision.transforms.v2") - - def test_zero_arg_construction_and_lazy_validation(self) -> None: - op = TorchvisionTransformOp() - with pytest.raises(ValueError, match="transform"): - op(_sample()) - - def test_contract_is_sample_scoped(self) -> None: - assert op_contract(TorchvisionTransformOp()).accepts == "sample" - - def test_input_only_flip_emits_chw_tensor(self) -> None: - op = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0)) - out = list(Flux(source=[_sample(idx=7)], ops=[op]))[0] - assert type(out.input) is torch.Tensor # tv_tensors subclass stripped - assert out.input.shape == (3, 4, 6) - assert np.array_equal(out.input.permute(1, 2, 0).numpy(), _image()[:, ::-1]) - assert np.array_equal(out.target, _mask()) # untouched in "none" mode - assert out.meta == {"idx": 7} - - def test_transforms_list_mask_mode_matches_albumentations(self) -> None: - # Cross-library parity: the same deterministic flip through either adapter - # yields the same pixels (layouts differ — CHW tensor vs HWC array). - tv = TorchvisionTransformOp(transforms=[self.v2.RandomHorizontalFlip(p=1.0)], target="mask")(_sample()) - alb = AlbumentationsOp(A.HorizontalFlip(p=1.0), target="mask")(_sample()) - assert np.array_equal(tv.input.permute(1, 2, 0).numpy(), alb.input) - assert np.array_equal(tv.target.numpy(), alb.target) - - def test_two_dim_input_gains_channel_axis(self) -> None: - out = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0))(Sample(_mask(), None, {})) - assert out.input.shape == (1, 4, 6) - - def test_pil_input_stays_pil(self) -> None: - out = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0))(Sample(Image.fromarray(_image()), None, {})) - assert isinstance(out.input, Image.Image) - assert np.array_equal(np.asarray(out.input), _image()[:, ::-1]) - - def test_boxes_mode_mirrors_coordinates(self) -> None: - sample = _detection_sample() - op = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0), target="boxes") - out = list(Flux(source=[sample], ops=[op]))[0] - width = _image().shape[1] - x0, _, x1, _ = sample.target["boxes"][0].tolist() - assert type(out.target["boxes"]) is torch.Tensor # BoundingBoxes subclass stripped - assert out.target["boxes"].dtype == torch.float32 - assert out.target["labels"].dtype == torch.int64 - assert out.target["boxes"][0].tolist() == [width - x1, 1.0, width - x0, 3.0] - - def test_boxes_mode_requires_detection_dict(self) -> None: - op = TorchvisionTransformOp(self.v2.RandomHorizontalFlip(p=1.0), target="boxes") - with pytest.raises(TypeError, match="detection"): - op(Sample(_image(), "not-a-dict", {})) - - def test_module_imports_without_torchvision(self) -> None: - # The ADAPTER module is entry-pointed for discovery, so importing it must NOT - # pull in torchvision (all library imports are lazy, inside __call__). The - # torchvision_transforms module deliberately DOES import it (generation), with a - # guarded fallback to zero ops when it is absent. - code = "import sys; import sampleflux.ops.torchvision; assert 'torchvision' not in sys.modules" - subprocess.run([sys.executable, "-c", code], check=True, cwd=str(Path(__file__).resolve().parents[1])) - - -class TestGeneratedTorchvisionOps: - v2 = pytest.importorskip("torchvision.transforms.v2") - - def test_family_generated(self) -> None: - import sampleflux.ops.torchvision_transforms as tvt - - assert len(tvt.__all__) > 30 - assert "TvRandomHorizontalFlip" in tvt.__all__ - assert "TvCompose" not in tvt.__all__ # containers excluded — chaining is native - - def test_flip_parity_and_marks(self) -> None: - import sampleflux.ops.torchvision_transforms as tvt - - cls = tvt.TvRandomHorizontalFlip - assert cls.__confluid_category__ == "op" - assert cls.__confluid_group__ == "augment/torchvision" - assert cls.__confluid_random__ is True - assert "__call__" in vars(cls) - out = list(Flux(source=[_sample()], ops=[cls(p=1.0, target="mask")]))[0] - assert np.array_equal(out.input.permute(1, 2, 0).numpy(), _image()[:, ::-1]) - assert np.array_equal(out.target.numpy(), _mask()[:, ::-1]) - - def test_short_name_yaml_loads(self) -> None: - import sampleflux.ops.torchvision_transforms # noqa: F401 (registers the Tv* names) - - op = confluid.load("!class:TvRandomHorizontalFlip\np: 1.0\ntarget: mask\n") - out = op(_sample()) - assert np.array_equal(out.target.numpy(), _mask()[:, ::-1]) diff --git a/tests/test_bag_interop.py b/tests/test_bag_interop.py deleted file mode 100644 index 71b06a7..0000000 --- a/tests/test_bag_interop.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Legacy ``Sample`` <-> ``TypedSample`` bridge — lossless round-trip, builder path, errors. - -Uses the modality-neutral core items plus a small test-local data-bearing wrapper (the shape a -signal item takes) so the wrapper round-trip is covered without importing a domain package. -""" - -from dataclasses import dataclass - -import numpy as np -import pytest - -from sampleflux.bag.interop import ENCODE_KEY, to_legacy, to_typed -from sampleflux.bag.items import Image, Label, Regions, register_item -from sampleflux.bag.sample import TypedSample -from sampleflux.sample import Sample - - -@register_item -@dataclass -class _WrapBlob: - data: object = None - tag: str = "x" - - -def _sample() -> TypedSample: - return TypedSample( - { - "image": Image(np.arange(48, dtype=np.float32).reshape(4, 4, 3), layout="HWC"), - "blob": _WrapBlob(np.arange(16, dtype=np.float32), tag="sig"), - "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(4, 8)), - "class": Label("drone_x", classes=["noise", "drone_x"]), - }, - roles={"regions": "target", "class": "target"}, - ) - - -class TestRoundTrip: - def test_lossless(self) -> None: - s = _sample() - assert to_typed(to_legacy(s)) == s - - def test_legacy_exposes_input_target(self) -> None: - legacy = to_legacy(_sample()) - assert isinstance(legacy, Sample) - assert np.asarray(legacy.input).shape == (4, 4, 3) # first input field payload (the image) - assert ENCODE_KEY in legacy.meta - - def test_reconstructs_item_types_and_meta(self) -> None: - back = to_typed(to_legacy(_sample())) - assert isinstance(back["image"], Image) and back["image"].layout == "HWC" - assert isinstance(back["blob"], _WrapBlob) and back["blob"].tag == "sig" - assert isinstance(back["regions"], Regions) and back["regions"].canvas == (4, 8) - assert back.role_of("class") == "target" - - def test_no_input_or_target(self) -> None: - s = TypedSample({"aux": _WrapBlob(np.ones(4))}, roles={"aux": "aux"}) - legacy = to_legacy(s) - assert legacy.input is None and legacy.target is None - assert to_typed(legacy) == s - - -class TestBuilderPath: - def test_builder_used_when_no_encoding(self) -> None: - raw = Sample(input=np.zeros((4, 4, 3)), target="cat", metadata={}) - - def builder(sample: Sample) -> TypedSample: - return TypedSample( - {"image": Image(sample.input), "class": Label(sample.target)}, - roles={"class": "target"}, - ) - - typed = to_typed(raw, builder=builder) - assert isinstance(typed["image"], Image) and typed["class"].value == "cat" - - def test_no_encoding_no_builder_raises(self) -> None: - with pytest.raises(ValueError, match="no embedded typed encoding"): - to_typed(Sample(input=np.zeros(3), target=None, metadata={})) diff --git a/tests/test_bag_io.py b/tests/test_bag_io.py index 33368f1..b7d9e34 100644 --- a/tests/test_bag_io.py +++ b/tests/test_bag_io.py @@ -10,7 +10,7 @@ Image, Label, Regions, - TypedSample, + Sample, decode_item, decode_sample, encode_item, @@ -78,7 +78,7 @@ def __init__(self, values: list) -> None: class TestSampleCodec: def test_sample_round_trip_fields_roles_order(self) -> None: - s = TypedSample( + s = Sample( { "image": Image(np.zeros((2, 2, 3), dtype=np.float32)), "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), diff --git a/tests/test_bag_pipeline.py b/tests/test_bag_pipeline.py index 009b0db..997dfdb 100644 --- a/tests/test_bag_pipeline.py +++ b/tests/test_bag_pipeline.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from sampleflux.bag import Image, Label, Mask, Pipeline, Regions, TypedSample +from sampleflux.bag import Image, Label, Mask, Pipeline, Regions, Sample from tests._bag_fixtures import FixtureFlip @@ -25,7 +25,7 @@ class TestTorchvisionAdapter: def test_normalize_touches_only_image(self) -> None: from sampleflux.bag.adapters import TorchvisionV2Adapter - s = TypedSample( + s = Sample( {"image": Image(np.ones((4, 5, 3), dtype=np.float32)), "class": Label("x")}, roles={"class": "target"}, ) @@ -38,7 +38,7 @@ def test_normalize_touches_only_image(self) -> None: def test_flip_moves_image_mask_boxes_together(self) -> None: from sampleflux.bag.adapters import TorchvisionV2Adapter - s = TypedSample( + s = Sample( { "image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32)), "mask": Mask(np.arange(6 * 8).reshape(6, 8).astype(np.int64)), @@ -54,12 +54,12 @@ def test_missing_transform_raises(self) -> None: from sampleflux.bag.adapters import TorchvisionV2Adapter with pytest.raises(ValueError, match="must be set"): - TorchvisionV2Adapter()(TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) + TorchvisionV2Adapter()(Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) def test_no_handled_field_is_noop(self) -> None: from sampleflux.bag.adapters import TorchvisionV2Adapter - s = TypedSample({"class": Label("x")}) + s = Sample({"class": Label("x")}) assert TorchvisionV2Adapter(self.v2.RandomHorizontalFlip(p=1.0))(s) == s @@ -69,7 +69,7 @@ def test_gaussnoise_touches_only_image(self) -> None: from sampleflux.bag.adapters import AlbumentationsAdapter - s = TypedSample( + s = Sample( {"image": Image(np.full((6, 6, 3), 0.5, dtype=np.float32)), "class": Label("x")}, roles={"class": "target"}, ) @@ -83,7 +83,7 @@ def test_bboxes_wrapped_and_returned(self) -> None: from sampleflux.bag.adapters import AlbumentationsAdapter - s = TypedSample( + s = Sample( { "image": Image(np.random.rand(10, 12, 3).astype(np.float32)), "regions": Regions(boxes=[[2, 3, 6, 7]], labels=[1], canvas=(10, 12)), @@ -97,7 +97,7 @@ def test_missing_transform_raises(self) -> None: from sampleflux.bag.adapters import AlbumentationsAdapter with pytest.raises(ValueError, match="must be set"): - AlbumentationsAdapter()(TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) + AlbumentationsAdapter()(Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) class TestMixedPipeline: @@ -111,7 +111,7 @@ def test_cross_library_bare_transforms(self) -> None: import albumentations as A rng = np.random.default_rng(0) - sample = TypedSample( + sample = Sample( { "image": Image(rng.random((16, 20, 3)).astype(np.float32)), "mask": Mask(rng.random((16, 20)) > 0.5), @@ -180,7 +180,7 @@ def factory(_obj: object) -> Transform: register_adapter(lambda o: isinstance(o, MyLibDouble), factory) - s = TypedSample({"image": Image(np.ones((2, 2, 3)))}) + s = Sample({"image": Image(np.ones((2, 2, 3)))}) out = Pipeline([MyLibDouble()])(s) assert np.allclose(np.asarray(out["image"]), 2.0) assert isinstance(coerce_transform(MyLibDouble()), FunctionTransform) diff --git a/tests/test_bag_sample.py b/tests/test_bag_sample.py index f8738ec..0395d03 100644 --- a/tests/test_bag_sample.py +++ b/tests/test_bag_sample.py @@ -1,14 +1,14 @@ -"""``TypedSample`` — role tags, views, copy-on-write mutators, array-safe equality.""" +"""``Sample`` — role tags, views, copy-on-write mutators, array-safe equality.""" import numpy as np import pytest from sampleflux.bag.items import Image, Label, Regions -from sampleflux.bag.sample import ROLES, TypedSample +from sampleflux.bag.sample import ROLES, Sample -def _sample() -> TypedSample: - return TypedSample( +def _sample() -> Sample: + return Sample( {"image": Image(np.ones(4)), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")}, roles={"regions": "target", "class": "target"}, ) @@ -16,7 +16,7 @@ def _sample() -> TypedSample: class TestRolesAndViews: def test_default_role_is_input(self) -> None: - s = TypedSample({"a": Image(np.zeros((1, 1, 3))), "b": Label()}) + s = Sample({"a": Image(np.zeros((1, 1, 3))), "b": Label()}) assert s.roles == {"a": "input", "b": "input"} def test_inputs_targets_aux(self) -> None: @@ -42,11 +42,11 @@ def test_roles_closed_set(self) -> None: class TestConstruction: def test_role_for_unknown_field_raises(self) -> None: with pytest.raises(KeyError, match="unknown field"): - TypedSample({"a": Label()}, roles={"b": "target"}) + Sample({"a": Label()}, roles={"b": "target"}) def test_invalid_role_raises(self) -> None: with pytest.raises(ValueError, match="invalid role"): - TypedSample({"a": Label()}, roles={"a": "output"}) # type: ignore[dict-item] + Sample({"a": Label()}, roles={"a": "output"}) # type: ignore[dict-item] class TestCopyOnWrite: @@ -93,22 +93,22 @@ def test_fields_and_roles_are_copies(self) -> None: class TestEquality: def test_equal_with_array_fields(self) -> None: - a = TypedSample({"img": Image(np.zeros((2, 2, 3)))}) - b = TypedSample({"img": Image(np.zeros((2, 2, 3)))}) + a = Sample({"img": Image(np.zeros((2, 2, 3)))}) + b = Sample({"img": Image(np.zeros((2, 2, 3)))}) assert a == b def test_unequal_arrays(self) -> None: - a = TypedSample({"img": Image(np.zeros((2, 2, 3)))}) - b = TypedSample({"img": Image(np.ones((2, 2, 3)))}) + a = Sample({"img": Image(np.zeros((2, 2, 3)))}) + b = Sample({"img": Image(np.ones((2, 2, 3)))}) assert a != b def test_unequal_roles_or_keys(self) -> None: - a = TypedSample({"x": Label("v")}) - assert a != TypedSample({"x": Label("v")}, roles={"x": "target"}) - assert a != TypedSample({"y": Label("v")}) + a = Sample({"x": Label("v")}) + assert a != Sample({"x": Label("v")}, roles={"x": "target"}) + assert a != Sample({"y": Label("v")}) def test_not_a_sample(self) -> None: - assert (TypedSample({"x": Label()}) == 5) is False + assert (Sample({"x": Label()}) == 5) is False def test_repr(self) -> None: assert "Image[input]" in repr(_sample()) diff --git a/tests/test_bag_transform.py b/tests/test_bag_transform.py index e23196a..819a8a7 100644 --- a/tests/test_bag_transform.py +++ b/tests/test_bag_transform.py @@ -7,12 +7,12 @@ import numpy as np import pytest -from sampleflux import Image, Label, Mask, Pipeline, Regions, Transform, TypedSample, as_transform +from sampleflux import Image, Label, Mask, Pipeline, Regions, Sample, Transform, as_transform from tests._bag_fixtures import FixtureFlip -def _seg() -> TypedSample: - return TypedSample( +def _seg() -> Sample: + return Sample( { "image": Image(np.arange(8 * 10 * 3).reshape(8, 10, 3).astype(np.float32)), "mask": Mask(np.arange(8 * 10).reshape(8, 10)), @@ -43,16 +43,16 @@ def test_only_filter(self) -> None: assert out["regions"].boxes == [[1, 1, 4, 4]] # regions skipped def test_image_layout_chw(self) -> None: - s = TypedSample({"image": Image(np.arange(3 * 4 * 5).reshape(3, 4, 5), layout="CHW")}) + s = Sample({"image": Image(np.arange(3 * 4 * 5).reshape(3, 4, 5), layout="CHW")}) out = FixtureFlip(p=1.0)(s) assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, :, ::-1]) def test_regions_uses_canvas_without_image(self) -> None: - s = TypedSample({"regions": Regions(boxes=[[2, 0, 5, 3]], canvas=(8, 10))}) + s = Sample({"regions": Regions(boxes=[[2, 0, 5, 3]], canvas=(8, 10))}) assert FixtureFlip(p=1.0)(s)["regions"].boxes == [[5, 0, 8, 3]] def test_regions_without_reference_width_raises(self) -> None: - s = TypedSample({"regions": Regions(boxes=[[2, 0, 5, 3]])}) # no image, no canvas + s = Sample({"regions": Regions(boxes=[[2, 0, 5, 3]])}) # no image, no canvas with pytest.raises(ValueError, match="no reference width"): FixtureFlip(p=1.0)(s) @@ -83,13 +83,13 @@ def test_v2_flip_matches_native_fixture(self) -> None: class TestPipelineAndFunction: def test_pipeline_is_sequential(self) -> None: - s = TypedSample({"x": Image(np.ones((2, 2, 3), dtype=np.float32))}) + s = Sample({"x": Image(np.ones((2, 2, 3), dtype=np.float32))}) double = as_transform(lambda d: d * 2, handles=(Image,)) out = Pipeline([double, double])(s) assert np.allclose(np.asarray(out["x"]), 4.0) def test_function_transform_only_filter(self) -> None: - s = TypedSample({"a": Image(np.ones((2, 2, 3))), "b": Image(np.ones((2, 2, 3)))}) + s = Sample({"a": Image(np.ones((2, 2, 3))), "b": Image(np.ones((2, 2, 3)))}) out = as_transform(lambda d: d + 1, handles=(Image,), only=["a"])(s) assert np.allclose(np.asarray(out["a"]), 2.0) and np.allclose(np.asarray(out["b"]), 1.0) @@ -100,12 +100,12 @@ def test_pipeline_repr(self) -> None: class TestBaseTransform: def test_default_get_params_and_passthrough(self) -> None: # A transform with no kernels leaves every field alone. - s = TypedSample({"x": Label("v")}) + s = Sample({"x": Label("v")}) assert Transform()(s) == s def test_decode_not_implemented(self) -> None: with pytest.raises(NotImplementedError, match="no decode"): - FixtureFlip().decode(TypedSample({"image": Image(np.zeros((2, 2, 3)))})) + FixtureFlip().decode(Sample({"image": Image(np.zeros((2, 2, 3)))})) def test_zero_arg_construction(self) -> None: assert FixtureFlip().p == 0.5 and Transform().only is None diff --git a/tests/test_categories.py b/tests/test_categories.py index 25d7e3d..0ee26c6 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -1,4 +1,4 @@ -# mypy: disable-error-code="attr-defined" +# mypy: disable-error-code="attr-defined,union-attr" """Discovery-category coverage for sampleflux ``@configurable`` classes. These ``category=`` tags drive navigaitor's ``list_configurable_classes(category=...)`` @@ -12,24 +12,22 @@ from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp from sampleflux.ops.albumentations import AlbumentationsOp from sampleflux.ops.configure import ConfigureOp -from sampleflux.ops.copy import CopyInputOp from sampleflux.ops.debug import PrintSampleOp from sampleflux.ops.enable import Enable from sampleflux.ops.formula import FormulaOp -from sampleflux.ops.image import ConvertToImageOp, NormalizeToUint8Op -from sampleflux.ops.metadata import DropMetadataOp -from sampleflux.ops.numpy import RescaleOp, StandardizeOp, ThresholdOp +from sampleflux.ops.image import ConvertToImage +from sampleflux.ops.numpy import ConnectedComponents, Threshold from sampleflux.ops.parallel import Parallel from sampleflux.ops.sink import SampleSinkOp -from sampleflux.ops.stash import StashTargetOp, UnstashTargetOp +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole from sampleflux.ops.target import ( - CocoToTorchVisionDetectionOp, - DecodeTargetOp, - EncodeTargetOp, - MasksToDetectionBoxesOp, - MetadataToTargetOp, + CocoToTorchVisionDetection, + DecodeTarget, + EncodeTarget, + MasksToDetectionBoxes, + MetadataToTarget, ) -from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.torch import ToTensor from sampleflux.ops.torchvision import TorchvisionTransformOp from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource @@ -39,34 +37,18 @@ def test_engine_classes_tagged() -> None: - """The generic, task-agnostic *engines* — composition primitives that compose sources + ops. - - ``Flux`` / ``JointFlux`` carry ``category="engine"``. They (and ``DatasetSplit``, now a - ``source``) are canvas-composable in FluxStudio: its allowlist now includes ``engine`` and the - source-typed constructor params (``source`` / ``fluxes`` / ``ops``) render as wired sockets.""" assert Flux.__confluid_category__ == "engine" assert JointFlux.__confluid_category__ == "engine" def test_raw_callable_wrappers_uncategorised() -> None: - """``FilterOp`` / ``WrappedOp`` wrap a *raw Python callable*, so they are neither an ``op`` - (nothing to wire) nor an ``engine`` — they carry NO category (bare ``@configurable``) and are - excluded from FluxStudio by being uncategorised, like a module-level helper function. So even - once ``engine`` is added to the allowlist these wrappers stay out (correct — they're not buildable).""" assert getattr(FilterOp, "__confluid_category__", None) is None assert getattr(WrappedOp, "__confluid_category__", None) is None - # Still registered/configurable, just untagged. assert FilterOp.__confluid_configurable__ is True assert WrappedOp.__confluid_configurable__ is True def test_source_classes_tagged() -> None: - """``HuggingFaceSource`` is a concrete data *source* (it loads a dataset). - - ``DatasetSplit`` / ``RangeSource`` / ``ConcatSource`` are also ``source``s: they yield - ``Sample``s and are wired into a trainer's ``source:`` slot, each exposing a derived *view* - of other source(s) (split / contiguous slice / concatenation) — they apply no ops, so they - are sources, not engines.""" assert HuggingFaceSource.__confluid_category__ == "source" assert DatasetSplit.__confluid_category__ == "source" assert RangeSource.__confluid_category__ == "source" @@ -74,131 +56,111 @@ def test_source_classes_tagged() -> None: def test_op_classes_tagged() -> None: - """Concrete ``Sample → Sample`` ops carry ``category="op"`` (the FluxStudio op-node allowlist).""" - assert RescaleOp.__confluid_category__ == "op" - assert StandardizeOp.__confluid_category__ == "op" - assert ThresholdOp.__confluid_category__ == "op" - assert Enable.__confluid_category__ == "op" - assert TransformChain.__confluid_category__ == "op" - assert SampleSinkOp.__confluid_category__ == "op" - assert MetadataToTargetOp.__confluid_category__ == "op" - assert EncodeTargetOp.__confluid_category__ == "op" - assert DecodeTargetOp.__confluid_category__ == "op" - assert CocoToTorchVisionDetectionOp.__confluid_category__ == "op" - assert MasksToDetectionBoxesOp.__confluid_category__ == "op" - assert ConfigureOp.__confluid_category__ == "op" - assert FormulaOp.__confluid_category__ == "op" - assert AlbumentationsOp.__confluid_category__ == "op" - assert TorchvisionTransformOp.__confluid_category__ == "op" + for cls in ( + Threshold, + ConnectedComponents, + ToTensor, + ConvertToImage, + Enable, + TransformChain, + Parallel, + SampleSinkOp, + MetadataToTarget, + EncodeTarget, + DecodeTarget, + CocoToTorchVisionDetection, + MasksToDetectionBoxes, + ConfigureOp, + FormulaOp, + AlbumentationsOp, + TorchvisionTransformOp, + SetRole, + RenameField, + DropField, + CopyField, + SelectFields, + PrintSampleOp, + ): + assert cls.__confluid_category__ == "op", cls.__name__ def test_augmentation_adapters_random_tagged() -> None: - """The library-augmentation adapters are stochastic (the wrapped library draws its own - random parameters per call), so they carry ``random=True`` — the confluid mark UIs use - to inject cache-busting (e.g. FluxStudio's ``IS_CHANGED``).""" assert AlbumentationsOp.__confluid_random__ is True assert TorchvisionTransformOp.__confluid_random__ is True def test_storage_sink_classes_tagged() -> None: - """The SampleFlux storage SINKS carry ``category="sink"`` so FluxStudio surfaces them as - ``DatasetProcessor`` sink nodes (``SAMPLEFLUX_OBJECT:sink``). Their matching SOURCES stay - UNcategorised — they read a sink's layout back via YAML ``!class:``, they are not canvas nodes. - (``SampleSinkOp`` is the op-FORM sink, ``category="op"`` — a different thing, asserted above.)""" assert HDF5Sink.__confluid_category__ == "sink" assert ZarrGroupSink.__confluid_category__ == "sink" assert ZarrBatchSink.__confluid_category__ == "sink" assert DirectorySink.__confluid_category__ == "sink" - # The matching source is NOT tagged, so the positive allowlist surfaces only the sink half. assert getattr(HDF5Source, "__confluid_category__", None) is None def test_op_group_tags() -> None: - """Ops carry a path-like ``group`` (FluxStudio palette nesting: Taidal/SampleFlux/Op/). - - Presentation-only — orthogonal to the category that gates discovery. A renamed/dropped group - re-files the node in the palette but never hides it; pinned so the taxonomy is a regression gate.""" - assert RescaleOp.__confluid_group__ == "numpy" - assert StandardizeOp.__confluid_group__ == "numpy" - assert ThresholdOp.__confluid_group__ == "numpy" - assert ToTensorOp.__confluid_group__ == "torch" - assert CopyInputOp.__confluid_group__ == "structure" - assert DropMetadataOp.__confluid_group__ == "structure" + assert Threshold.__confluid_group__ == "numpy" + assert ConnectedComponents.__confluid_group__ == "numpy" + assert ToTensor.__confluid_group__ == "torch" + assert ConvertToImage.__confluid_group__ == "image" + assert SetRole.__confluid_group__ == "structure" + assert SelectFields.__confluid_group__ == "structure" assert PrintSampleOp.__confluid_group__ == "debug" - assert StashTargetOp.__confluid_group__ == "structure" - assert UnstashTargetOp.__confluid_group__ == "structure" - assert MetadataToTargetOp.__confluid_group__ == "structure" - assert EncodeTargetOp.__confluid_group__ == "structure" - assert DecodeTargetOp.__confluid_group__ == "structure" - assert CocoToTorchVisionDetectionOp.__confluid_group__ == "structure" - assert MasksToDetectionBoxesOp.__confluid_group__ == "structure" + assert MetadataToTarget.__confluid_group__ == "structure" + assert EncodeTarget.__confluid_group__ == "structure" + assert DecodeTarget.__confluid_group__ == "structure" + assert CocoToTorchVisionDetection.__confluid_group__ == "structure" + assert MasksToDetectionBoxes.__confluid_group__ == "structure" assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" assert TransformChain.__confluid_group__ == "compose" assert ConfigureOp.__confluid_group__ == "compose" assert FormulaOp.__confluid_group__ == "compose" - assert ConvertToImageOp.__confluid_group__ == "image" - assert NormalizeToUint8Op.__confluid_group__ == "image" assert SampleSinkOp.__confluid_group__ == "sink" assert AlbumentationsOp.__confluid_group__ == "augment" assert TorchvisionTransformOp.__confluid_group__ == "augment" def test_categories_enumerable_via_registry() -> None: - """Importing the classes registers them; the category index must surface them. - - The navigaitor picker queries ``list_classes(category=...)``, so the index — - not just the class attribute — has to carry the tag. - """ registry = get_registry() assert {"Flux", "JointFlux"} <= registry.list_classes(category="engine") - # DatasetSplit is a source now, not an engine. assert "DatasetSplit" not in registry.list_classes(category="engine") - # FilterOp / WrappedOp are uncategorised, so they appear in NO category index. assert not ({"FilterOp", "WrappedOp"} & registry.list_classes(category="engine")) assert {"HuggingFaceSource", "DatasetSplit", "RangeSource", "ConcatSource"} <= registry.list_classes( category="source" ) assert { - "RescaleOp", - "StandardizeOp", - "ThresholdOp", + "Threshold", + "ConnectedComponents", + "ToTensor", + "ConvertToImage", "Enable", "SampleSinkOp", - "MetadataToTargetOp", - "EncodeTargetOp", - "DecodeTargetOp", - "CocoToTorchVisionDetectionOp", - "MasksToDetectionBoxesOp", + "MetadataToTarget", + "EncodeTarget", + "DecodeTarget", + "CocoToTorchVisionDetection", + "MasksToDetectionBoxes", "TransformChain", } <= registry.list_classes(category="op") - # The storage sinks surface under the NEW "sink" category index (FluxStudio's allowlist + the - # navigaitor sink picker). SampleSinkOp is category="op", so it is NOT here. assert {"HDF5Sink", "ZarrGroupSink", "ZarrBatchSink", "DirectorySink"} <= registry.list_classes(category="sink") assert "SampleSinkOp" not in registry.list_classes(category="sink") def test_groups_enumerable_via_registry() -> None: - """The registry's group index must surface the tagged ops (``list_classes(group=...)``).""" registry = get_registry() - assert { - "RescaleOp", - "StandardizeOp", - "ThresholdOp", - } <= registry.list_classes(group="numpy") - # The FFT ops exist in BOTH framework groups (a numpy + a torch variant under the one name, - # exactly like RescaleOp/StandardizeOp), so they surface under the torch group too. - assert {"ToTensorOp"} <= registry.list_classes(group="torch") - assert {"ConvertToImageOp", "NormalizeToUint8Op"} <= registry.list_classes(group="image") + assert {"Threshold", "ConnectedComponents"} <= registry.list_classes(group="numpy") + assert {"ToTensor"} <= registry.list_classes(group="torch") + assert {"ConvertToImage"} <= registry.list_classes(group="image") assert {"Parallel", "Enable", "TransformChain"} <= registry.list_classes(group="compose") assert {"SampleSinkOp"} <= registry.list_classes(group="sink") assert { - "MetadataToTargetOp", - "EncodeTargetOp", - "DecodeTargetOp", - "CocoToTorchVisionDetectionOp", - "MasksToDetectionBoxesOp", + "MetadataToTarget", + "EncodeTarget", + "DecodeTarget", + "CocoToTorchVisionDetection", + "MasksToDetectionBoxes", + "SetRole", + "SelectFields", } <= registry.list_classes(group="structure") assert {"AlbumentationsOp", "TorchvisionTransformOp"} <= registry.list_classes(group="augment") - # group × category intersect, like task × role. assert "TransformChain" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_context.py b/tests/test_context.py deleted file mode 100644 index aaf4168..0000000 --- a/tests/test_context.py +++ /dev/null @@ -1,442 +0,0 @@ -"""Tests for the per-sample Context (`sampleflux.context`) and the context ops -(`sampleflux.ops.context`: Save / Use / Drop / Apply / Capture / Mix). - -Covers the Phase-2 contract of the graph execution model: -- a flat op list containing context ops executes a fan-out/fan-in graph on the plain - Flux engine (sequential, spawn-parallel, streamed, and random-access routes); -- Context never touches ``sample.metadata`` (the metadata-untouched invariant); -- copy-vs-move semantics mirror the stash family (`Use` deep-copies unless it drops); -- every context op round-trips through confluid dump/load (Pipeline Parity). -""" - -from pathlib import Path - -import numpy as np -import pytest -from confluid import configurable, dump, load, materialize, output - -from sampleflux.context import Context, activate, current, require -from sampleflux.core import Flux -from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use -from sampleflux.ops.swap import SwapInputTargetOp -from sampleflux.sample import Sample - -# --------------------------------------------------------------------------- -# Test ops (module-level so they pickle for the spawn route) -# --------------------------------------------------------------------------- - - -@configurable -class AddOp: - """Add a constant to the (numeric or array) input. - - Args: - amount: Value added to ``sample.input`` on every call. - """ - - def __init__(self, amount: float = 1.0) -> None: - self.amount = amount - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + self.amount) - - -@configurable -class ScaleOp: - """Multiply the input by a factor. - - Args: - factor: Multiplier applied to ``sample.input`` on every call. - """ - - def __init__(self, factor: float = 2.0) -> None: - self.factor = factor - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input * self.factor) - - -@configurable -class StampOp: - """Write one metadata key (tests branch-metadata survival through Mix). - - Args: - key: Metadata key to write. - value: Value written under ``key``. - """ - - def __init__(self, key: str = "stamp", value: str = "x") -> None: - self.key = key - self.value = value - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(metadata={**sample.meta, self.key: self.value}) - - -@configurable -class DrawOp: - """Pass-through op with a stochastic-style @output (captures must read the real run).""" - - def __init__(self) -> None: - self._last: float = 0.0 - self._calls: int = 0 - - @property - @output - def drawn(self) -> float: - """The value produced by the last application.""" - return self._last - - def __call__(self, sample: Sample) -> Sample: - self._calls += 1 - self._last = float(sample.input) * 10.0 + self._calls - return sample - - -@configurable -class MutateInPlaceOp: - """Deliberately mutate the input array IN PLACE (isolation tests).""" - - def __call__(self, sample: Sample) -> Sample: - sample.input[0] = -999.0 - return sample - - -def _samples(n: int = 3) -> list: - return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] - - -# --------------------------------------------------------------------------- -# Context core -# --------------------------------------------------------------------------- - - -class TestContext: - def test_put_get_delete_live(self) -> None: - ctx = Context() - ctx.put("a", 1) - ctx.put("b", 2) - assert ctx.get("a") == 1 - assert ctx.live() == ("a", "b") - assert "a" in ctx and len(ctx) == 2 - ctx.delete("a") - assert ctx.live() == ("b",) - - def test_get_missing_is_actionable(self) -> None: - with pytest.raises(KeyError, match="live cells"): - Context().get("nope") - - def test_delete_missing_is_actionable(self) -> None: - with pytest.raises(KeyError, match="missing cell"): - Context().delete("nope") - - def test_copy_is_shallow_with_independent_cell_set(self) -> None: - ctx = Context() - payload = [1, 2] - ctx.put("a", payload) - clone = ctx.copy() - clone.delete("a") - assert "a" in ctx # independent cell set - assert ctx.get("a") is payload # shared values (shallow) - - def test_activate_sets_and_resets(self) -> None: - assert current() is None - ctx = Context() - with activate(ctx): - assert current() is ctx - assert current() is None - - def test_require_outside_engine_is_actionable(self) -> None: - with pytest.raises(RuntimeError, match="no active Context"): - require("Save") - - -# --------------------------------------------------------------------------- -# Context ops — unit behavior -# --------------------------------------------------------------------------- - - -class TestContextOps: - def test_save_requires_name(self) -> None: - with activate(Context()): - with pytest.raises(ValueError, match="'name'"): - Save()(Sample(1)) - - def test_use_requires_name(self) -> None: - with activate(Context()): - with pytest.raises(ValueError, match="'name'"): - Use()(Sample(1)) - - def test_save_then_use_copies_by_default(self) -> None: - s = Sample(input=np.array([1.0, 2.0]), metadata={"m": 1}) - with activate(Context()) as ctx: - Save(name="cell")(s) - restored = Use(name="cell")(Sample(input=None)) - assert restored.input is not s.input # deep copy - np.testing.assert_array_equal(restored.input, s.input) - assert "cell" in ctx # kept - - def test_use_with_drop_moves_without_copy(self) -> None: - s = Sample(input=np.array([1.0, 2.0])) - with activate(Context()) as ctx: - Save(name="cell")(s) - restored = Use(name="cell", drop=True)(Sample(input=None)) - assert restored.input is s.input # move: no copy - assert "cell" not in ctx # freed - - def test_two_readers_are_isolated_against_inplace_mutation(self) -> None: - s = Sample(input=np.array([1.0, 2.0])) - with activate(Context()): - Save(name="fork")(s) - branch_a = Use(name="fork")(Sample(input=None)) - MutateInPlaceOp()(branch_a) # mutates branch A's copy in place - branch_b = Use(name="fork", drop=True)(Sample(input=None)) - assert branch_b.input[0] == 1.0 # untouched by branch A - - def test_use_coerces_raw_cell_value(self) -> None: - with activate(Context()) as ctx: - ctx.put("raw", 42.0) - restored = Use(name="raw", drop=True)(Sample(input=None)) - assert restored.input == 42.0 - - def test_drop_frees_cells_and_flags_liveness_bugs(self) -> None: - with activate(Context()) as ctx: - ctx.put("a", 1) - ctx.put("b", 2) - Drop(names=["a", "b"])(Sample(1)) - assert ctx.live() == () - with pytest.raises(KeyError, match="missing cell"): - Drop(names=["a"])(Sample(1)) - - def test_drop_empty_is_noop_without_context(self) -> None: - # No names -> never needs the Context (works outside an engine too). - assert Drop()(Sample(1)).input == 1 - - def test_apply_sets_param_from_sample_cell_input(self) -> None: - with activate(Context()) as ctx: - ctx.put("thresh", Sample(input=5.0)) - op = Apply(op=AddOp(amount=0.0), param="amount", source="thresh", drop=True) - result = op(Sample(input=1.0)) - assert result is not None and result.input == 6.0 - assert "thresh" not in ctx - - def test_apply_sets_param_from_raw_cell_value(self) -> None: - with activate(Context()) as ctx: - ctx.put("factor", 3.0) - result = Apply(op=ScaleOp(), param="factor", source="factor")(Sample(input=2.0)) - assert result is not None and result.input == 6.0 - - def test_apply_validations(self) -> None: - with activate(Context()): - with pytest.raises(ValueError, match="'op'"): - Apply(param="p", source="s")(Sample(1)) - with pytest.raises(ValueError, match="'param'"): - Apply(op=AddOp(), source="s")(Sample(1)) - with pytest.raises(ValueError, match="'source'"): - Apply(op=AddOp(), param="p")(Sample(1)) - - def test_capture_records_live_output_into_cell(self) -> None: - with activate(Context()) as ctx: - draw = DrawOp() - result = Capture(op=draw, output="drawn", name="snr")(Sample(input=2.0)) - assert result is not None - assert ctx.get("snr") == 21.0 # 2*10 + 1st call — the REAL run's value - # A second application overwrites with the fresh draw (stochastic-correct). - Capture(op=draw, output="drawn", name="snr")(Sample(input=2.0)) - assert ctx.get("snr") == 22.0 - - def test_capture_reads_through_apply_wrapper(self) -> None: - with activate(Context()) as ctx: - ctx.put("noop", 0.0) - inner = DrawOp() - wrapped = Apply(op=inner, param="_unused", source="noop") - Capture(op=wrapped, output="drawn", name="d")(Sample(input=1.0)) - assert ctx.get("d") == 11.0 - - def test_capture_missing_output_is_actionable(self) -> None: - with activate(Context()): - with pytest.raises(AttributeError, match="has no @output attribute"): - Capture(op=AddOp(), output="nope")(Sample(1.0)) - - def test_capture_then_apply_wires_output_to_param(self) -> None: - with activate(Context()): - Capture(op=DrawOp(), output="drawn", name="d")(Sample(input=1.0)) - result = Apply(op=AddOp(amount=0.0), param="amount", source="d", drop=True)(Sample(input=0.5)) - assert result is not None and result.input == 0.5 + 11.0 - - def test_mix_slots_and_metadata_merge_order(self) -> None: - with activate(Context()) as ctx: - ctx.put("a", Sample(input="A", target="tA", metadata={"who": "a", "a_only": 1})) - ctx.put("b", Sample(input="B", target="tB", metadata={"who": "b", "b_only": 2})) - incoming = Sample(input="in", target="t_in", metadata={"who": "incoming", "in_only": 0}) - mixed = Mix(input_from="a", target_from="b", drop=["a", "b"])(incoming) - assert mixed is not None - assert mixed.input == "A" and mixed.target == "tB" - # incoming first, then input_from, then target_from (last write wins) - assert mixed.meta["who"] == "b" - assert mixed.meta["in_only"] == 0 and mixed.meta["a_only"] == 1 and mixed.meta["b_only"] == 2 - assert ctx.live() == () - - def test_mix_metadata_from_wins_last(self) -> None: - with activate(Context()) as ctx: - ctx.put("a", Sample(input="A", metadata={"who": "a"})) - ctx.put("m", Sample(input=None, metadata={"who": "meta"})) - mixed = Mix(input_from="a", metadata_from="m", drop=["a", "m"])(Sample(input="in", metadata={"who": "i"})) - assert mixed is not None and mixed.meta["who"] == "meta" - - def test_mix_metadata_from_accepts_raw_dict_and_rejects_nondict(self) -> None: - with activate(Context()) as ctx: - ctx.put("m", {"k": "v"}) - mixed = Mix(metadata_from="m")(Sample(input="in")) - assert mixed is not None and mixed.meta["k"] == "v" - ctx.put("bad", 3.0) - with pytest.raises(TypeError, match="metadata_from"): - Mix(metadata_from="bad")(Sample(input="in")) - - def test_mix_empty_slots_keep_incoming(self) -> None: - with activate(Context()): - incoming = Sample(input="in", target="t", metadata={"m": 1}) - mixed = Mix()(incoming) - assert mixed is not None - assert mixed.input == "in" and mixed.target == "t" and mixed.meta == {"m": 1} - - def test_ops_outside_engine_raise_actionable(self) -> None: - with pytest.raises(RuntimeError, match="no active Context"): - Save(name="x")(Sample(1)) - - -# --------------------------------------------------------------------------- -# Engine integration — the four execution routes -# --------------------------------------------------------------------------- - -# A fan-out/fan-in graph as a flat op list: -# fork the incoming value; branch A computes value+1 and swaps it into its TARGET slot -# (Mix's target_from reads the cell-sample's target field); branch B computes value*2 on -# the stream; Mix yields input = B's (stream), target = A's (cell). -_GRAPH_OPS = [ - Save(name="fork"), - AddOp(amount=1.0), # branch A rides the stream - SwapInputTargetOp(), # park A's result in the target field for Mix - Save(name="branch_a"), - Use(name="fork", drop=True), # branch B restarts from the fork - ScaleOp(factor=2.0), - Mix(target_from="branch_a", drop=["branch_a"]), # input = B (stream), target = A -] - - -def _expected_graph(values: list) -> list: - return [(v * 2.0, v + 1.0) for v in values] - - -class TestEngineRoutes: - def test_sequential_graph_execution(self) -> None: - flux = Flux(source=_samples(4), ops=list(_GRAPH_OPS)) - got = [(s.input, s.target) for s in flux] - assert _expected_graph([0.0, 1.0, 2.0, 3.0]) == got - - def test_metadata_untouched_invariant(self) -> None: - # Context wiring must not leak anything into sample.metadata. - flux = Flux(source=_samples(3), ops=list(_GRAPH_OPS)) - for i, s in enumerate(flux): - assert s.meta == {"idx": i} - - def test_random_access_getitem(self) -> None: - flux = Flux(source=_samples(5), ops=list(_GRAPH_OPS)) - s = flux[3] - assert s.input == 6.0 - - def test_spawn_parallel_parity(self) -> None: - seq = [s.input for s in Flux(source=_samples(4), ops=list(_GRAPH_OPS))] - par = [s.input for s in Flux(source=_samples(4), ops=list(_GRAPH_OPS)).parallel(2)] - assert seq == par - - def test_streamed_route_with_parallel_op(self) -> None: - from sampleflux.ops.parallel import Parallel - - # Whole graph INSIDE Parallel: each worker's _worker_task provides the Context. - flux = Flux(source=_samples(4), ops=[Parallel(ops=list(_GRAPH_OPS), workers=2)]) - got = [(s.input, s.target) for s in flux] - assert got == _expected_graph([0.0, 1.0, 2.0, 3.0]) - - def test_streamed_route_cells_may_not_cross_stream_boundary(self) -> None: - from sampleflux.ops.parallel import Parallel - - flux = Flux(source=_samples(2), ops=[Save(name="fork"), Parallel(ops=[AddOp()], workers=1)]) - with pytest.raises(RuntimeError, match="stream-level op"): - list(flux) - - def test_streamed_route_context_ops_before_and_after_boundary(self) -> None: - from sampleflux.ops.parallel import Parallel - - # Cells used and FREED before the boundary, new cells after — both legal. - ops = [ - Save(name="pre"), - Use(name="pre", drop=True), - Parallel(ops=[AddOp(amount=1.0)], workers=1), - Save(name="post"), - Mix(target_from="post", drop=["post"]), - ] - results = list(Flux(source=_samples(3), ops=ops)) - assert [s.input for s in results] == [1.0, 2.0, 3.0] - - def test_context_is_fresh_per_sample(self) -> None: - # A cell saved for sample N must never be visible to sample N+1: use a - # drop-less Save; if contexts leaked across samples, Use would see the - # PREVIOUS sample's fork (values would shift) or cells would pile up. - ops = [Save(name="fork"), AddOp(amount=100.0), Use(name="fork")] # no drop - results = list(Flux(source=_samples(3), ops=ops)) - assert [s.input for s in results] == [0.0, 1.0, 2.0] - - -# --------------------------------------------------------------------------- -# Confluid round-trip (Pipeline Parity) + YAML -# --------------------------------------------------------------------------- - - -class TestSerialization: - def test_every_context_op_dump_load_round_trips(self) -> None: - ops = [ - Save(name="fork"), - Use(name="fork", drop=True), - Drop(names=["a", "b"]), - Apply(op=AddOp(amount=2.0), param="amount", source="cell", drop=True), - Capture(op=DrawOp(), output="drawn", name="snr"), - Mix(input_from="a", target_from="b", metadata_from="m", drop=["a"]), - ] - for op in ops: - text = dump(op) - rebuilt = materialize(load(text)) - assert type(rebuilt) is type(op) - for attr, value in vars(op).items(): - if attr.startswith("_") or attr == "op": - continue # nested op compared structurally below - assert getattr(rebuilt, attr) == value, f"{type(op).__name__}.{attr}" - - rebuilt_apply = materialize(load(dump(ops[3]))) - assert type(rebuilt_apply.op).__name__ == "AddOp" and rebuilt_apply.op.amount == 2.0 - - def test_graph_ops_yaml_executes_via_from_ops_yaml(self, tmp_path: Path) -> None: - yaml_text = """ -ops: - - !class:sampleflux.ops.context.Save(name=fork) - - !class:tests.test_context.AddOp(amount=1.0) - - !class:sampleflux.ops.swap.SwapInputTargetOp() - - !class:sampleflux.ops.context.Save(name=branch_a) - - !class:sampleflux.ops.context.Use(name=fork,drop=true) - - !class:tests.test_context.ScaleOp(factor=2.0) - - !class:sampleflux.ops.context.Mix(target_from=branch_a) - drop: [branch_a] -""" - path = tmp_path / "graph_ops.yaml" - path.write_text(yaml_text) - flux = Flux.from_ops_yaml(str(path), source=_samples(3)) - results = list(flux) - assert [s.input for s in results] == [0.0, 2.0, 4.0] - assert [s.target for s in results] == [1.0, 2.0, 3.0] - - def test_linear_pipeline_metadata_byte_identical(self) -> None: - # A straight sequence (no context ops) — Context threading must be invisible. - flux = Flux(source=_samples(3), ops=[AddOp(amount=1.0)]) - for i, s in enumerate(flux): - assert s.meta == {"idx": i} - assert s.input == float(i) + 1.0 diff --git a/tests/test_coverage_gap.py b/tests/test_coverage_gap.py deleted file mode 100644 index be0ff0a..0000000 --- a/tests/test_coverage_gap.py +++ /dev/null @@ -1,189 +0,0 @@ -from pathlib import Path -from typing import Any - -import numpy as np -import pytest - -from sampleflux.core import Flux -from sampleflux.discovery import get_callable_path, resolve_callable -from sampleflux.sample import Sample -from sampleflux.storage.base import Storage -from sampleflux.storage.directory import DirectorySink -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource - - -def test_storage_base_close() -> None: - # hits base.py:36 - s = Storage() - s.close() - - -def test_directory_open_overwrite(tmp_path: Path) -> None: - # hits directory.py:27 (pass) - d = tmp_path / "ovr" - d.mkdir() - sink = DirectorySink(d, overwrite=True) - sink.open() - - -def test_hdf5_none_file() -> None: - # hits hdf5.py:45, 59, 90 - source = HDF5Source("nonexistent.h5") - # Manually ensure _file is None (it is by default) - assert source._file is None - # We need to bypass open() or mock it to stay None - # But __iter__ calls open(). - # Let's mock open to do nothing - source.open = lambda: source # type: ignore - assert list(source) == [] # hits line 45 - assert len(source) == 0 # hits line 59 - - sink = HDF5Sink("nonexistent.h5") - sink.open = lambda: sink # type: ignore - sink.write(Sample(input=1)) # hits line 90 - - -def test_resolve_callable_import_error() -> None: - # hits discovery.py:57 - with pytest.raises(ImportError, match="Cannot resolve"): - resolve_callable("nonexistent_mod_totally:func") - - -def test_get_callable_path_main_no_file(monkeypatch: Any) -> None: - # hits discovery.py:33 (pass) - import sys - import types - - m = types.ModuleType("__main__") - # No __file__ attribute - monkeypatch.setitem(sys.modules, "__main__", m) - - def local_f() -> None: - pass - - local_f.__module__ = "__main__" - # Should fallback to valueerror or just skip the main block if it fails - try: - get_callable_path(local_f) - except ValueError: - pass - - -def test_hdf5_metadata_exception(tmp_path: Path) -> None: - # hits hdf5.py:except branch in metadata loop - h5_path = tmp_path / "meta_err.h5" - sink = HDF5Sink(h5_path) - # Metadata that h5py might not like (e.g. nested dict) - sample = Sample(input=np.array([1]), metadata={"bad": {"nested": "value"}}) - sink.write(sample) - sink.close() - - -def test_zarr_group_open_overwrite(tmp_path: Path) -> None: - # hits zarr.py:29 (pass) - z_path = tmp_path / "z_ovr.zarr" - z_path.mkdir() - sink = ZarrGroupSink(z_path, overwrite=True) - sink.open() - - -def test_flux_iter_none_source() -> None: - # hits core.py:183, 193 - f = Flux(None) - assert list(f._iter_sequential()) == [] - assert list(f._iter_parallel()) == [] - - -def test_zarr_group_flush() -> None: - # hits zarr.py:56 - sink = ZarrGroupSink("test.zarr") - sink.flush() - - -def test_sample_from_any_tuple_1() -> None: - # hits sample.py:22 - s = Sample.from_any((1,)) - assert s.input == 1 - assert s.target is None - - -def test_sample_from_any_empty_tuple() -> None: - # hits sample.py new branch - s = Sample.from_any(()) - assert s.input is None - - -def test_optional_context_manager_direct() -> None: - # hits core.py:177 - from sampleflux.core import Flux - - f = Flux([1]) - - class SimpleSink: - def write(self, s: Any) -> None: - pass - - def flush(self) -> None: - pass - - f.to_sink(SimpleSink()) - - -def test_discovery_main_failure(monkeypatch: Any) -> None: - # hits discovery.py:30 (pass) - import sys - import types - - m = types.ModuleType("__main__") - # NO __file__ - monkeypatch.setitem(sys.modules, "__main__", m) - - def f() -> None: - pass - - f.__module__ = "__main__" - try: - get_callable_path(f) - except ValueError: - pass - - -def test_zarr_batch_chunks(tmp_path: Path) -> None: - # hits zarr.py: chunks logic - p = tmp_path / "chunks.zarr" - sink = ZarrBatchSink(p, shape=[10], chunks=[1, 10], overwrite=True) - sink.write(Sample(input=np.zeros(10))) - - -def test_zarr_group_source_none_root() -> None: - # hits the unopened-root guard branches in ZarrGroupSource - source = ZarrGroupSource("nonexistent.zarr") - source.open = lambda: source # type: ignore - assert list(source) == [] - assert len(source) == 0 - - -def test_zarr_batch_source_none_array() -> None: - # hits the unopened-array guard branches in ZarrBatchSource - source = ZarrBatchSource("nonexistent.zarr") - source.open = lambda: source # type: ignore - assert list(source) == [] - assert len(source) == 0 - - -def test_zarr_sources_close(tmp_path: Path) -> None: - # hits ZarrGroupSource.close / ZarrBatchSource.close - g = tmp_path / "g.zarr" - ZarrGroupSink(g, overwrite=True).write(Sample(input=np.zeros(3))) - gsrc = ZarrGroupSource(g) - gsrc.open() - gsrc.close() - assert gsrc._root is None - - b = tmp_path / "b.zarr" - ZarrBatchSink(b, shape=[3], overwrite=True).write(Sample(input=np.zeros(3))) - bsrc = ZarrBatchSource(b) - bsrc.open() - bsrc.close() - assert bsrc._data_arr is None diff --git a/tests/test_enable.py b/tests/test_enable.py deleted file mode 100644 index f3df796..0000000 --- a/tests/test_enable.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Tests for :class:`sampleflux.ops.enable.Enable` and :class:`sampleflux.ops.sink.SampleSinkOp`. - -These modality-neutral compose helpers moved here from ``waivefront.processing`` — -they thread any ``Sample`` through any ops and have no signal dependency. -""" - -from typing import Any, Dict, List - -import confluid -import pytest - -from sampleflux.ops.enable import Enable -from sampleflux.ops.sink import SampleSinkOp -from sampleflux.sample import Sample - - -class _CountingOp: - """Plain callable that records every invocation on a shared counter dict.""" - - def __init__(self, counter: Dict[str, int]) -> None: - self.counter = counter - - def __call__(self, sample: Sample) -> Sample: - self.counter["calls"] += 1 - new_meta = dict(sample.meta) - new_meta["counted"] = True - return sample._replace(metadata=new_meta) - - -def _sample() -> Sample: - return Sample(input=None, target=None, metadata={"x": 1}) - - -def _wrap(ops_: Any, **toggle: bool) -> Enable: - """Build an Enable + set the toggle attribute the way Confluid would. - - ``ops_`` accepts either a single callable (wrapped into a 1-list) or a - list, so existing single-op test cases stay concise. - """ - if not isinstance(ops_, list): - ops_ = [ops_] - e = Enable(ops=ops_) - for k, v in toggle.items(): - setattr(e, k, v) - return e - - -def test_enable_disabled_passes_sample_through_untouched() -> None: - counter: Dict[str, int] = {"calls": 0} - wrapped = _wrap(_CountingOp(counter), visualize=False) - out = wrapped(_sample()) - assert out is not None - assert counter["calls"] == 0 - assert "counted" not in out.meta - - -def test_enable_enabled_invokes_inner_op() -> None: - counter: Dict[str, int] = {"calls": 0} - wrapped = _wrap(_CountingOp(counter), visualize=True) - out = wrapped(_sample()) - assert out is not None - assert counter["calls"] == 1 - assert out.meta["counted"] is True - assert wrapped.flag_name == "visualize" - - -def test_enable_requires_exactly_one_boolean_attribute() -> None: - no_toggle = Enable(ops=[lambda s: s]) - with pytest.raises(RuntimeError, match="exactly one boolean toggle"): - no_toggle(_sample()) - - two_toggles = _wrap(lambda s: s, visualize=True, debug=False) - with pytest.raises(RuntimeError, match="exactly one boolean toggle"): - two_toggles(_sample()) - - -def test_enable_flag_name_is_arbitrary() -> None: - """Any kwarg name works — the chosen name is the CLI flag the user types.""" - counter: Dict[str, int] = {"calls": 0} - wrapped = _wrap(_CountingOp(counter), debug_overlay=True) - wrapped(_sample()) - assert counter["calls"] == 1 - assert wrapped.flag_name == "debug_overlay" - - -def test_enable_multi_op_threads_sample_through_each() -> None: - counter_a: Dict[str, int] = {"calls": 0} - counter_b: Dict[str, int] = {"calls": 0} - - class _TagOp: - def __init__(self, tag: str, counter: Dict[str, int]) -> None: - self.tag = tag - self.counter = counter - - def __call__(self, sample: Sample) -> Sample: - self.counter["calls"] += 1 - new_meta = dict(sample.meta) - tags = list(new_meta.get("tags", [])) - tags.append(self.tag) - new_meta["tags"] = tags - return sample._replace(metadata=new_meta) - - wrapped = _wrap( - [_TagOp("first", counter_a), _TagOp("second", counter_b)], - visualize=True, - ) - out = wrapped(_sample()) - assert out is not None - assert counter_a["calls"] == 1 - assert counter_b["calls"] == 1 - assert out.meta["tags"] == ["first", "second"] - - -def test_enable_multi_op_disabled_skips_entire_chain() -> None: - counter: Dict[str, int] = {"calls": 0} - wrapped = _wrap( - [_CountingOp(counter), _CountingOp(counter)], - visualize=False, - ) - wrapped(_sample()) - assert counter["calls"] == 0 - - -def test_enable_zero_arg_construction_then_rejects_empty_ops_on_call() -> None: - """Zero-arg / empty-ops construction succeeds (lazy convention); the non-empty - requirement is enforced on first call, not in ``__init__``.""" - empty = Enable() # zero-arg construction must work - assert empty.ops == [] - empty.visualize = True # type: ignore[attr-defined] - with pytest.raises(ValueError, match="non-empty 'ops' list"): - empty(_sample()) - - -def test_enable_preserves_name_attr_and_toggle_independence() -> None: - """Two Enable instances with distinct names carry their names through. - - Confirms that (a) ``name`` is a plain string attr that survives - construction + post-construction setattr, (b) a string-valued name - doesn't collide with ``_toggle()``'s bool-attr filter, and (c) the - two wrappers can be toggled independently when each has its own - boolean attribute. - """ - counter_a: Dict[str, int] = {"calls": 0} - counter_b: Dict[str, int] = {"calls": 0} - - overlay = Enable(ops=[_CountingOp(counter_a)]) - overlay.name = "overlay" # type: ignore[attr-defined] # set by Confluid at flow time - overlay.visualize = True # type: ignore[attr-defined] - - ls = Enable(ops=[_CountingOp(counter_b)]) - ls.name = "labelstudio" # type: ignore[attr-defined] - ls.visualize = False # type: ignore[attr-defined] - - overlay(_sample()) - ls(_sample()) - - assert counter_a["calls"] == 1 - assert counter_b["calls"] == 0 - # Names are preserved verbatim; _toggle's bool filter ignores them. - assert overlay.name == "overlay" # type: ignore[attr-defined] - assert ls.name == "labelstudio" # type: ignore[attr-defined] - assert overlay.flag_name == "visualize" - assert ls.flag_name == "visualize" - - -def test_enable_yaml_load_with_cli_style_override(tmp_path: Any) -> None: - """Mimic what Liquify's --visualize true override does to Fluid kwargs.""" - yaml_text = """\ -wrapper: - !class:sampleflux.ops.enable.Enable - visualize: false - ops: - - !class:sampleflux.ops.copy.CopySampleOp {} -""" - cfg = tmp_path / "enable.yaml" - cfg.write_text(yaml_text) - loaded = confluid.load(cfg) - enable_fluid = loaded["wrapper"] - assert "visualize" in enable_fluid.kwargs - # Liquify CLI override path mutates Fluid.kwargs in-place before flow(). - enable_fluid.kwargs["visualize"] = True - materialized = confluid.flow(enable_fluid) - assert isinstance(materialized, Enable) - assert materialized.enabled is True - assert materialized.flag_name == "visualize" - - -# --- SampleSinkOp & Enable.close propagation -------------------------------- - - -class _RecordingSink: - """Captures the open/write/flush/close lifecycle for assertions.""" - - def __init__(self) -> None: - self.calls: List[str] = [] - self.writes: List[Sample] = [] - - def open(self) -> "_RecordingSink": - self.calls.append("open") - return self - - def write(self, sample: Sample) -> None: - self.calls.append("write") - self.writes.append(sample) - - def flush(self) -> None: - self.calls.append("flush") - - def close(self) -> None: - self.calls.append("close") - - -def test_sample_sink_op_lazy_open_then_writes() -> None: - """``open()`` fires once on first call, write() per call, returning the sample.""" - sink = _RecordingSink() - op = SampleSinkOp(sink=sink) - - s1 = Sample(input=None, target=None, metadata={"i": 0}) - s2 = Sample(input=None, target=None, metadata={"i": 1}) - out1 = op(s1) - out2 = op(s2) - - assert out1 is s1 and out2 is s2 # pass-through - assert sink.calls == ["open", "write", "write"] - assert sink.writes == [s1, s2] - - -def test_sample_sink_op_close_flushes_and_closes() -> None: - """``close()`` calls flush() then close() so buffered sinks finalize cleanly.""" - sink = _RecordingSink() - op = SampleSinkOp(sink=sink) - op(Sample(input=None, target=None, metadata={})) - op.close() - assert sink.calls == ["open", "write", "flush", "close"] - - -def test_sample_sink_op_zero_arg_construction_then_rejects_none_on_call() -> None: - """Zero-arg construction works (lazy convention); a missing sink raises on first call.""" - op = SampleSinkOp() # zero-arg construction must work - assert op.sink is None - with pytest.raises(ValueError, match="non-None 'sink'"): - op(Sample(input=None, target=None, metadata={})) - - -def test_enable_close_propagates_into_inner_ops() -> None: - """Closing an Enable wrapper drives close() on every inner op that owns one.""" - sink = _RecordingSink() - inner = SampleSinkOp(sink=sink) - wrapped = _wrap(inner, visualize=True) - wrapped(Sample(input=None, target=None, metadata={})) - wrapped.close() - assert sink.calls == ["open", "write", "flush", "close"] - - -def test_enable_close_on_disabled_wrapper_still_propagates() -> None: - """Even when toggled off (and thus never invoked), close() must still reach - inner ops in case they were opened independently — it must NEVER raise.""" - sink = _RecordingSink() - inner = SampleSinkOp(sink=sink) - wrapped = _wrap(inner, visualize=False) - # Never called; close still safe. - wrapped.close() - # The SampleSinkOp was never opened, so there's no write — but flush+close - # are forwarded unconditionally by SampleSinkOp.close. - assert sink.calls == ["flush", "close"] diff --git a/tests/test_expanding_ops.py b/tests/test_expanding_ops.py deleted file mode 100644 index be965f5..0000000 --- a/tests/test_expanding_ops.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Tests for 1→N expanding ops and iterable-only pipeline semantics.""" - -from typing import Iterator, List, Optional - -import pytest -from confluid import configurable - -from sampleflux.core import Flux, _worker_task, _worker_task_multi -from sampleflux.kinds import op_contract -from sampleflux.ops.context import Save, Use -from sampleflux.sample import Sample - -# --------------------------------------------------------------------------- -# Fixture ops (module-level so they pickle for spawn parity) -# --------------------------------------------------------------------------- - - -@configurable -class SplitOp: - """Expand one sample into ``count`` children (index appended to metadata). - - Args: - count: Number of children yielded per incoming sample. - """ - - def __init__(self, count: int = 2) -> None: - self.count = count - - def __call__(self, sample: Sample) -> Iterator[Sample]: - for i in range(self.count): - yield sample._replace(input=sample.input * 10 + i, metadata={**sample.meta, "child": i}) - - -@configurable -class MarkedSplitOp: - """An expansion op detected via the explicit EXPANDS marker (untyped __call__).""" - - EXPANDS = True - - def __call__(self, sample): # type: ignore[no-untyped-def] - return [sample, sample] - - -@configurable -class AddOp: - """Add a constant to the input. - - Args: - amount: Value added to ``sample.input``. - """ - - def __init__(self, amount: float = 1.0) -> None: - self.amount = amount - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + self.amount) - - -@configurable -class DropOddChildOp: - """Filter inside an expansion: drop children with odd input.""" - - def __call__(self, sample: Sample) -> Optional[Sample]: - return None if int(sample.input) % 2 else sample - - -@configurable -class EmptySplitOp: - """An expanding op that yields nothing (drops the sample entirely).""" - - def __call__(self, sample: Sample) -> Iterator[Sample]: - return iter(()) - - -def _samples(n: int = 2) -> List[Sample]: - return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] - - -# --------------------------------------------------------------------------- -# Engine routes -# --------------------------------------------------------------------------- - - -class TestExpansion: - def test_sequential_expansion_depth_first_order(self) -> None: - out = list(Flux(source=_samples(2), ops=[SplitOp(count=2), AddOp(amount=0.5)])) - # sample 0 -> children 0,1 -> +0.5 ; sample 1 -> 10,11 -> +0.5 - assert [s.input for s in out] == [0.5, 1.5, 10.5, 11.5] - assert [s.meta["child"] for s in out] == [0, 1, 0, 1] - - def test_chained_expansions(self) -> None: - out = list(Flux(source=_samples(1), ops=[SplitOp(count=2), SplitOp(count=2)])) - # 0 -> [0, 1] -> [00,01,10,11] depth-first - assert [s.input for s in out] == [0.0, 1.0, 10.0, 11.0] - - def test_none_drop_inside_expansion(self) -> None: - out = list(Flux(source=_samples(1), ops=[SplitOp(count=4), DropOddChildOp()])) - assert [s.input for s in out] == [0.0, 2.0] - - def test_empty_expansion_drops_the_sample(self) -> None: - assert list(Flux(source=_samples(3), ops=[EmptySplitOp()])) == [] - - def test_marked_expansion_via_class_attr(self) -> None: - assert op_contract(MarkedSplitOp()).expands is True - out = list(Flux(source=_samples(1), ops=[MarkedSplitOp()])) - assert len(out) == 2 - - def test_spawn_parallel_parity(self) -> None: - seq = [s.input for s in Flux(source=_samples(3), ops=[SplitOp(count=2), AddOp()])] - par = [s.input for s in Flux(source=_samples(3), ops=[SplitOp(count=2), AddOp()]).parallel(2)] - assert seq == par - - def test_streamed_route_expansion(self) -> None: - from sampleflux.ops.parallel import Parallel - - ops = [SplitOp(count=2), Parallel(ops=[AddOp(amount=0.5)], workers=1)] - out = list(Flux(source=_samples(2), ops=ops)) - assert sorted(s.input for s in out) == [0.5, 1.5, 10.5, 11.5] - - def test_batch_over_expanded_stream(self) -> None: - chunks = list(Flux(source=_samples(2), ops=[SplitOp(count=3)]).batch(4)) - assert [len(c) for c in chunks] == [4, 2] - - def test_expansion_with_context_ops(self) -> None: - # A fork saved BEFORE the expansion is readable by each child (shallow ctx copy). - ops = [Save(name="fork"), SplitOp(count=2), Use(name="fork")] - out = list(Flux(source=_samples(2), ops=ops)) - # Use restores the pre-split fork for every child -> inputs are the originals. - assert [s.input for s in out] == [0.0, 0.0, 1.0, 1.0] - - -class TestIterableOnly: - def test_len_raises_with_actionable_message(self) -> None: - flux = Flux(source=_samples(3), ops=[SplitOp()]) - with pytest.raises(TypeError, match="ITERABLE-ONLY.*SplitOp|SplitOp.*ITERABLE-ONLY"): - len(flux) - - def test_getitem_raises(self) -> None: - flux = Flux(source=_samples(3), ops=[SplitOp()]) - with pytest.raises(TypeError, match="iterable-only|ITERABLE-ONLY"): - _ = flux[0] - - def test_non_expanding_pipeline_keeps_random_access(self) -> None: - flux = Flux(source=_samples(3), ops=[AddOp()]) - assert len(flux) == 3 and flux[1].input == 2.0 - - def test_strict_worker_task_rejects_expansion(self) -> None: - with pytest.raises(TypeError, match="1→N expanding"): - _worker_task(Sample(input=1.0), [SplitOp()]) - - def test_worker_task_multi_returns_all(self) -> None: - results = _worker_task_multi(Sample(input=1.0, metadata={}), [SplitOp(count=3)]) - assert [s.input for s in results] == [10.0, 11.0, 12.0] - - -class TestContextIsolationAcrossChildren: - def test_children_have_independent_cell_sets(self) -> None: - @configurable - class SaveChildIdOp: - def __call__(self, sample: Sample) -> Sample: - from sampleflux.context import require - - require("SaveChildIdOp").put("mine", sample.meta["child"]) - return sample - - @configurable - class ReadBackOp: - def __call__(self, sample: Sample) -> Sample: - from sampleflux.context import require - - return sample._replace(target=require("ReadBackOp").get("mine")) - - out = list(Flux(source=_samples(1), ops=[SplitOp(count=3), SaveChildIdOp(), ReadBackOp()])) - assert [s.target for s in out] == [0, 1, 2] # no cross-child leakage - - -def test_flowgraph_rejects_expanding_step() -> None: - from sampleflux.flow import FlowGraph - - graph = FlowGraph(source=_samples(1), flow={"split": SplitOp()}) - with pytest.raises(NotImplementedError, match="expanding"): - list(graph) diff --git a/tests/test_flow.py b/tests/test_flow.py deleted file mode 100644 index e6ec335..0000000 --- a/tests/test_flow.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Tests for the flow document, the FlowGraph engine, and the flow⇄ops converters. - -The load-bearing contract: **execution parity both ways** — a flow document run natively -by FlowGraph equals the same flow lowered (`to_ops`) and run by the serial Flux engine, -and a flat context-ops list run by Flux equals its lifted (`from_ops`) flow run by -FlowGraph. Round-tripping re-lowers to an execution-equivalent list. -""" - -from pathlib import Path -from typing import Callable, Optional - -import pytest -from confluid import configurable, output - -from sampleflux.context import Context, activate -from sampleflux.core import Flux -from sampleflux.flow import FlowGraph, FlowStep, from_ops, parse_flow, to_ops -from sampleflux.ops.context import Apply, Capture, Drop, Mix, Save, Use -from sampleflux.ops.swap import SwapInputTargetOp -from sampleflux.sample import Sample - -# --------------------------------------------------------------------------- -# Test ops (module-level so they pickle for spawn parity) -# --------------------------------------------------------------------------- - - -@configurable -class AddOp: - """Add a constant to the input. - - Args: - amount: Value added to ``sample.input``. - """ - - def __init__(self, amount: float = 1.0) -> None: - self.amount = amount - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + self.amount) - - -@configurable -class ScaleOp: - """Multiply the input by a factor. - - Args: - factor: Multiplier applied to ``sample.input``. - """ - - def __init__(self, factor: float = 2.0) -> None: - self.factor = factor - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input * self.factor) - - -@configurable -class TenfoldOutputOp: - """Pass-through with a DETERMINISTIC @output (parity tests need reproducibility).""" - - def __init__(self) -> None: - self._last: float = 0.0 - - @property - @output - def tenfold(self) -> float: - """Ten times the last seen input.""" - return self._last - - def __call__(self, sample: Sample) -> Sample: - self._last = float(sample.input) * 10.0 - return sample - - -@configurable -class DropOddOp: - """Filter: drop samples with odd integer input.""" - - def __call__(self, sample: Sample) -> Optional[Sample]: - return None if int(sample.input) % 2 else sample - - -def _samples(n: int = 4) -> list: - return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] - - -def _key(sample: Sample) -> tuple: - target = sample.target - if isinstance(target, Sample): - target = ("sample", target.input, target.target) - return (sample.input, target, tuple(sorted(sample.meta.items()))) - - -def _assert_parity(flow_doc: dict, outputs: str = "", n: int = 4) -> None: - """FlowGraph-native == Flux-over-lowered, sample for sample (fresh ops per engine).""" - import copy - - native = FlowGraph(source=_samples(n), flow=copy.deepcopy(flow_doc), outputs=outputs) - lowered_steps, out = parse_flow(copy.deepcopy(flow_doc), outputs) - serial = Flux(source=_samples(n), ops=to_ops(lowered_steps, out)) - got_native = [_key(s) for s in native] - got_serial = [_key(s) for s in serial] - assert got_native == got_serial, f"engine parity broken:\n native={got_native}\n serial={got_serial}" - - -# --------------------------------------------------------------------------- -# parse_flow validation -# --------------------------------------------------------------------------- - - -class TestParseFlow: - def test_linear_defaults(self) -> None: - steps, out = parse_flow({"a": AddOp(), "b": ScaleOp()}) - assert [s.name for s in steps] == ["a", "b"] - assert steps[1].from_ is None and out == "b" - - def test_forward_reference_rejected(self) -> None: - with pytest.raises(ValueError, match="EARLIER step"): - parse_flow({"a": {"op": AddOp(), "from": "b"}, "b": ScaleOp()}) - - def test_dotted_step_name_rejected(self) -> None: - with pytest.raises(ValueError, match="may not contain"): - parse_flow({"a.b": AddOp()}) - - def test_unknown_step_key_rejected(self) -> None: - with pytest.raises(ValueError, match="unknown step key"): - parse_flow({"a": AddOp(), "b": {"op": ScaleOp(), "sideways": "a"}}) - - def test_bind_requires_known_step_and_op(self) -> None: - with pytest.raises(ValueError, match="does not name an earlier step"): - parse_flow({"a": {"op": AddOp(), "bind": {"amount": "ghost"}}}) - with pytest.raises(ValueError, match="bind requires an op"): - parse_flow({"a": AddOp(), "b": {"from": "a", "bind": {"x": "a"}}}) - - def test_outputs_must_name_a_step(self) -> None: - with pytest.raises(ValueError, match="outputs"): - parse_flow({"a": AddOp()}, outputs="ghost") - - def test_reserved_ctor_param_collision_rejected(self) -> None: - @configurable - class BadOp: - """Op with a reserved-name ctor param. - - Args: - bind: Collides with the reserved flow step key. - """ - - def __init__(self, bind: str = "") -> None: - self.bind = bind - - def __call__(self, sample: Sample) -> Sample: - return sample - - with pytest.raises(ValueError, match="reserved flow step keys"): - parse_flow({"a": BadOp()}) - - def test_duplicate_step_name_rejected(self) -> None: - # dicts dedupe keys silently, so build the parsed list directly - steps = [ - FlowStep("a", AddOp(), None, None, None, {}), - FlowStep("a", ScaleOp(), None, None, None, {}), - ] - graph = FlowGraph(source=_samples(1), flow=steps) - assert graph.steps # duplicate FlowStep lists are the caller's problem; parse_flow guards dicts - - -# --------------------------------------------------------------------------- -# Engine parity (the load-bearing contract) -# --------------------------------------------------------------------------- - - -class TestEngineParity: - def test_linear(self) -> None: - _assert_parity({"a": AddOp(amount=1.0), "b": ScaleOp(factor=3.0)}) - - def test_linear_lowering_is_bare(self) -> None: - ops = to_ops({"a": AddOp(amount=1.0), "b": ScaleOp(factor=3.0)}) - assert [type(o).__name__ for o in ops] == ["AddOp", "ScaleOp"] # zero context ops - - def test_fan_out_fan_in(self) -> None: - _assert_parity( - { - "a": AddOp(amount=1.0), - "b": {"op": SwapInputTargetOp(), "from": "a"}, - "c": {"op": AddOp(amount=5.0), "from": "a"}, - "out": {"from": "c", "target_from": "b"}, - } - ) - - def test_bind_step_result(self) -> None: - _assert_parity( - { - "thresh": ScaleOp(factor=0.5), - "shifted": {"op": AddOp(), "from": "thresh", "bind": {"amount": "thresh"}}, - } - ) - - def test_bind_at_output(self) -> None: - _assert_parity( - { - "probe": TenfoldOutputOp(), - "shifted": {"op": AddOp(), "bind": {"amount": "probe.tenfold"}}, - } - ) - - def test_outputs_earlier_step(self) -> None: - _assert_parity({"a": AddOp(amount=1.0), "b": ScaleOp(factor=3.0)}, outputs="a") - - def test_identity_first_step_source_fork(self) -> None: - _assert_parity( - { - "src": {}, - "a": AddOp(amount=1.0), - "b": {"op": ScaleOp(factor=2.0), "from": "src"}, - "out": {"from": "b", "target_from": "a"}, - } - ) - - def test_filtering_drops_in_both_engines(self) -> None: - flow_doc = {"f": DropOddOp(), "a": AddOp(amount=1.0)} - _assert_parity(flow_doc) - native = FlowGraph(source=_samples(4), flow={"f": DropOddOp(), "a": AddOp(amount=1.0)}) - assert [s.input for s in native] == [1.0, 3.0] - - def test_metadata_from_slot(self) -> None: - _assert_parity( - { - "a": AddOp(amount=1.0), - "b": {"op": ScaleOp(factor=2.0), "from": "a"}, - "out": {"from": "b", "metadata_from": "a"}, - } - ) - - -class TestReverseParity: - """Flux(ops) == FlowGraph(from_ops(ops)) — lifting preserves execution.""" - - def _assert_reverse(self, ops_builder: Callable[[], list], n: int = 4) -> None: - serial = Flux(source=_samples(n), ops=ops_builder()) - flow_doc, out = from_ops(ops_builder()) - native = FlowGraph(source=_samples(n), flow=flow_doc, outputs=out) - assert [_key(s) for s in serial] == [_key(s) for s in native] - - def test_linear_list(self) -> None: - self._assert_reverse(lambda: [AddOp(amount=1.0), ScaleOp(factor=3.0)]) - - def test_hand_written_graph_list(self) -> None: - self._assert_reverse( - lambda: [ - Save(name="fork"), - AddOp(amount=1.0), - SwapInputTargetOp(), - Save(name="branch_a"), - Use(name="fork", drop=True), - ScaleOp(factor=2.0), - Mix(target_from="branch_a", drop=["branch_a"]), - ] - ) - - def test_apply_capture_list(self) -> None: - self._assert_reverse( - lambda: [ - Capture(op=TenfoldOutputOp(), output="tenfold", name="t"), - Apply(op=AddOp(), param="amount", source="t", drop=True), - ] - ) - - def test_drop_ops_vanish_from_lifted_flow(self) -> None: - flow_doc, _ = from_ops([Save(name="x"), AddOp(), Drop(names=["x"])]) - assert all("drop" not in str(v).lower() or "op" in v for v in flow_doc.values()) - - -class TestRoundTrip: - def test_flow_to_ops_to_flow_execution_equivalent(self) -> None: - original = { - "a": AddOp(amount=1.0), - "b": {"op": SwapInputTargetOp(), "from": "a"}, - "c": {"op": AddOp(amount=5.0), "from": "a"}, - "out": {"from": "c", "target_from": "b"}, - } - lowered = to_ops(dict(original)) - lifted, out = from_ops(lowered) - relowered = to_ops(lifted, out) - a = [_key(s) for s in Flux(source=_samples(4), ops=lowered)] - b = [_key(s) for s in Flux(source=_samples(4), ops=relowered)] - assert a == b - - def test_lowered_graph_leaves_context_empty(self) -> None: - ops = to_ops( - { - "a": AddOp(amount=1.0), - "b": {"op": SwapInputTargetOp(), "from": "a"}, - "out": {"from": "a", "target_from": "b"}, - } - ) - ctx = Context() - sample: Sample = Sample(input=1.0, metadata={}) - with activate(ctx): - for op in ops: - result = op(sample) - assert result is not None - sample = result - assert ctx.live() == () # automatic liveness freed every cell - - -# --------------------------------------------------------------------------- -# FlowGraph engine surface -# --------------------------------------------------------------------------- - - -class TestFlowGraphSurface: - def test_zero_arg_construction(self) -> None: - graph = FlowGraph() - with pytest.raises(ValueError, match="flow is not set"): - _ = graph.steps - - def test_len_and_getitem(self) -> None: - graph = FlowGraph(source=_samples(5), flow={"a": AddOp(amount=1.0)}) - assert len(graph) == 5 - assert graph[2].input == 3.0 - - def test_getitem_filtered_raises_indexerror(self) -> None: - graph = FlowGraph(source=_samples(4), flow={"f": DropOddOp()}) - with pytest.raises(IndexError, match="filtered"): - _ = graph[1] - - def test_batch(self) -> None: - graph = FlowGraph(source=_samples(4), flow={"a": AddOp()}).batch(3) - chunks = list(graph) - assert [len(c) for c in chunks] == [3, 1] - - def test_parallel_spawn_parity(self) -> None: - flow_doc = { - "a": AddOp(amount=1.0), - "b": {"op": SwapInputTargetOp(), "from": "a"}, - "c": {"op": AddOp(amount=5.0), "from": "a"}, - "out": {"from": "c", "target_from": "b"}, - } - seq = [_key(s) for s in FlowGraph(source=_samples(4), flow=dict(flow_doc))] - par = [_key(s) for s in FlowGraph(source=_samples(4), flow=dict(flow_doc)).parallel(2)] - assert seq == par - - def test_to_flux_twin(self) -> None: - graph = FlowGraph(source=_samples(3), flow={"a": AddOp(amount=2.0)}) - assert [s.input for s in graph.to_flux()] == [2.0, 3.0, 4.0] - - def test_collect(self) -> None: - graph = FlowGraph(source=_samples(2), flow={"a": AddOp()}) - assert len(graph.collect()) == 2 - - -# --------------------------------------------------------------------------- -# YAML round-trips -# --------------------------------------------------------------------------- - -_FLOW_YAML = """ -flow: - a: !class:tests.test_flow.AddOp(amount=1.0) - b: !class:sampleflux.ops.swap.SwapInputTargetOp() - from: a - c: !class:tests.test_flow.AddOp(amount=5.0) - from: a - out: - from: c - target_from: b -outputs: out -""" - - -class TestYaml: - def test_flowgraph_from_yaml(self, tmp_path: Path) -> None: - path = tmp_path / "graph.yaml" - path.write_text(_FLOW_YAML) - graph = FlowGraph.from_yaml(str(path), source=_samples(3)) - results = list(graph) - # v -> a=v+1 -> c=a+5=v+6 (input); target = b's target = a's swapped input = v+1 - assert [s.input for s in results] == [6.0, 7.0, 8.0] - assert [s.target for s in results] == [1.0, 2.0, 3.0] - - def test_flux_from_flow_yaml_matches_native(self, tmp_path: Path) -> None: - path = tmp_path / "graph.yaml" - path.write_text(_FLOW_YAML) - native = [_key(s) for s in FlowGraph.from_yaml(str(path), source=_samples(3))] - serial = [_key(s) for s in Flux.from_flow_yaml(str(path), source=_samples(3))] - assert native == serial - - def test_flowgraph_from_ops_yaml(self, tmp_path: Path) -> None: - ops_yaml = """ -ops: - - !class:tests.test_flow.AddOp(amount=1.0) - - !class:tests.test_flow.ScaleOp(factor=3.0) -""" - path = tmp_path / "ops.yaml" - path.write_text(ops_yaml) - graph = FlowGraph.from_ops_yaml(str(path), source=_samples(3)) - assert [s.input for s in graph] == [3.0, 6.0, 9.0] diff --git a/tests/test_flux.py b/tests/test_flux.py deleted file mode 100644 index 29c8420..0000000 --- a/tests/test_flux.py +++ /dev/null @@ -1,263 +0,0 @@ -from typing import Any - -import numpy as np -import pytest - -from sampleflux.core import Flux -from sampleflux.sample import Sample - - -def test_basic_flux() -> None: - source = [np.array([1, 2, 3]), np.array([4, 5, 6])] - pipeline = Flux(source) - results = list(pipeline) - assert len(results) == 2 - assert isinstance(results[0], Sample) - assert np.array_equal(results[0].input, source[0]) - - -def double_it(x: np.ndarray) -> np.ndarray: - return x * 2 - - -def is_greater_than_two(s: Sample) -> bool: - # Use np.any() or similar to ensure a single boolean is returned for mypy - return bool(np.any(s.input > 2)) - - -def test_flux_map() -> None: - source = [np.array([1, 2, 3])] - pipeline = Flux(source).map(double_it) - results = list(pipeline) - assert np.array_equal(results[0].input, np.array([2, 4, 6])) - - -def test_flux_filter() -> None: - source = [np.array([1]), np.array([2]), np.array([3]), np.array([4])] - pipeline = Flux(source).filter(is_greater_than_two) - results = list(pipeline) - assert len(results) == 2 - assert results[0].input == 3 - - -class MockSink: - def __init__(self) -> None: - self.written: list[Sample] = [] - - def write(self, sample: Sample) -> None: - self.written.append(sample) - - def flush(self) -> None: - pass - - -def test_flux_to_sink() -> None: - source = [np.array([1, 2]), np.array([3, 4])] - sink = MockSink() - Flux(source).to_sink(sink) - assert len(sink.written) == 2 - assert np.array_equal(sink.written[0].input, source[0]) - - -def full_transform(s: Sample) -> Sample: - return s._replace(input=s.input * 2) - - -def test_wrapped_op_all() -> None: - source = [np.array([10])] - pipeline = Flux(source).map(full_transform, select="all") - results = pipeline.collect() - assert results[0].input == 20 - - -def test_filter_op() -> None: - from sampleflux.core import FilterOp - - op = FilterOp(lambda s: bool(s.input > 5)) - s1 = Sample(input=10) - s2 = Sample(input=2) - assert op(s1) == s1 - assert op(s2) is None - - -def fail_op(x: Any) -> Any: - raise ValueError("Intentional failure") - - -def test_wrapped_op_error() -> None: - source = [np.array([1])] - pipeline = Flux(source).map(fail_op) - with pytest.raises(ValueError, match="Intentional failure"): - pipeline.collect() - - -def target_transform(t: Any) -> Any: - return t + 10 - - -def test_wrapped_op_target() -> None: - source = [Sample(input=1, target=5)] - # select="target" hits lines 69-70 - pipeline = Flux(source).map(target_transform, select="target") - results = pipeline.collect() - assert results[0].target == 15 - - -def test_wrapped_op_fallback() -> None: - # select="unknown" hits line 73 - source = [Sample(input=1)] - pipeline = Flux(source).map(lambda x: x, select="unknown") - results = pipeline.collect() - assert results[0].input == 1 - - -def test_worker_task_none() -> None: - from sampleflux.core import _worker_task - - # hits line 83 by using two ops, first returning None - assert _worker_task(Sample(input=1), [lambda s: None, lambda s: s]) is None - - -def test_flux_from_source() -> None: - # hits from_source classmethod - f = Flux.from_source([1, 2, 3]) - assert len(f) == 3 - - -def test_flux_len_fallback() -> None: - # hits line 134 (len 0 if no source or no len) - f = Flux() - assert len(f) == 0 - f2 = Flux(iter([1, 2])) # iter doesn't have len - assert len(f2) == 0 - - -# --- Indexed access (__getitem__) --- - - -def test_getitem_basic() -> None: - source = [np.array([10]), np.array([20]), np.array([30])] - flux = Flux(source) - sample = flux[1] - assert isinstance(sample, Sample) - assert np.array_equal(sample.input, np.array([20])) - - -def test_getitem_with_ops() -> None: - source = [np.array([1]), np.array([2]), np.array([3])] - flux = Flux(source).map(double_it) - assert np.array_equal(flux[0].input, np.array([2])) - assert np.array_equal(flux[2].input, np.array([6])) - - -def test_getitem_no_source() -> None: - flux = Flux() - with pytest.raises(TypeError): - flux[0] - - -def test_getitem_non_indexable_source() -> None: - flux = Flux(iter([1, 2, 3])) # iterators don't support indexing - with pytest.raises(TypeError): - flux[0] - - -def test_getitem_out_of_range() -> None: - source = [np.array([1])] - flux = Flux(source) - with pytest.raises(IndexError): - flux[5] - - -class _IterableWithLen: - """Iterable source with ``__iter__`` + ``__len__`` but NO ``__getitem__``. - - Mirrors the shape of ``waivefront.regions_source.RegionsJsonSource``. - Tracks how many times ``__iter__`` is invoked so tests can prove Flux - materializes only once per Flux lifetime. - """ - - def __init__(self, items: list) -> None: - self._items = items - self.iter_calls = 0 - - def __iter__(self) -> Any: - self.iter_calls += 1 - return iter(self._items) - - def __len__(self) -> int: - return len(self._items) - - -def test_getitem_iterable_with_len_materializes_once() -> None: - """Flux caches iterable-only sources on first __getitem__ and reuses the cache.""" - source = _IterableWithLen([np.array([1]), np.array([2]), np.array([3])]) - flux = Flux(source) - - # Cache is empty until first random access. - assert flux._indexable_cache is None - assert source.iter_calls == 0 - - s0 = flux[0] - s1 = flux[1] - s2 = flux[2] - - assert np.array_equal(s0.input, np.array([1])) - assert np.array_equal(s1.input, np.array([2])) - assert np.array_equal(s2.input, np.array([3])) - # The source was iterated exactly once across the three random-access calls. - assert source.iter_calls == 1 - assert flux._indexable_cache is not None - assert len(flux._indexable_cache) == 3 - - -def test_getitem_indexable_source_does_not_populate_cache() -> None: - """List-backed (indexable) sources take the fast path and don't trigger the cache.""" - source = [np.array([1]), np.array([2])] - flux = Flux(source) - _ = flux[0] - _ = flux[1] - # Fast path: cache is never populated. - assert flux._indexable_cache is None - - -def test_getitem_on_deferred_fluid_source_raises_actionable_error() -> None: - """When the user hands Flux a still-deferred Confluid Class marker, indexing - must raise with a human-readable hint — not the cryptic ``num_samples=0`` - or ``does not support indexing``. - """ - from confluid import Class - - class Dummy: - def __init__(self, x: int = 0) -> None: - self.x = x - - # Deliberately wrong type — this test verifies Flux raises a clear error - # when handed a still-deferred Confluid marker as the source. - flux = Flux(Class(Dummy, x=1)) # type: ignore[arg-type] - with pytest.raises(TypeError, match="deferred Confluid marker"): - flux[0] - with pytest.raises(TypeError, match="deferred Confluid marker"): - len(flux) - - -def test_deferred_fluid_op_raises_actionable_error() -> None: - """Same guard applies to ops: a still-deferred Class in ops[] surfaces a - clear message pointing the YAML author at the `!class:X()` fix instead of - letting DataLoader workers report `'Class' object is not callable`. - """ - from confluid import Class - - class DummyOp: - def __init__(self, factor: int = 1) -> None: - self.factor = factor - - def __call__(self, sample: Sample) -> Sample: - return sample - - source = [np.array([1]), np.array([2])] - flux = Flux(source, ops=[Class(DummyOp, factor=2)]) - with pytest.raises(TypeError, match=r"Flux\.ops\[0\] is still a deferred Confluid marker"): - flux[0] - with pytest.raises(TypeError, match=r"Flux\.ops\[0\] is still a deferred Confluid marker"): - list(flux) diff --git a/tests/test_from_ops_yaml.py b/tests/test_from_ops_yaml.py deleted file mode 100644 index 3aedcd8..0000000 --- a/tests/test_from_ops_yaml.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for ``Flux.from_ops_yaml`` — attach an exported ops-only YAML to a source. - -The YAML shape is what ``fluxstudio.export`` emits: a ``{ops: [!class:...()]}`` document. -The key behaviour under test is that the helper **materializes** the deferred ``!class:`` -markers (which ``confluid.load`` leaves un-flowed when nested under a mapping key) before -attaching them, so iteration sees live callables rather than ``Instance`` markers. -""" - -from pathlib import Path - -import torch - -from sampleflux import Flux, Sample -from sampleflux.ops.torch import RescaleOp # noqa: F401 - import registers the @configurable for !class: resolution - -OPS_YAML = """ops: -- !class:sampleflux.ops.torch.RescaleOp() - in_min: 0.0 - in_max: 255.0 -- !class:sampleflux.ops.torch.RescaleOp() - in_min: 0.0 - in_max: 1.0 - out_max: 10.0 -""" - - -def test_from_ops_yaml_materializes_and_attaches(tmp_path: Path) -> None: - path = tmp_path / "ops.yaml" - path.write_text(OPS_YAML) - src = [Sample(input=torch.tensor([0.0, 255.0]), target=None, metadata={})] - - flux = Flux.from_ops_yaml(str(path), source=src) - - # Materialized to live ops — NOT deferred Instance markers (which iteration would reject). - assert [type(o).__name__ for o in flux.ops] == ["RescaleOp", "RescaleOp"] - out = list(flux)[0].input - assert torch.allclose(out, torch.tensor([0.0, 10.0])) - - -def test_from_ops_yaml_without_ops_key_is_empty(tmp_path: Path) -> None: - path = tmp_path / "empty.yaml" - path.write_text("other: 1\n") - - flux = Flux.from_ops_yaml(str(path), source=[]) - - assert flux.ops == [] diff --git a/tests/test_image_ops.py b/tests/test_image_ops.py deleted file mode 100644 index 59e4e4f..0000000 --- a/tests/test_image_ops.py +++ /dev/null @@ -1,499 +0,0 @@ -"""Tests for :mod:`sampleflux.ops.image` — generic value→image conversion. - -``ConvertToImageOp`` is the generic image-conversion op (normalize → colormap → -optional flip → resize), and ``value_to_image`` / ``sample_to_image`` back it -(and FluxStudio's preview). The signal-specific overlay drawing lives in -waivefront (``RenderOverlaysOp``) and is tested there. -""" - -import json -from typing import get_args - -import numpy as np -import pytest -import torch -from PIL import Image - -from sampleflux.ops.image import ( - COLORMAPS, - TEXT_POSITIONS, - Colormap, - ConvertToImageOp, - NormalizeToUint8Op, - TextPosition, - _apply_colormap, - array_histogram, - channel_count, - confusion_matrices_payload, - confusion_matrix_payload, - draw_text, - sample_to_image, - select_channel, - value_to_image, -) -from sampleflux.sample import Sample - - -def _sample(value: object) -> Sample: - return Sample(input=value, target=None, metadata={}) - - -# --------------------------------------------------------------------------- -# ConvertToImageOp -# --------------------------------------------------------------------------- - - -def test_convert_2d_map_to_exact_size_pil_and_publishes_dims() -> None: - arr = np.linspace(0.0, 1.0, 64 * 32, dtype=np.float32).reshape(64, 32) - out = ConvertToImageOp(colormap="gray", width=128, height=256)(_sample(arr)) - assert isinstance(out.input, Image.Image) - assert out.input.size == (128, 256) - assert out.meta["image_width_px"] == 128 - assert out.meta["image_height_px"] == 256 - - -def test_convert_max_size_path_bounds_longest_side() -> None: - out = ConvertToImageOp(max_size=256)(_sample(np.zeros((1000, 400), dtype=np.float32))) - assert max(out.input.size) == 256 - # Dims are published from the actual rendered raster. - assert out.meta["image_width_px"] == out.input.width - assert out.meta["image_height_px"] == out.input.height - - -def test_convert_flip_vertical_mirrors_top_to_bottom() -> None: - m = np.zeros((10, 4), dtype=np.float32) - m[0, :] = 1.0 # row 0 bright - noflip = np.asarray(ConvertToImageOp(colormap="gray", flip_vertical=False)(_sample(m)).input.convert("L")) - flip = np.asarray(ConvertToImageOp(colormap="gray", flip_vertical=True)(_sample(m)).input.convert("L")) - assert noflip[0].mean() > noflip[-1].mean(), "no-flip: row 0 stays at the top" - assert flip[-1].mean() > flip[0].mean(), "flip: row 0 moves to the bottom" - - -def test_convert_colormap_gray_is_monochrome_color_is_not() -> None: - arr = np.linspace(0.0, 1.0, 100, dtype=np.float32).reshape(10, 10) - gray = np.asarray(ConvertToImageOp(colormap="gray", width=16, height=16)(_sample(arr)).input) - color = np.asarray(ConvertToImageOp(colormap="viridis", width=16, height=16)(_sample(arr)).input) - assert np.array_equal(gray[..., 0], gray[..., 1]) and np.array_equal(gray[..., 1], gray[..., 2]) - assert not np.array_equal(color[..., 0], color[..., 1]) - - -def test_convert_accepts_torch_chw_tensor() -> None: - out = ConvertToImageOp(width=64, height=48)(_sample(torch.rand(3, 100, 200))) - assert isinstance(out.input, Image.Image) - assert out.input.size == (64, 48) - - -def test_convert_accepts_pil_passthrough() -> None: - out = ConvertToImageOp(width=20, height=20)(_sample(Image.new("RGB", (8, 8), color=(10, 20, 30)))) - assert isinstance(out.input, Image.Image) - assert out.input.size == (20, 20) - - -# --------------------------------------------------------------------------- -# value_to_image / sample_to_image — generic, modality-agnostic preview -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "make_input", - [ - lambda: Image.new("L", (12, 10)), # PIL greyscale - lambda: Image.new("RGB", (12, 10)), # PIL RGB - lambda: np.random.rand(10, 12).astype(np.float32), # 2-D float -> colormap - lambda: (np.random.rand(10, 12, 3) * 255).astype(np.uint8), # 3-D HWC uint8 - lambda: np.random.rand(3, 10, 12).astype(np.float32), # 3-D CHW -> transposed - lambda: np.random.rand(10, 12) > 0.5, # boolean mask - lambda: np.random.rand(10, 12, 1).astype(np.float32), # singleton channel - lambda: np.random.rand(10, 12, 4).astype(np.float32), # RGBA -> drop alpha - lambda: torch.rand(3, 10, 12), # torch CHW tensor - ], -) -def test_sample_to_image_returns_hwc_uint8_rgb(make_input) -> None: # type: ignore[no-untyped-def] - img = sample_to_image(Sample(input=make_input())) - assert img.dtype == np.uint8 - assert img.ndim == 3 and img.shape[2] == 3 - - -def test_sample_to_image_falls_back_to_text_for_non_array() -> None: - img = sample_to_image(Sample(input=[(0, 1, 2, 3), (4, 5, 6, 7)])) - assert img.dtype == np.uint8 and img.ndim == 3 and img.shape[2] == 3 - - -def test_sample_to_image_bounds_longest_side() -> None: - img = sample_to_image(Sample(input=np.zeros((2000, 500), dtype=np.float32)), max_size=256) - assert max(img.shape[:2]) <= 256 - - -def test_sample_to_image_gray_is_monochrome_color_is_not() -> None: - arr = np.linspace(0.0, 1.0, 100).reshape(10, 10).astype(np.float32) - gray = sample_to_image(Sample(input=arr), colormap="gray") - color = sample_to_image(Sample(input=arr), colormap="viridis") - assert np.array_equal(gray[..., 0], gray[..., 1]) - assert not np.array_equal(color[..., 0], color[..., 1]) - - -def test_sample_to_image_flat_array_is_all_zero() -> None: - img = sample_to_image(Sample(input=np.full((8, 8), 5.0, dtype=np.float32)), colormap="gray") - assert int(img.max()) == 0 - - -def test_value_to_image_renders_an_arbitrary_value() -> None: - img = value_to_image(np.eye(12, dtype=bool), colormap="gray") - assert img.dtype == np.uint8 and img.ndim == 3 and img.shape[2] == 3 - - -def test_sample_to_image_delegates_to_value_to_image() -> None: - arr = np.linspace(0.0, 1.0, 64).reshape(8, 8).astype(np.float32) - assert np.array_equal(sample_to_image(Sample(input=arr)), value_to_image(arr)) - - -# --------------------------------------------------------------------------- -# NormalizeToUint8Op -# --------------------------------------------------------------------------- - - -def test_normalize_to_uint8_auto_minmax_spans_full_range() -> None: - arr = np.linspace(-3.0, 7.0, 100, dtype=np.float32).reshape(10, 10) - out = NormalizeToUint8Op.normalize_to_uint8(arr) - assert out.dtype == np.uint8 - assert int(out.min()) == 0 and int(out.max()) == 255 - - -def test_normalize_to_uint8_fixed_range_clamps_outside() -> None: - arr = np.array([[-10.0, 0.0, 5.0, 20.0]], dtype=np.float32) - out = NormalizeToUint8Op.normalize_to_uint8(arr, vmin=0.0, vmax=10.0) - # -10 and 0 clamp to 0; 5 is mid (≈127); 20 clamps to 255. - assert list(out.ravel()) == [0, 0, 127, 255] - - -def test_normalize_to_uint8_flat_array_is_all_zero() -> None: - out = NormalizeToUint8Op.normalize_to_uint8(np.full((4, 4), 9.0, dtype=np.float32)) - assert int(out.max()) == 0 - - -def test_normalize_to_uint8_handles_non_finite() -> None: - arr = np.array([[0.0, np.nan, np.inf, -np.inf, 4.0]], dtype=np.float32) - out = NormalizeToUint8Op.normalize_to_uint8(arr) - # NaN/-inf fold to the low bound (0), +inf to the high bound (4 → 255). - assert out[0, 0] == 0 and out[0, 1] == 0 and out[0, 3] == 0 - assert out[0, 2] == 255 and out[0, 4] == 255 - - -def test_normalize_to_uint8_op_converts_sample_input() -> None: - arr = np.linspace(0.0, 1.0, 64, dtype=np.float32).reshape(8, 8) - out = NormalizeToUint8Op()(Sample(input=arr)) - assert out.input.dtype == np.uint8 and out.input.shape == (8, 8) - - -def test_normalize_to_uint8_op_accepts_torch_tensor() -> None: - out = NormalizeToUint8Op()(Sample(input=torch.linspace(0, 1, 16).reshape(4, 4))) - assert isinstance(out.input, np.ndarray) and out.input.dtype == np.uint8 - - -def test_normalize_to_uint8_op_rejects_inverted_range() -> None: - with pytest.raises(ValueError, match="vmin must be < vmax"): - NormalizeToUint8Op(vmin=10.0, vmax=1.0)(Sample(input=np.zeros((2, 2), dtype=np.float32))) - - -# --------------------------------------------------------------------------- -# Colormap closed-Literal contract -# --------------------------------------------------------------------------- - - -def test_colormaps_tuple_is_the_literal_set() -> None: - assert COLORMAPS == get_args(Colormap) - assert "viridis" in COLORMAPS and "gray" in COLORMAPS - - -def test_every_colormap_in_the_literal_set_renders() -> None: - spec_u8 = np.linspace(0, 255, 64, dtype=np.uint8).reshape(8, 8) - for cmap in COLORMAPS: - img = _apply_colormap(spec_u8, cmap) - assert img.mode == "RGB" and img.size == (8, 8) - - -# --------------------------------------------------------------------------- -# select_channel — reduce an arbitrary array/tensor to a 2-D map for one channel -# --------------------------------------------------------------------------- - - -def test_select_channel_2d_passthrough() -> None: - arr = np.arange(12, dtype=np.float32).reshape(3, 4) - out = select_channel(arr, channel=-1) - assert out.shape == (3, 4) - np.testing.assert_array_equal(out, arr) - - -def test_select_channel_1d_becomes_strip() -> None: - out = select_channel(np.arange(5, dtype=np.float32)) - assert out.shape == (1, 5) - - -def test_select_channel_scalar_becomes_cell() -> None: - assert select_channel(np.float32(3.0)).shape == (1, 1) - - -def test_select_channel_chw_picks_plane() -> None: - # 3 channels first (smallest axis) → channel 1 is the middle plane. - arr = np.stack([np.full((4, 5), c, dtype=np.float32) for c in range(3)], axis=0) - out = select_channel(arr, channel=1) - assert out.shape == (4, 5) - assert float(out.mean()) == 1.0 - - -def test_select_channel_hwc_picks_plane() -> None: - arr = np.stack([np.full((4, 5), c, dtype=np.float32) for c in range(3)], axis=-1) - out = select_channel(arr, channel=2) - assert out.shape == (4, 5) - assert float(out.mean()) == 2.0 - - -def test_select_channel_all_is_mean_across_channels() -> None: - arr = np.stack([np.zeros((4, 5), dtype=np.float32), np.full((4, 5), 4.0, dtype=np.float32)], axis=0) - out = select_channel(arr, channel=-1) - assert out.shape == (4, 5) - assert float(out.mean()) == 2.0 # mean of {0, 4} - - -def test_select_channel_out_of_range_clamps() -> None: - # Spatial dims (5×4) larger than the channel count (3), so the smallest-axis heuristic - # unambiguously identifies axis 0 as channels (the channels-are-fewest assumption). - arr = np.stack([np.full((5, 4), c, dtype=np.float32) for c in range(3)], axis=0) - # channel 99 clamps to the last channel (index 2). - assert float(select_channel(arr, channel=99).mean()) == 2.0 - - -def test_select_channel_complex_uses_magnitude() -> None: - arr = np.array([[3 + 4j, 0]], dtype=np.complex64) # |3+4j| = 5 - out = select_channel(arr) - assert out.shape == (1, 2) - assert float(out[0, 0]) == 5.0 - - -def test_select_channel_torch_tensor() -> None: - out = select_channel(torch.arange(6, dtype=torch.float32).reshape(2, 3)) - assert isinstance(out, np.ndarray) and out.shape == (2, 3) - - -def test_select_channel_non_array_yields_unit_map() -> None: - assert select_channel("not an array").shape == (1, 1) - - -def test_channel_count() -> None: - assert channel_count(np.zeros((4, 5), dtype=np.float32)) == 1 # 2-D → 1 - assert channel_count(np.zeros((3, 4, 5), dtype=np.float32)) == 3 # CHW - assert channel_count(np.zeros((4, 5, 3), dtype=np.float32)) == 3 # HWC - assert channel_count("not an array") == 0 - - -# --------------------------------------------------------------------------- -# array_histogram — bin an array's values + summary statistics -# --------------------------------------------------------------------------- - - -def test_array_histogram_shape_and_stats() -> None: - arr = np.linspace(0.0, 1.0, 100, dtype=np.float32) - hist = array_histogram(arr, bins=10) - assert len(hist["counts"]) == 10 - assert len(hist["bin_edges"]) == 11 # bins + 1 - assert sum(hist["counts"]) == hist["count"] == 100 - assert hist["min"] == 0.0 and hist["max"] == 1.0 - assert hist["channels"] == 1 - assert abs(hist["mean"] - 0.5) < 1e-3 - - -def test_array_histogram_excludes_non_finite() -> None: - arr = np.array([0.0, 1.0, np.nan, np.inf, -np.inf, 2.0], dtype=np.float32) - hist = array_histogram(arr, bins=4) - # Only the 3 finite values (0, 1, 2) are counted; stats are finite. - assert hist["count"] == 3 - assert sum(hist["counts"]) == 3 - assert hist["min"] == 0.0 and hist["max"] == 2.0 - assert np.isfinite(hist["mean"]) and np.isfinite(hist["std"]) - - -def test_array_histogram_all_nan_is_empty_but_well_formed() -> None: - hist = array_histogram(np.full((4,), np.nan, dtype=np.float32), bins=8) - assert hist["count"] == 0 - assert hist["counts"] == [0] * 8 - assert len(hist["bin_edges"]) == 9 - assert hist["min"] is None and hist["max"] is None and hist["mean"] is None and hist["std"] is None - - -def test_array_histogram_flat_array_bins_into_first_bin() -> None: - hist = array_histogram(np.full((10,), 5.0, dtype=np.float32), bins=4) - assert hist["count"] == 10 - assert sum(hist["counts"]) == 10 - assert hist["min"] == 5.0 and hist["max"] == 5.0 - - -def test_array_histogram_single_channel_vs_all() -> None: - # channel 0 is all zeros, channel 1 is all ones. - arr = np.stack([np.zeros((4, 4), dtype=np.float32), np.ones((4, 4), dtype=np.float32)], axis=0) - only0 = array_histogram(arr, bins=4, channel=0) - assert only0["count"] == 16 and only0["min"] == 0.0 and only0["max"] == 0.0 - allc = array_histogram(arr, bins=4, channel=-1) - assert allc["count"] == 32 and allc["min"] == 0.0 and allc["max"] == 1.0 - - -def test_array_histogram_non_array_is_empty() -> None: - hist = array_histogram("text", bins=8) - assert hist["count"] == 0 and hist["channels"] == 0 - - -@pytest.mark.parametrize( - "arr", - [ - np.tile(np.arange(256, dtype=np.uint8), (256, 1)), # 256x256 uint8 gray gradient (65536 px) - np.linspace(-120.0, 0.0, 1024 * 512, dtype=np.float32), # large float32 dB spectrogram - (np.random.RandomState(0).rand(512, 512) * 255).astype(np.float32), # 262144 px random gray - ], -) -def test_array_histogram_large_array_does_not_raise(arr: np.ndarray) -> None: - # Regression: numpy 2.2.x's uniform-bins fast path (`bins=, range=(lo,hi)`) block-accumulates - # via np.bincount for arrays >65536 elements and miscomputes the bincount length on the workspace - # build — `n += bincount(...)` raised "operands could not be broadcast together with shapes - # (256,) (257,) (256,)" on any real image. array_histogram uses explicit linspace edges to avoid it. - finite = arr[np.isfinite(arr)] - hist = array_histogram(arr, bins=256) - assert len(hist["counts"]) == 256 and len(hist["bin_edges"]) == 257 - assert sum(hist["counts"]) == hist["count"] == finite.size # every value still counted - - -# --------------------------------------------------------------------------- # -# confusion_matrix_payload — the math behind FluxStudio's Confusion Matrix viewer. -# --------------------------------------------------------------------------- # - - -def test_confusion_matrix_payload_counts_and_class_names() -> None: - m = np.array([[50, 2, 1], [3, 47, 0], [0, 1, 49]]) - p = confusion_matrix_payload(m, class_names=["cat", "dog", "fox"]) - assert p["n_classes"] == 3 and p["total"] == 153 - assert p["counts"] == [[50, 2, 1], [3, 47, 0], [0, 1, 49]] - assert p["class_names"] == ["cat", "dog", "fox"] - - -def test_confusion_matrix_payload_normalizations() -> None: - p = confusion_matrix_payload([[8, 2], [0, 10]]) - # true = row-normalized (each true-class row sums to 1) - assert p["normalized"]["true"] == [[0.8, 0.2], [0.0, 1.0]] - # pred = column-normalized (each predicted-class column sums to 1) - assert p["normalized"]["pred"] == [ - [1.0, pytest.approx(0.166667, abs=1e-5)], - [0.0, pytest.approx(0.833333, abs=1e-5)], - ] - # all = total-normalized - assert p["normalized"]["all"][0][0] == pytest.approx(8 / 20) - - -def test_confusion_matrix_payload_zero_row_is_none_not_nan() -> None: - # A class with no samples (empty row) normalizes to None (undefined), never 0/0 = NaN. - p = confusion_matrix_payload([[0, 0], [1, 3]]) - assert p["normalized"]["true"][0] == [None, None] - assert json.dumps(p, allow_nan=False) # JSON-safe: no bare NaN tokens - - -def test_confusion_matrix_payload_defaults_to_index_labels() -> None: - p = confusion_matrix_payload([[1, 0], [0, 1]]) - assert p["class_names"] == ["0", "1"] - - -def test_confusion_matrix_payload_pads_or_trims_class_names_to_n() -> None: - assert confusion_matrix_payload([[1, 0], [0, 1]], class_names=["only"])["class_names"] == ["only", "1"] - assert confusion_matrix_payload([[1, 0], [0, 1]], class_names=["a", "b", "c"])["class_names"] == ["a", "b"] - - -def test_confusion_matrix_payload_non_square_is_well_formed() -> None: - p = confusion_matrix_payload(np.zeros((2, 3))) - assert p["n_classes"] == 0 and "message" in p - p2 = confusion_matrix_payload("not a matrix") - assert p2["n_classes"] == 0 - - -def test_confusion_matrix_payload_accepts_torch_tensor() -> None: - p = confusion_matrix_payload(torch.tensor([[5, 1], [0, 4]])) - assert p["counts"] == [[5, 1], [0, 4]] and p["n_classes"] == 2 - - -# --------------------------------------------------------------------------- # -# confusion_matrices_payload — extract EVERY confusion matrix from a metrics result. -# --------------------------------------------------------------------------- # - - -def test_confusion_matrices_payload_extracts_all_square_2d_entries() -> None: - # A full metrics dict: scalars + a 1-D vector + TWO confusion matrices. Only the square-2D - # entries are extracted, in dict order, each tagged with its metric name. - metrics = { - "test/acc": 0.93, - "test/cm_a": [[5, 1], [0, 4]], - "test/per_class": [0.9, 0.8], # 1-D vector — NOT a confusion matrix - "test/cm_b": [[10, 2, 1], [0, 9, 1], [1, 0, 8]], - } - payloads = confusion_matrices_payload(metrics, class_names=["a", "b", "c"]) - assert [(p["name"], p["n_classes"]) for p in payloads] == [("test/cm_a", 2), ("test/cm_b", 3)] - # class_names are trimmed per matrix (cm_a has only 2 classes). - assert payloads[0]["class_names"] == ["a", "b"] - assert json.dumps(payloads, allow_nan=False) # JSON-safe - - -def test_confusion_matrices_payload_none_when_no_square_metric() -> None: - assert confusion_matrices_payload({"acc": 0.9, "vec": [1, 2, 3]}) == [] - - -def test_confusion_matrices_payload_bare_matrix_is_one_named_default() -> None: - payloads = confusion_matrices_payload([[1, 0], [0, 1]]) - assert [p["name"] for p in payloads] == ["confusion_matrix"] - - -def test_confusion_matrices_payload_accepts_tensor_values() -> None: - payloads = confusion_matrices_payload({"cm": torch.tensor([[5, 1], [0, 4]])}) - assert payloads[0]["name"] == "cm" and payloads[0]["counts"] == [[5, 1], [0, 4]] - - -# --------------------------------------------------------------------------- -# draw_text — render text onto an image / a fresh canvas -# --------------------------------------------------------------------------- - - -def test_text_positions_is_the_literal_set() -> None: - assert TEXT_POSITIONS == get_args(TextPosition) - assert "center" in TEXT_POSITIONS and "top-left" in TEXT_POSITIONS and len(TEXT_POSITIONS) == 9 - - -def test_draw_text_blank_canvas_dims_and_dtype() -> None: - img = draw_text("Hi", None, width=200, height=80, background="black", color="white") - assert img.shape == (80, 200, 3) and img.dtype == np.uint8 - assert int((img > 0).sum()) > 0 # white text drawn on the black canvas - - -def test_draw_text_onto_existing_image_preserves_dims_and_copies() -> None: - base = np.zeros((64, 128, 3), dtype=np.uint8) - out = draw_text("label", base, color="red", position="center") - assert out.shape == (64, 128, 3) - assert (out[..., 0] > 0).any() # red text pixels present - assert int(base.sum()) == 0 # the input image is not mutated (drawn on a copy) - - -def test_draw_text_positions_place_block_differently() -> None: - tl = draw_text("X", None, width=120, height=120, position="top-left") - br = draw_text("X", None, width=120, height=120, position="bottom-right") - assert tl[:60].sum() > tl[60:].sum() # top-left lights the upper half - assert br[60:].sum() > br[:60].sum() # bottom-right lights the lower half - - -def test_draw_text_wrap_uses_more_vertical_lines() -> None: - long = "alpha bravo charlie delta echo foxtrot golf hotel india juliet" - - def text_row_span(im: np.ndarray) -> int: - rows = np.where((im > 0).any(axis=(1, 2)))[0] - return int(rows.max() - rows.min() + 1) if rows.size else 0 - - wrapped = draw_text(long, None, width=120, height=240, wrap=True, position="top-left") - nowrap = draw_text(long, None, width=120, height=240, wrap=False, position="top-left") - # Wrapping spreads the text over more rows than a single (unwrapped) line. - assert text_row_span(wrapped) > text_row_span(nowrap) - - -def test_draw_text_accepts_torch_tensor_image() -> None: - out = draw_text("t", torch.zeros(3, 32, 48)) # CHW float tensor - assert isinstance(out, np.ndarray) and out.ndim == 3 and out.shape[2] == 3 diff --git a/tests/test_joint.py b/tests/test_joint.py deleted file mode 100644 index 9fd4d06..0000000 --- a/tests/test_joint.py +++ /dev/null @@ -1,107 +0,0 @@ -from typing import Any, Iterator, List - -import confluid # type: ignore[import-not-found] - -from sampleflux.core import Flux -from sampleflux.sample import Sample - - -@confluid.configurable -class MockSource: - """A configurable data source for testing that doesn't dump the full payload.""" - - def __init__(self, name: str = "test") -> None: - self.name = name - self._data: List[Any] = [] - - def set_data(self, data: List[Any]) -> None: - self._data = data - - def __iter__(self) -> Iterator[Sample]: - for item in self._data: - yield Sample.from_any(item) - - def __len__(self) -> int: - return len(self._data) - - -def multiply(data: Any, factor: float = 1.0) -> Any: - return data * factor - - -def add(data: Any, val: float = 0.0) -> Any: - return data + val - - -def test_joint_flux_logic() -> None: - """Verify that JointFlux aggregates streams and preserves per-source ops.""" - src_a = MockSource(name="src_a") - src_a.set_data([1.0, 2.0]) - flux_a = Flux(src_a).map(multiply, factor=10.0) - - src_b = MockSource(name="src_b") - src_b.set_data([3.0, 4.0]) - flux_b = Flux(src_b).map(add, val=100.0) - - joint = Flux.joint([flux_a, flux_b]) - - assert len(joint) == 4 - results = joint.collect() - assert len(results) == 4 - assert results[0].input == 10.0 - assert results[1].input == 20.0 - assert results[2].input == 103.0 - assert results[3].input == 104.0 - - -def test_joint_serialization() -> None: - """Verify that a hierarchical JointFlux tree is fully serializable via DataSource definitions.""" - src_a = MockSource(name="source_a") - flux_a = Flux(src_a).map(multiply, factor=10.0) - - src_b = MockSource(name="source_b") - flux_b = Flux(src_b).map(add, val=5.0) - - # Wrap in a global pipeline - pipeline = Flux.joint([flux_a, flux_b]).map(multiply, factor=2.0) - - # 1. Serialize - yaml_state = confluid.dump(pipeline) - assert "!class:MockSource" in yaml_state - assert "name: source_a" in yaml_state - - # 2. Reconstruct — instances dump with () so they reload as live objects - new_pipeline = confluid.load(yaml_state) - - # 3. Manually provide data to the reconstructed sources - new_pipeline.source.fluxes[0].source.set_data([1.0]) - new_pipeline.source.fluxes[1].source.set_data([2.0]) - - results = list(new_pipeline) - # (1 * 10) * 2 = 20 - # (2 + 5) * 2 = 14 - assert results[0].input == 20.0 - assert results[1].input == 14.0 - - -def test_joint_parallel_execution() -> None: - """Verify that JointFlux works with the parallel engine.""" - src_a = MockSource(name="a") - src_a.set_data([1.0] * 5) - flux_a = Flux(src_a).map(multiply, factor=10.0) - - src_b = MockSource(name="b") - src_b.set_data([2.0] * 5) - flux_b = Flux(src_b).map(add, val=5.0) - - # Run joint stream in parallel - pipeline = Flux.joint([flux_a, flux_b]).parallel(workers=2) - - results = pipeline.collect() - assert len(results) == 10 - # First 5: 1.0 * 10 = 10.0 - for i in range(5): - assert results[i].input == 10.0 - # Next 5: 2.0 + 5 = 7.0 - for i in range(5, 10): - assert results[i].input == 7.0 diff --git a/tests/test_kinds.py b/tests/test_kinds.py deleted file mode 100644 index 30f330f..0000000 --- a/tests/test_kinds.py +++ /dev/null @@ -1,580 +0,0 @@ -"""Tests for op-kind introspection (`sampleflux.kinds`) and the native multi-type engine.""" - -from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, cast - -import numpy as np -import pytest -from confluid import configurable - -from sampleflux.core import Flux -from sampleflux.kinds import SAMPLE_KINDS, Input, OpContract, Target, classify_carrier, op_contract -from sampleflux.sample import InputMeta, Pair, Sample, TargetMeta - -# --------------------------------------------------------------------------- -# Fixture ops (module-level so they pickle for spawn parity) -# --------------------------------------------------------------------------- - - -@configurable -class SampleOp: - """A classic annotated Sample op.""" - - def __call__(self, sample: Sample) -> Optional[Sample]: - return sample._replace(input=sample.input + 1) - - -@configurable -class PairOp: - """A metadata-free pair op: works on (input, target) tuples.""" - - def __call__(self, pair: Tuple[Any, Any]) -> Tuple[Any, Any]: - data, label = pair - return data * 2, label - - -@configurable -class UntypedOp: - """No annotations at all — works on anything (today's behavior).""" - - def __call__(self, sample): # type: ignore[no-untyped-def] - return sample - - -@configurable -class ExpandingOp: - """A 1→N op, detected from the Iterator return annotation.""" - - def __call__(self, sample: Sample) -> Iterator[Sample]: - yield sample - yield sample - - -@configurable -class ExpandingIterableOp: - """A 1→N op via Iterable[...].""" - - def __call__(self, sample: Sample) -> Iterable[Sample]: - return [sample, sample] - - -@configurable -class OverriddenOp: - """Introspection-opaque op relying on explicit class-attr overrides.""" - - SAMPLE_KIND_IN = "pair" - SAMPLE_KIND_OUT = "pair" - EXPANDS = False - - def __call__(self, *args): # type: ignore[no-untyped-def] - return args[0] - - -class StringAnnotatedOp: - """PEP-563-style string annotations must resolve (get_type_hints).""" - - def __call__(self, sample: "Sample") -> "Sample": - return sample - - -# --------------------------------------------------------------------------- -# classify_carrier / op_contract -# --------------------------------------------------------------------------- - - -class TestClassify: - def test_kinds_taxonomy_is_closed(self) -> None: - assert SAMPLE_KINDS == ( - "sample", - "pair", - "input", - "target", - "metadata", - "input_meta", - "target_meta", - "value", - "any", - ) - - def test_classify_carrier(self) -> None: - assert classify_carrier(Sample(1)) == "sample" - assert classify_carrier((np.zeros(3), 7)) == "pair" - assert classify_carrier(np.zeros(3)) == "value" - assert classify_carrier((1, 2, 3)) == "value" # only 2-tuples are pairs - - -class TestOpContract: - def test_sample_op(self) -> None: - assert op_contract(SampleOp()) == OpContract("sample", "sample", False) - - def test_pair_op(self) -> None: - assert op_contract(PairOp()) == OpContract("pair", "pair", False) - - def test_untyped_op_is_any(self) -> None: - assert op_contract(UntypedOp()) == OpContract("any", "any", False) - - def test_expanding_iterator_and_iterable(self) -> None: - assert op_contract(ExpandingOp()) == OpContract("sample", "sample", True) - assert op_contract(ExpandingIterableOp()) == OpContract("sample", "sample", True) - - def test_class_attr_overrides(self) -> None: - assert op_contract(OverriddenOp()) == OpContract("pair", "pair", False) - - def test_string_annotations_resolve(self) -> None: - assert op_contract(StringAnnotatedOp()).accepts == "sample" - - def test_pair_return_is_not_expansion(self) -> None: - # A Tuple return is a PAIR carrier, never a 1→N expansion. - contract = op_contract(PairOp()) - assert contract.produces == "pair" and contract.expands is False - - def test_introspection_failure_degrades_to_any(self) -> None: - class Broken: - pass - - # Inject unresolvable string annotations dynamically (mypy-safe: no fake name in source). - def _call(self, x): # type: ignore[no-untyped-def] - return x - - _call.__annotations__ = {"x": "NoSuchType", "return": "NoSuchType"} - Broken.__call__ = _call # type: ignore[method-assign, assignment] - assert op_contract(Broken()) == OpContract("any", "any", False) - - -# --------------------------------------------------------------------------- -# Native multi-type engine -# --------------------------------------------------------------------------- - - -class TestNativeFlux: - def test_pair_source_through_pair_op_stays_pairs(self) -> None: - pairs = [(np.full(2, float(i)), i) for i in range(3)] - flux = Flux(source=pairs, ops=[PairOp()], native=True) - out = list(flux) - assert all(isinstance(item, tuple) and len(item) == 2 for item in out) - assert out[1][0][0] == 2.0 and out[1][1] == 1 - - def test_pair_source_promoted_for_sample_op_sticky(self) -> None: - pairs = [(float(i), i) for i in range(3)] - flux = Flux(source=pairs, ops=[SampleOp()], native=True) - out = list(flux) - assert all(isinstance(item, Sample) for item in out) # promotion is sticky - assert [s.input for s in out] == [1.0, 2.0, 3.0] - assert all(s.meta == {} for s in out) - - def test_mixed_chain_pair_then_sample_op(self) -> None: - pairs = [(float(i), i) for i in range(3)] - flux = Flux(source=pairs, ops=[PairOp(), SampleOp()], native=True) - out = list(flux) - # PairOp doubled the value natively, then SampleOp promoted and added 1. - assert [s.input for s in out] == [1.0, 3.0, 5.0] - - def test_pair_op_on_sample_carrier_preserves_metadata(self) -> None: - samples = [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(3)] - flux = Flux(source=samples, ops=[PairOp()], native=True) - out = list(flux) - assert [s.input for s in out] == [0.0, 2.0, 4.0] - assert [s.meta["idx"] for s in out] == [0, 1, 2] # metadata rides through the pair view - - def test_untyped_op_receives_carrier_verbatim(self) -> None: - seen: list = [] - - @configurable - class Probe: - def __call__(self, x): # type: ignore[no-untyped-def] - seen.append(type(x).__name__) - return x - - list(Flux(source=[(1.0, 2)], ops=[Probe()], native=True)) - assert seen == ["tuple"] # NOT coerced - - def test_default_mode_unchanged(self) -> None: - # native=False (the default): 2-tuples coerce to Samples exactly as before. - out = list(Flux(source=[(1.0, 2)], ops=[SampleOp()])) - assert isinstance(out[0], Sample) and out[0].input == 2.0 - - def test_native_spawn_parallel_parity(self) -> None: - pairs = [(float(i), i) for i in range(4)] - seq = list(Flux(source=list(pairs), ops=[PairOp()], native=True)) - par = list(Flux(source=list(pairs), ops=[PairOp()], native=True).parallel(2)) - assert [(a[0], a[1]) for a in seq] == [(b[0], b[1]) for b in par] - - def test_native_getitem(self) -> None: - pairs = [(float(i), i) for i in range(4)] - flux = Flux(source=pairs, ops=[PairOp()], native=True) - item = flux[2] - assert item[0] == 4.0 and item[1] == 2 - - def test_native_filter_drop(self) -> None: - @configurable - class DropEven: - def __call__(self, pair: Tuple[Any, Any]) -> Optional[Tuple[Any, Any]]: - return None if pair[1] % 2 == 0 else pair - - out = list(Flux(source=[(0.0, 0), (1.0, 1), (2.0, 2)], ops=[DropEven()], native=True)) - assert [p[1] for p in out] == [1] - - -# --------------------------------------------------------------------------- -# Collate registry -# --------------------------------------------------------------------------- - - -class TestCollate: - def test_sample_default_list_form_metadata(self) -> None: - import torch - - from sampleflux.collate import collate - - batch = [Sample(input=torch.ones(2) * i, target=torch.tensor(i), metadata={"i": i}) for i in range(3)] - out = collate(batch) - assert isinstance(out, Sample) and out.is_batched - assert out.input.shape == (3, 2) and out.batch_meta[2]["i"] == 2 - - def test_pair_default(self) -> None: - from sampleflux.collate import collate - - data, labels = collate([(np.ones(2), 1), (np.zeros(2), 0)]) - assert data.shape == (2, 2) and list(labels) == [1, 0] - - def test_value_default(self) -> None: - from sampleflux.collate import collate - - out = collate([np.ones(2), np.zeros(2)]) - assert out.shape == (2, 2) - - def test_explicit_key_and_registration(self) -> None: - from sampleflux.collate import collate, get_collate, register_collate, registered_collates - - @register_collate("yolo_test") - def yolo_collate(items): # type: ignore[no-untyped-def] - return list(items) - - assert "yolo_test" in registered_collates() - assert get_collate("yolo_test") is yolo_collate - assert collate([(1, 2)], key="yolo_test") == [(1, 2)] - - def test_unknown_key_names_known(self) -> None: - from sampleflux.collate import get_collate - - with pytest.raises(KeyError, match="known:"): - get_collate("nope_nothing") - - def test_empty_batch_raises(self) -> None: - from sampleflux.collate import collate - - with pytest.raises(ValueError, match="empty"): - collate([]) - - def test_stack_fallback_to_list(self) -> None: - from sampleflux.collate import collate - - out = collate(["a", "b"], key="value") - assert out == ["a", "b"] - - -# --------------------------------------------------------------------------- -# The field-scope grid + call styles (the (input, target, metadata) taxonomy) -# --------------------------------------------------------------------------- - - -@configurable -class BareInputOp: - """Processes ONLY the input value (any array/tensor/dict), declared via the Input alias.""" - - def __call__(self, x: Input): # type: ignore[no-untyped-def] - return x * 2 - - -@configurable -class BareTargetOp: - """Processes ONLY the target value.""" - - def __call__(self, t: Target): # type: ignore[no-untyped-def] - return t + 100 - - -@configurable -class UnpackedPairOp: - """transform(input, target) — the classic AI signature, unpacked.""" - - def __call__(self, input, target): # type: ignore[no-untyped-def] - return input * 2, target + 1 - - -@configurable -class UnpackedInputMetaOp: - """transform(input, metadata) — input with its metadata, unpacked.""" - - def __call__(self, input, metadata): # type: ignore[no-untyped-def] - metadata["seen"] = True - return input + 1, metadata - - -@configurable -class UnpackedTargetMetaOp: - """transform(target, metadata) — target side selected by the first param name.""" - - def __call__(self, target, metadata): # type: ignore[no-untyped-def] - return target * 10, {**metadata, "t": True} - - -@configurable -class UnpackedSampleOp: - """transform(input, target, metadata) — the full triple, unpacked.""" - - def __call__(self, input, target, metadata): # type: ignore[no-untyped-def] - return input + 1, target + 1, {**metadata, "s": True} - - -@configurable -class PackedInputMetaOp: - """Packed InputMeta view — the op receives a named (input, metadata) object.""" - - def __call__(self, view: InputMeta) -> InputMeta: - meta = cast(Dict[str, Any], view.metadata) # per-sample ops always see the dict form - return InputMeta(view.input * 3, {**meta, "packed": True}) - - -@configurable -class PackedTargetMetaOp: - """Packed TargetMeta view.""" - - def __call__(self, view: TargetMeta) -> TargetMeta: - return TargetMeta(view.target - 1, view.metadata) - - -@configurable -class PackedNamedPairOp: - """Packed Pair view (the named 2-tuple form).""" - - def __call__(self, p: Pair) -> Pair: - return Pair(p.input + 0.5, p.target) - - -@configurable -class OptionalExtraArgOp: - """One REQUIRED param + optional extras — must stay single-argument (packed/any).""" - - def __call__(self, sample, extra=None): # type: ignore[no-untyped-def] - return sample - - -class TestGridContracts: - def test_bare_field_marks(self) -> None: - assert op_contract(BareInputOp()) == OpContract("input", "any", False, "packed") - assert op_contract(BareTargetOp()).accepts == "target" - - def test_unpacked_pair(self) -> None: - assert op_contract(UnpackedPairOp()) == OpContract("pair", "any", False, "unpacked", ("input", "target")) - - def test_unpacked_meta_variants_by_param_names(self) -> None: - assert op_contract(UnpackedInputMetaOp()) == OpContract( - "input_meta", "any", False, "unpacked", ("input", "metadata") - ) - assert op_contract(UnpackedTargetMetaOp()).accepts == "target_meta" - - def test_unpacked_sample_triple(self) -> None: - assert op_contract(UnpackedSampleOp()) == OpContract( - "sample", "any", False, "unpacked", ("input", "target", "metadata") - ) - - def test_packed_views(self) -> None: - assert op_contract(PackedInputMetaOp()).accepts == "input_meta" - assert op_contract(PackedInputMetaOp()).style == "packed" - assert op_contract(PackedTargetMetaOp()).accepts == "target_meta" - assert op_contract(PackedNamedPairOp()).accepts == "pair" - - def test_optional_extras_keep_single_arg_semantics(self) -> None: - # Required arity 1 -> packed/any: op(sample) exactly as today. - assert op_contract(OptionalExtraArgOp()) == OpContract("any", "any", False, "packed") - - def test_call_style_override(self) -> None: - class Opaque: - SAMPLE_KIND_IN = "pair" - CALL_STYLE = "unpacked" - - def __call__(self, *args): # type: ignore[no-untyped-def] - return args[0], args[1] - - contract = op_contract(Opaque()) - assert contract.accepts == "pair" and contract.style == "unpacked" - - def test_classify_carrier_views_before_tuple_rule(self) -> None: - assert classify_carrier(InputMeta(1, {"a": 1})) == "input_meta" - assert classify_carrier(TargetMeta(1, {})) == "target_meta" - assert classify_carrier(Pair(1, 2)) == "pair" - assert classify_carrier((1, 2)) == "pair" # the plain tuple stays a pair - - -class TestGridEngineBinding: - def _samples(self, n: int = 2) -> list: - return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] - - def test_bare_input_op_preserves_target_and_meta(self) -> None: - out = list(Flux(source=self._samples(), ops=[BareInputOp()])) - assert [s.input for s in out] == [0.0, 2.0] - assert [s.target for s in out] == [0, 1] - assert [s.meta["idx"] for s in out] == [0, 1] - - def test_bare_target_op(self) -> None: - out = list(Flux(source=self._samples(), ops=[BareTargetOp()])) - assert [s.target for s in out] == [100, 101] - assert [s.input for s in out] == [0.0, 1.0] - - def test_unpacked_pair_op_merges_back(self) -> None: - out = list(Flux(source=self._samples(), ops=[UnpackedPairOp()])) - assert [(s.input, s.target) for s in out] == [(0.0, 1), (2.0, 2)] - assert [s.meta["idx"] for s in out] == [0, 1] # metadata preserved - - def test_unpacked_input_meta_op(self) -> None: - out = list(Flux(source=self._samples(), ops=[UnpackedInputMetaOp()])) - assert [s.input for s in out] == [1.0, 2.0] - assert all(s.meta["seen"] is True for s in out) - assert [s.target for s in out] == [0, 1] # target untouched - - def test_unpacked_target_meta_op(self) -> None: - out = list(Flux(source=self._samples(), ops=[UnpackedTargetMetaOp()])) - assert [s.target for s in out] == [0, 10] - assert all(s.meta["t"] is True for s in out) - assert [s.input for s in out] == [0.0, 1.0] - - def test_unpacked_sample_op(self) -> None: - out = list(Flux(source=self._samples(), ops=[UnpackedSampleOp()])) - assert [(s.input, s.target) for s in out] == [(1.0, 1), (2.0, 2)] - assert all(s.meta["s"] is True and "idx" in s.meta for s in out) - - def test_packed_views_merge_back(self) -> None: - out = list(Flux(source=self._samples(), ops=[PackedInputMetaOp(), PackedTargetMetaOp()])) - assert [s.input for s in out] == [0.0, 3.0] - assert [s.target for s in out] == [-1, 0] - assert all(s.meta["packed"] is True for s in out) - - def test_packed_named_pair(self) -> None: - out = list(Flux(source=self._samples(), ops=[PackedNamedPairOp()])) - assert [s.input for s in out] == [0.5, 1.5] - assert [s.meta["idx"] for s in out] == [0, 1] - - def test_grid_chain_mixes_all_styles(self) -> None: - ops = [BareInputOp(), UnpackedPairOp(), PackedInputMetaOp(), SampleOp()] - out = list(Flux(source=self._samples(1), ops=ops)) - # 0.0 -> *2=0.0 -> pair(*2, +1)=(0.0, 1) -> *3=0.0 -> SampleOp(+1 input)=1.0 - assert out[0].input == 1.0 and out[0].target == 1 - assert out[0].meta["packed"] is True and out[0].meta["idx"] == 0 - - def test_none_drops_in_every_scope(self) -> None: - @configurable - class DropPair: - def __call__(self, input, target): # type: ignore[no-untyped-def] - return None - - assert list(Flux(source=self._samples(), ops=[DropPair()])) == [] - - def test_pair_scope_single_return_is_a_loud_error(self) -> None: - @configurable - class BadPair: - def __call__(self, input, target): # type: ignore[no-untyped-def] - return input # ambiguous — must be a 2-tuple / Sample / None - - with pytest.raises(TypeError, match="same arity"): - list(Flux(source=self._samples(1), ops=[BadPair()])) - - def test_native_bare_input_on_value_carrier_stays_value(self) -> None: - out = list(Flux(source=[1.0, 2.0], ops=[BareInputOp()], native=True)) - assert out == [2.0, 4.0] # no promotion — bare values stay bare - - def test_native_unpacked_pair_on_pair_carrier_stays_pair(self) -> None: - out = list(Flux(source=[(1.0, 1), (2.0, 2)], ops=[UnpackedPairOp()], native=True)) - assert out == [(2.0, 2), (4.0, 3)] - - def test_native_view_carrier_promotes_field_correct(self) -> None: - # An InputMeta carrier + a sample-op: from_any must NOT misread metadata as target. - out = list(Flux(source=[InputMeta(1.0, {"m": 1})], ops=[SampleOp()], native=True)) - assert out[0].input == 2.0 and out[0].target is None and out[0].meta == {"m": 1} - - def test_view_collate_defaults(self) -> None: - from sampleflux.collate import collate - - batch = collate([InputMeta(np.ones(2), {"i": 0}), InputMeta(np.zeros(2), {"i": 1})]) - assert isinstance(batch, InputMeta) and batch.input.shape == (2, 2) - metas = cast(List[Dict[str, Any]], batch.metadata) # batched form: list of per-item dicts - assert metas[1] == {"i": 1} - - -# --------------------------------------------------------------------------- -# Combination bindings — mixed views as separate arguments -# --------------------------------------------------------------------------- - - -@configurable -class DualViewOp: - """transform(InputMeta, TargetMeta) — both fields, each WITH its metadata.""" - - def __call__(self, im: InputMeta, tm: TargetMeta): # type: ignore[no-untyped-def] - meta = cast(Dict[str, Any], im.metadata) - return InputMeta(im.input * 2, {**meta, "im": True}), TargetMeta( - tm.target + 1, {**meta, "im": True, "tm": True} - ) - - -@configurable -class MixedMarkViewOp: - """transform(Input, TargetMeta) — a bare input value + the target with metadata.""" - - def __call__(self, x: Input, tm: TargetMeta): # type: ignore[no-untyped-def] - return x + 0.5, tm - - -@configurable -class MetadataOnlyOp: - """transform(metadata) — the metadata-only cell of the grid, declared by dict annotation.""" - - def __call__(self, m: dict) -> dict: - return {**m, "canonical": True} - - -class TestCombinationBindings: - def _samples(self, n: int = 2) -> list: - return [Sample(input=float(i), target=i, metadata={"idx": i}) for i in range(n)] - - def test_dual_view_contract(self) -> None: - contract = op_contract(DualViewOp()) - assert contract.bindings == ("input_meta", "target_meta") - assert contract.accepts == "sample" # covers input+target+metadata — the grid summary - assert contract.style == "unpacked" - - def test_dual_view_execution_merges_all_fields(self) -> None: - out = list(Flux(source=self._samples(), ops=[DualViewOp()])) - assert [s.input for s in out] == [0.0, 2.0] - assert [s.target for s in out] == [1, 2] - # last metadata-bearing element wins (it layered im's write too) - assert all(s.meta["im"] is True and s.meta["tm"] is True and "idx" in s.meta for s in out) - - def test_mixed_mark_and_view(self) -> None: - contract = op_contract(MixedMarkViewOp()) - assert contract.bindings == ("input", "target_meta") - out = list(Flux(source=self._samples(1), ops=[MixedMarkViewOp()])) - assert out[0].input == 0.5 and out[0].target == 0 and out[0].meta == {"idx": 0} - - def test_metadata_only_scope(self) -> None: - contract = op_contract(MetadataOnlyOp()) - assert contract.accepts == "metadata" and contract.style == "packed" - out = list(Flux(source=self._samples(1), ops=[MetadataOnlyOp()])) - assert out[0].meta == {"idx": 0, "canonical": True} - assert out[0].input == 0.0 and out[0].target == 0 # untouched - - def test_wrong_arity_return_is_a_loud_error(self) -> None: - @configurable - class Bad: - def __call__(self, im: InputMeta, tm: TargetMeta): # type: ignore[no-untyped-def] - return im # must be a 2-tuple / Sample / None - - with pytest.raises(TypeError, match="same arity"): - list(Flux(source=self._samples(1), ops=[Bad()])) - - def test_binding_names_override_positions(self) -> None: - @configurable - class TargetFirst: - def __call__(self, target, input): # type: ignore[no-untyped-def] - return target, input - - assert op_contract(TargetFirst()).bindings == ("target", "input") diff --git a/tests/test_labels.py b/tests/test_labels.py index 203612d..e6c3274 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -4,9 +4,9 @@ import pytest +from sampleflux import Label, Sample from sampleflux.labels import LabelMap -from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp -from sampleflux.sample import Sample +from sampleflux.ops.target import DecodeTarget, EncodeTarget # --------------------------------------------------------------------------- # Construction & lazy validation @@ -85,24 +85,24 @@ def test_from_label_names_empty_raises() -> None: def test_encode_op_encodes_target() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) op = lm.encode_op() - assert isinstance(op, EncodeTargetOp) - out = op(Sample(input=None, target="dog", metadata={})) - assert out.target == 1 + assert isinstance(op, EncodeTarget) + out = op(Sample({"y": Label("dog")}, roles={"y": "target"})) + assert out["y"].value == 1 def test_decode_op_inverts_encoding() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) op = lm.decode_op() - assert isinstance(op, DecodeTargetOp) - out = op(Sample(input=None, target=0, metadata={})) - assert out.target == "cat" + assert isinstance(op, DecodeTarget) + out = op(Sample({"y": Label(0)}, roles={"y": "target"})) + assert out["y"].value == "cat" def test_encode_op_ignore_unknown() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) op = lm.encode_op(ignore_unknown=True, default=-1) - out = op(Sample(input=None, target="fish", metadata={})) - assert out.target == -1 + out = op(Sample({"y": Label("fish")}, roles={"y": "target"})) + assert out["y"].value == -1 # --------------------------------------------------------------------------- diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index 8d03c02..18dc48b 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -14,10 +14,18 @@ from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp from sampleflux.ops.albumentations import AlbumentationsOp -from sampleflux.ops.numpy import ConnectedComponentsOp, StandardizeOp, ThresholdOp -from sampleflux.ops.target import DecodeTargetOp, EncodeTargetOp, MetadataToTargetOp -from sampleflux.ops.torch import StandardizeOp as TorchStandardizeOp -from sampleflux.ops.torch import ToTensorOp +from sampleflux.ops.configure import ConfigureOp +from sampleflux.ops.image import ConvertToImage +from sampleflux.ops.numpy import ConnectedComponents, Threshold +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole +from sampleflux.ops.target import ( + CocoToTorchVisionDetection, + DecodeTarget, + EncodeTarget, + MasksToDetectionBoxes, + MetadataToTarget, +) +from sampleflux.ops.torch import ToTensor from sampleflux.ops.torchvision import TorchvisionTransformOp from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import HuggingFaceSource @@ -28,14 +36,21 @@ JointFlux, FilterOp, WrappedOp, - StandardizeOp, - ThresholdOp, - ConnectedComponentsOp, - ToTensorOp, - TorchStandardizeOp, - MetadataToTargetOp, - EncodeTargetOp, - DecodeTargetOp, + Threshold, + ConnectedComponents, + ConvertToImage, + ToTensor, + MetadataToTarget, + EncodeTarget, + DecodeTarget, + CocoToTorchVisionDetection, + MasksToDetectionBoxes, + SetRole, + RenameField, + DropField, + CopyField, + SelectFields, + ConfigureOp, TransformChain, AlbumentationsOp, TorchvisionTransformOp, diff --git a/tests/test_ops.py b/tests/test_ops.py deleted file mode 100644 index 38860a6..0000000 --- a/tests/test_ops.py +++ /dev/null @@ -1,1292 +0,0 @@ -"""Tests for sampleflux.ops: torch and numpy variants.""" - -import os - -import numpy as np -import pytest -import torch -from PIL import Image - -from sampleflux.ops import ( - ConfigureOp, - CopyInputOp, - CopyMetadataOp, - CopySampleOp, - CopyTargetOp, - FormulaOp, - RescaleOp, - SqueezeOp, - StandardizeOp, - StashInputOp, - StashTargetOp, - SwapInputTargetOp, - ToTensorOp, - UnsqueezeOp, - UnstashInputOp, - UnstashTargetOp, -) -from sampleflux.ops import numpy as np_ops -from sampleflux.ops.numpy import MaxOp, ThresholdOp -from sampleflux.sample import Sample - -# --------------------------------------------------------------------------- -# ToTensorOp -# --------------------------------------------------------------------------- - - -class TestToTensorOp: - """Tests for ToTensorOp.""" - - def test_pil_image_with_normalize(self) -> None: - img = Image.fromarray(np.full((28, 28), 128, dtype=np.uint8)) - result = ToTensorOp(normalize=True)(Sample(input=img)) - assert isinstance(result.input, torch.Tensor) - assert result.input.dtype == torch.float32 - assert result.input.max() <= 1.0 - - def test_pil_image_without_normalize(self) -> None: - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - result = ToTensorOp(normalize=False)(Sample(input=img)) - assert isinstance(result.input, torch.Tensor) - assert result.input.dtype == torch.uint8 - assert result.input.max() == 200 - - def test_2d_array_adds_channel_dim(self) -> None: - arr = np.zeros((28, 28), dtype=np.uint8) - result = ToTensorOp(normalize=False)(Sample(input=arr)) - assert result.input.shape == (1, 28, 28) - - def test_3d_array_transposes(self) -> None: - arr = np.zeros((28, 28, 3), dtype=np.uint8) - result = ToTensorOp(normalize=False)(Sample(input=arr)) - assert result.input.shape == (3, 28, 28) - - def test_normalize_float_above_one(self) -> None: - arr = np.array([[128.0, 255.0]], dtype=np.float32) - result = ToTensorOp(normalize=True)(Sample(input=arr)) - assert result.input.max() == 1.0 - - def test_non_array_passthrough(self) -> None: - t = torch.tensor([1.0, 2.0]) - result = ToTensorOp(normalize=False)(Sample(input=t)) - assert torch.equal(result.input, t) - - def test_preserves_target_and_metadata(self) -> None: - arr = np.zeros((28, 28), dtype=np.uint8) - result = ToTensorOp()(Sample(input=arr, target=5, metadata={"k": "v"})) - assert result.target == 5 - assert result.meta == {"k": "v"} - - def test_mode_rgb_forces_three_channels_from_mixed_pil_modes(self) -> None: - # A mixed-mode image dataset (RGBA / grayscale / palette) → uniform 3-channel - # RGB for a fixed-channel model (e.g. torchvision Faster R-CNN's 3-ch normalize). - op = ToTensorOp(mode="RGB") - for mode, arr in ( - ("RGBA", np.zeros((8, 8, 4), dtype=np.uint8)), - ("L", np.zeros((8, 8), dtype=np.uint8)), - ("RGB", np.zeros((8, 8, 3), dtype=np.uint8)), - ): - out = op(Sample(input=Image.fromarray(arr, mode=mode))) - assert out.input.shape == (3, 8, 8), f"{mode} → {tuple(out.input.shape)}" - - def test_mode_none_leaves_channels_as_is(self) -> None: - # Default mode=None arrays the image verbatim — RGBA stays 4-channel. - rgba = Image.fromarray(np.zeros((8, 8, 4), dtype=np.uint8), mode="RGBA") - assert ToTensorOp()(Sample(input=rgba)).input.shape == (4, 8, 8) - - -# --------------------------------------------------------------------------- -# Torch RescaleOp -# --------------------------------------------------------------------------- - - -class TestRescaleOp: - """Tests for torch RescaleOp.""" - - def test_default_output_range(self) -> None: - tensor = torch.tensor([0.0, 128.0, 255.0]) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor)) - assert result.input[0] == 0.0 - assert abs(result.input[1] - 128.0 / 255.0) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_custom_input_range(self) -> None: - tensor = torch.tensor([10.0, 55.0, 100.0]) - result = RescaleOp(in_min=10.0, in_max=100.0)(Sample(input=tensor)) - assert abs(result.input[0] - 0.0) < 1e-6 - assert abs(result.input[1] - 0.5) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_custom_output_range(self) -> None: - tensor = torch.tensor([0.0, 0.5, 1.0]) - result = RescaleOp(in_min=0.0, in_max=1.0, out_min=10.0, out_max=20.0)(Sample(input=tensor)) - assert abs(result.input[0] - 10.0) < 1e-6 - assert abs(result.input[1] - 15.0) < 1e-6 - assert abs(result.input[2] - 20.0) < 1e-6 - - def test_clip_true_clamps(self) -> None: - tensor = torch.tensor([-50.0, 0.0, 128.0, 300.0]) - result = RescaleOp(in_min=0.0, in_max=255.0, clip=True)(Sample(input=tensor)) - assert result.input[0] == 0.0 - assert result.input[3] == 1.0 - - def test_clip_false_extrapolates(self) -> None: - tensor = torch.tensor([-255.0, 510.0]) - result = RescaleOp(in_min=0.0, in_max=255.0, clip=False)(Sample(input=tensor)) - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 2.0) < 1e-6 - - def test_uint8_converts_to_float(self) -> None: - tensor = torch.tensor([0, 128, 255], dtype=torch.uint8) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float32 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - tensor = torch.tensor([0.0, 255.0], dtype=torch.float64) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float64 - - def test_preserves_target_and_metadata(self) -> None: - tensor = torch.tensor([128.0]) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor, target=7, metadata={"key": "val"})) - assert result.target == 7 - assert result.meta == {"key": "val"} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="RescaleOp expects a torch.Tensor"): - RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=np.array([1, 2, 3]))) - - def test_validation_rejects_bad_input_range(self) -> None: - op = RescaleOp(in_min=10.0, in_max=10.0) # lazy: construction succeeds - with pytest.raises(ValueError, match="require in_min < in_max"): - op(Sample(input=torch.zeros(2))) - - def test_validation_rejects_bad_output_range(self) -> None: - op = RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) - with pytest.raises(ValueError, match="require out_min < out_max"): - op(Sample(input=torch.zeros(2))) - - def test_pipeline_to_tensor_then_rescale(self) -> None: - """Integration: ToTensorOp(normalize=False) -> RescaleOp().""" - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - sample = Sample(input=img) - sample = ToTensorOp(normalize=False)(sample) - sample = RescaleOp(in_min=0.0, in_max=255.0)(sample) - assert sample.input.dtype == torch.float32 - assert abs(sample.input.max().item() - 200.0 / 255.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Torch StandardizeOp -# --------------------------------------------------------------------------- - - -class TestStandardizeOp: - """Tests for torch StandardizeOp.""" - - def test_scalar_mean_and_std(self) -> None: - tensor = torch.tensor([2.0, 4.0, 6.0]) - result = StandardizeOp(mean=4.0, std=2.0)(Sample(input=tensor)) - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 0.0) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_per_channel_mean_and_std(self) -> None: - tensor = torch.ones(3, 2, 2) - tensor[0] *= 10.0 - tensor[1] *= 20.0 - tensor[2] *= 30.0 - result = StandardizeOp(mean=[10.0, 20.0, 30.0], std=[1.0, 1.0, 1.0])(Sample(input=tensor)) - assert torch.allclose(result.input, torch.zeros(3, 2, 2)) - - def test_uint8_converts_to_float(self) -> None: - tensor = torch.tensor([100, 200], dtype=torch.uint8) - result = StandardizeOp(mean=150.0, std=50.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float32 - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - tensor = torch.tensor([1.0, 2.0], dtype=torch.float64) - result = StandardizeOp(mean=0.0, std=1.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float64 - - def test_preserves_target_and_metadata(self) -> None: - tensor = torch.tensor([5.0]) - result = StandardizeOp(mean=0.0, std=1.0)(Sample(input=tensor, target=3, metadata={"a": 1})) - assert result.target == 3 - assert result.meta == {"a": 1} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="StandardizeOp expects a torch.Tensor"): - StandardizeOp(mean=0.0, std=1.0)(Sample(input=[1, 2, 3])) - - def test_1d_per_channel(self) -> None: - tensor = torch.tensor([10.0]) - result = StandardizeOp(mean=[10.0], std=[5.0])(Sample(input=tensor)) - assert abs(result.input[0] - 0.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Numpy StandardizeOp -# --------------------------------------------------------------------------- - - -class TestNpStandardizeOp: - """Tests for numpy StandardizeOp.""" - - def test_scalar_mean_and_std(self) -> None: - arr = np.array([2.0, 4.0, 6.0], dtype=np.float32) - result = np_ops.StandardizeOp(mean=4.0, std=2.0)(Sample(input=arr)) - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 0.0) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_per_channel_mean_and_std(self) -> None: - arr = np.ones((3, 2, 2), dtype=np.float32) - arr[0] *= 10.0 - arr[1] *= 20.0 - arr[2] *= 30.0 - result = np_ops.StandardizeOp(mean=[10.0, 20.0, 30.0], std=[1.0, 1.0, 1.0])(Sample(input=arr)) - assert np.allclose(result.input, np.zeros((3, 2, 2))) - - def test_uint8_converts_to_float32(self) -> None: - arr = np.array([100, 200], dtype=np.uint8) - result = np_ops.StandardizeOp(mean=150.0, std=50.0)(Sample(input=arr)) - assert result.input.dtype == np.float32 - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - arr = np.array([1.0, 2.0], dtype=np.float64) - result = np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=arr)) - assert result.input.dtype == np.float64 - - def test_preserves_target_and_metadata(self) -> None: - arr = np.array([5.0], dtype=np.float32) - result = np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=arr, target=3, metadata={"a": 1})) - assert result.target == 3 - assert result.meta == {"a": 1} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="StandardizeOp expects an np.ndarray"): - np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=[1, 2, 3])) - - def test_pil_image_input(self) -> None: - img = Image.fromarray(np.full((28, 28), 150, dtype=np.uint8)) - result = np_ops.StandardizeOp(mean=150.0, std=50.0)(Sample(input=img)) - assert isinstance(result.input, np.ndarray) - assert np.allclose(result.input, 0.0) - - def test_1d_per_channel(self) -> None: - arr = np.array([10.0], dtype=np.float32) - result = np_ops.StandardizeOp(mean=[10.0], std=[5.0])(Sample(input=arr)) - assert abs(result.input[0] - 0.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Numpy ClipPercentilesOp -# --------------------------------------------------------------------------- - - -class TestClipPercentilesOp: - def test_happy_path_no_outliers(self) -> None: - arr = np.linspace(-50.0, -10.0, 1000).reshape(20, 50) - out = np_ops.ClipPercentilesOp(low=2, high=98)(Sample(input=arr)).input - assert out.min() == pytest.approx(float(np.percentile(arr, 2))) - assert out.max() == pytest.approx(float(np.percentile(arr, 98))) - - def test_ignores_inf_and_nan(self) -> None: - arr = np.linspace(-50.0, -10.0, 100).reshape(10, 10).copy() - arr[0, 0] = np.inf - arr[0, 1] = -np.inf - arr[0, 2] = np.nan - finite = arr[np.isfinite(arr)] - expected_lo = float(np.percentile(finite, 2)) - expected_hi = float(np.percentile(finite, 98)) - out = np_ops.ClipPercentilesOp(low=2, high=98)(Sample(input=arr)).input - assert out[0, 0] == pytest.approx(expected_hi) - assert out[0, 1] == pytest.approx(expected_lo) - assert np.isnan(out[0, 2]) - - def test_all_non_finite_passes_through(self) -> None: - arr = np.full((4, 4), np.nan) - sample = Sample(input=arr) - out = np_ops.ClipPercentilesOp()(sample) - assert out is sample - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ClipPercentilesOp expects an np.ndarray"): - np_ops.ClipPercentilesOp()(Sample(input=torch.tensor([1.0]))) - - @pytest.mark.parametrize("low,high", [(50, 50), (60, 50), (-1, 50), (50, 101)]) - def test_validation_rejects_bad_bounds(self, low: float, high: float) -> None: - op = np_ops.ClipPercentilesOp(low=low, high=high) # lazy: construction succeeds - with pytest.raises(ValueError, match="ClipPercentilesOp: require"): - op(Sample(input=np.array([1.0, 2.0, 3.0]))) - - -# --------------------------------------------------------------------------- -# Numpy RescaleOp -# --------------------------------------------------------------------------- - - -class TestNpRescaleOp: - def test_default_output_range(self) -> None: - arr = np.array([[-80.0, -50.0, -20.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0)(Sample(input=arr)).input - np.testing.assert_allclose(out, [[0.0, 0.5, 1.0]]) - - def test_custom_output_range(self) -> None: - arr = np.array([[-80.0, -50.0, -20.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0, out_min=10.0, out_max=20.0)(Sample(input=arr)).input - np.testing.assert_allclose(out, [[10.0, 15.0, 20.0]]) - - def test_clip_true_clamps(self) -> None: - arr = np.array([[-100.0, -50.0, 0.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0, clip=True)(Sample(input=arr)).input - np.testing.assert_allclose(out, [[0.0, 0.5, 1.0]]) - - def test_clip_false_extrapolates(self) -> None: - arr = np.array([[-100.0, -50.0, 0.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0, clip=False)(Sample(input=arr)).input - assert out[0, 0] == pytest.approx(-1.0 / 3.0) - assert out[0, 1] == pytest.approx(0.5) - assert out[0, 2] == pytest.approx(4.0 / 3.0) - - def test_uint8_converts_to_float32(self) -> None: - arr = np.array([0, 128, 255], dtype=np.uint8) - out = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr)).input - assert out.dtype == np.float32 - assert abs(out[2] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - arr = np.array([0.0, 255.0], dtype=np.float64) - out = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr)).input - assert out.dtype == np.float64 - - def test_pil_image_input(self) -> None: - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - out = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=img)).input - assert isinstance(out, np.ndarray) - assert out.dtype == np.float32 - assert abs(out.max() - 200.0 / 255.0) < 1e-6 - - def test_preserves_target_and_metadata(self) -> None: - arr = np.array([128.0], dtype=np.float32) - result = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr, target=7, metadata={"key": "val"})) - assert result.target == 7 - assert result.meta == {"key": "val"} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="RescaleOp expects an np.ndarray"): - np_ops.RescaleOp(in_min=0.0, in_max=1.0)(Sample(input=[1.0, 2.0])) - - def test_validation_rejects_bad_input_range(self) -> None: - op = np_ops.RescaleOp(in_min=10.0, in_max=10.0) # lazy: construction succeeds - with pytest.raises(ValueError, match="require in_min < in_max"): - op(Sample(input=np.zeros(2))) - - def test_validation_rejects_bad_output_range(self) -> None: - op = np_ops.RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) - with pytest.raises(ValueError, match="require out_min < out_max"): - op(Sample(input=np.zeros(2))) - - def test_pipeline_rescale_then_to_tensor(self) -> None: - """Integration: numpy RescaleOp -> ToTensorOp(normalize=False).""" - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - sample = Sample(input=img) - sample = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(sample) - sample = ToTensorOp(normalize=False)(sample) - assert isinstance(sample.input, torch.Tensor) - assert sample.input.dtype == torch.float32 - assert abs(sample.input.max().item() - 200.0 / 255.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Numpy ReplaceNonFiniteOp -# --------------------------------------------------------------------------- - - -class TestReplaceNonFiniteOp: - def test_numeric_value(self) -> None: - arr = np.array([[1.0, np.inf, 2.0], [-np.inf, np.nan, 3.0]]) - out = np_ops.ReplaceNonFiniteOp(value=-99.0)(Sample(input=arr)).input - assert out.tolist() == [[1.0, -99.0, 2.0], [-99.0, -99.0, 3.0]] - - def test_min_replacement(self) -> None: - arr = np.array([[1.0, np.inf, 2.0], [-np.inf, np.nan, 3.0]]) - out = np_ops.ReplaceNonFiniteOp(value="min")(Sample(input=arr)).input - assert out.tolist() == [[1.0, 1.0, 2.0], [1.0, 1.0, 3.0]] - - def test_max_replacement(self) -> None: - arr = np.array([[1.0, np.inf, 2.0], [-np.inf, np.nan, 3.0]]) - out = np_ops.ReplaceNonFiniteOp(value="max")(Sample(input=arr)).input - assert out.tolist() == [[1.0, 3.0, 2.0], [3.0, 3.0, 3.0]] - - def test_already_finite_passes_through(self) -> None: - arr = np.array([[1.0, 2.0, 3.0]]) - sample = Sample(input=arr) - out = np_ops.ReplaceNonFiniteOp(value="min")(sample) - assert out is sample - - def test_all_non_finite_passes_through(self) -> None: - arr = np.full((3, 3), np.nan) - sample = Sample(input=arr) - out = np_ops.ReplaceNonFiniteOp(value="min")(sample) - assert out is sample - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ReplaceNonFiniteOp expects an np.ndarray"): - np_ops.ReplaceNonFiniteOp()(Sample(input=torch.tensor([1.0]))) - - def test_validation_rejects_unknown_string(self) -> None: - op = np_ops.ReplaceNonFiniteOp(value="median") # lazy: construction succeeds - with pytest.raises(ValueError, match="value string must be 'min' or 'max'"): - op(Sample(input=np.array([1.0, np.inf]))) - - -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# Copy* ops -# --------------------------------------------------------------------------- - - -class TestCopyOps: - def test_copy_sample_deepcopies_all_fields(self) -> None: - meta = {"k": [1, 2, 3]} - sample = Sample(input=np.array([1.0, 2.0]), target=[10], metadata=meta) - out = CopySampleOp()(sample) - assert out.input is not sample.input - assert out.target is not sample.target - assert out.meta is not sample.meta - assert out.meta["k"] is not sample.meta["k"] - - def test_copy_input_only_copies_input(self) -> None: - sample = Sample(input=np.array([1.0]), target=[5], metadata={"k": "v"}) - out = CopyInputOp()(sample) - assert out.input is not sample.input - assert out.target is sample.target - assert out.meta is sample.meta - - def test_copy_target_only_copies_target(self) -> None: - sample = Sample(input=[1, 2], target=[10, 20], metadata={}) - out = CopyTargetOp()(sample) - assert out.target is not sample.target - assert out.input is sample.input - - def test_copy_metadata_breaks_aliasing(self) -> None: - meta = {"k": [1]} - sample = Sample(input=None, target=None, metadata=meta) - out = CopyMetadataOp()(sample) - out.meta["k"].append(2) - assert meta["k"] == [1] - - -# --------------------------------------------------------------------------- -# SwapInputTargetOp -# --------------------------------------------------------------------------- - - -class TestSwapInputTargetOp: - def test_swaps_input_and_target(self) -> None: - sample = Sample(input=1, target=2, metadata={"k": "v"}) - out = SwapInputTargetOp()(sample) - assert out.input == 2 - assert out.target == 1 - assert out.meta == {"k": "v"} - - -# --------------------------------------------------------------------------- -# StashInputOp / UnstashInputOp -# --------------------------------------------------------------------------- - - -class TestStashUnstash: - def test_stash_aliases_by_default(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=arr, target=None, metadata={}) - out = StashInputOp(key="snap")(sample) - assert out.meta["snap"] is arr - assert out.input is arr - - def test_stash_with_copy_deepcopies(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=arr, target=None, metadata={}) - out = StashInputOp(key="snap", copy=True)(sample) - assert out.meta["snap"] is not arr - np.testing.assert_array_equal(out.meta["snap"], arr) - - def test_unstash_default_copies_to_isolate_branches(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - out = UnstashInputOp(key="snap")(sample) - assert out.input is not arr - np.testing.assert_array_equal(out.input, arr) - - def test_unstash_no_copy_aliases(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - out = UnstashInputOp(key="snap", copy=False)(sample) - assert out.input is arr - - def test_two_unstashes_with_in_place_mutation_dont_corrupt(self) -> None: - """Default copy=True prevents branch-A's in-place write from leaking into branch-B. - - A multi-unstash of the SAME key needs remove=False on the NON-final unstashes so the - snapshot survives (the compiler emits exactly this for a fan-out); the LAST unstash - cleans it up. - """ - arr = np.array([1.0, 2.0, 3.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - a = UnstashInputOp(key="snap", remove=False)(sample) # keep the key for branch B - a.input.fill(99.0) # in-place mutation on branch A's restored array - b = UnstashInputOp(key="snap")(sample) # final unstash → removes the key - np.testing.assert_array_equal(b.input, [1.0, 2.0, 3.0]) - assert "snap" not in sample.meta # cleaned up by the final unstash - - def test_unstash_removes_key_by_default(self) -> None: - sample = Sample(input=None, target=None, metadata={"snap": np.array([1.0, 2.0]), "keep": 1}) - out = UnstashInputOp(key="snap")(sample) - np.testing.assert_array_equal(out.input, [1.0, 2.0]) - assert "snap" not in out.meta # removed by default - assert out.meta["keep"] == 1 # other keys untouched - - def test_unstash_keeps_key_when_remove_false(self) -> None: - sample = Sample(input=None, target=None, metadata={"snap": np.array([1.0])}) - out = UnstashInputOp(key="snap", remove=False)(sample) - assert "snap" in out.meta - - -# --------------------------------------------------------------------------- -# StashTargetOp / UnstashTargetOp -# --------------------------------------------------------------------------- - - -class TestStashUnstashTarget: - def test_stash_target_aliases_by_default(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=arr, metadata={}) - out = StashTargetOp(key="snap")(sample) - assert out.meta["snap"] is arr - assert out.target is arr - - def test_stash_target_with_copy_deepcopies(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=arr, metadata={}) - out = StashTargetOp(key="snap", copy=True)(sample) - assert out.meta["snap"] is not arr - np.testing.assert_array_equal(out.meta["snap"], arr) - - def test_unstash_target_default_copies_to_isolate_branches(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - out = UnstashTargetOp(key="snap")(sample) - assert out.target is not arr - np.testing.assert_array_equal(out.target, arr) - - def test_unstash_target_no_copy_aliases(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - out = UnstashTargetOp(key="snap", copy=False)(sample) - assert out.target is arr - - def test_unstash_target_missing_key_raises_lazily(self) -> None: - sample = Sample(input=None, target=None, metadata={}) - with pytest.raises(KeyError): - UnstashTargetOp(key="nope")(sample) - - def test_stash_restore_round_trip_preserves_fork_target(self) -> None: - """The DAG→sequential pattern: snapshot at a fork, restore after a branch replaced it.""" - sample = Sample(input=None, target="fork-target", metadata={}) - stashed = StashTargetOp(key="fork")(sample) - branched = stashed._replace(target="branch-target") - restored = UnstashTargetOp(key="fork")(branched) - assert restored.target == "fork-target" - assert "fork" not in restored.meta # removed by default after restore - - def test_unstash_target_keeps_key_when_remove_false(self) -> None: - sample = Sample(input=None, target=None, metadata={"snap": np.array([1.0])}) - out = UnstashTargetOp(key="snap", remove=False)(sample) - assert "snap" in out.meta - - -# --------------------------------------------------------------------------- -# FormulaOp -# --------------------------------------------------------------------------- - - -class TestFormulaOp: - def test_evaluates_formula_over_input(self) -> None: - sample = Sample(input=10.0, target=None, metadata={}) - out = FormulaOp(formula="a * 0.2")(sample) - assert out.input == 2.0 - - def test_custom_var_binding(self) -> None: - sample = Sample(input=9.0, target=None, metadata={}) - out = FormulaOp(formula="sqrt(b)", var="b")(sample) - assert out.input == 3.0 - - def test_math_namespace_and_helpers(self) -> None: - sample = Sample(input=-4.2, target=None, metadata={}) - out = FormulaOp(formula="round(abs(a))")(sample) - assert out.input == 4 - - def test_no_builtins_in_namespace(self) -> None: - sample = Sample(input=1.0, target=None, metadata={}) - with pytest.raises(ValueError, match="failed"): - FormulaOp(formula="__import__('os').getcwd()")(sample) - - def test_bad_formula_raises_value_error(self) -> None: - sample = Sample(input=1.0, target=None, metadata={}) - with pytest.raises(ValueError, match="failed"): - FormulaOp(formula="a +")(sample) - - def test_empty_formula_raises_lazily(self) -> None: - sample = Sample(input=1.0, target=None, metadata={}) - with pytest.raises(ValueError, match="non-empty"): - FormulaOp(formula=" ")(sample) - - def test_default_is_identity(self) -> None: - sample = Sample(input=7.5, target=None, metadata={}) - assert FormulaOp()(sample).input == 7.5 - - -# --------------------------------------------------------------------------- -# ConfigureOp (the helios Configure pattern) -# --------------------------------------------------------------------------- - - -class TestConfigureOp: - def test_computes_injects_and_applies(self) -> None: - """compute-chain value → metadata + target attribute → target applied to the ORIGINAL sample.""" - sample = Sample(input=np.array([1.0, 5.0, 3.0]), target=None, metadata={}) - op = ConfigureOp( - ops=[MaxOp()], - target=ThresholdOp(low_op=">="), - param="low_level", - ) - out = op(sample) - assert out is not None - target = op.target - assert isinstance(target, ThresholdOp) and target.low_level == 5.0 # injected per sample - assert out.meta["low_level"] == 5.0 # traceability: the value rides metadata too - np.testing.assert_array_equal(out.input, [False, True, False]) # threshold on the ORIGINAL array - - def test_empty_compute_chain_uses_incoming_input(self) -> None: - class _Target: - def __init__(self) -> None: - self.level: object = None - - def __call__(self, s: Sample) -> Sample: - return s - - target = _Target() - sample = Sample(input=7.5, target=None, metadata={}) - out = ConfigureOp(target=target, param="level")(sample) - assert out is not None - assert target.level == 7.5 # no compute chain → the incoming input IS the value - assert out.meta["level"] == 7.5 - - def test_key_overrides_metadata_key(self) -> None: - sample = Sample(input=np.array([2.0]), target=None, metadata={}) - op = ConfigureOp(ops=[MaxOp()], target=ThresholdOp(low_op=">="), param="low_level", key="thr") - out = op(sample) - assert out is not None and out.meta["thr"] == 2.0 - assert "low_level" not in out.meta - - def test_missing_target_or_param_raise_lazily(self) -> None: - sample = Sample(input=np.array([1.0]), target=None, metadata={}) - with pytest.raises(ValueError, match="'target' op is required"): - ConfigureOp(param="x")(sample) - with pytest.raises(ValueError, match="'param'"): - ConfigureOp(target=ThresholdOp())(sample) - - def test_compute_chain_filtering_drops_sample(self) -> None: - """A compute op returning None propagates the drop (FilterOp semantics).""" - sample = Sample(input=np.array([1.0]), target=None, metadata={}) - op = ConfigureOp(ops=[lambda s: None], target=ThresholdOp(), param="low_level") - assert op(sample) is None - - def test_fluid_markers_flow_lazily(self) -> None: - """!class: markers in ops/target are flowed at first call (YAML-built ConfigureOp).""" - from confluid.fluid import Class - - sample = Sample(input=np.array([1.0, 4.0]), target=None, metadata={}) - op = ConfigureOp( - ops=[Class(MaxOp)], - target=Class(ThresholdOp, low_op=">="), - param="low_level", - ) - out = op(sample) - assert out is not None - assert isinstance(op.target, ThresholdOp) and op.target.low_level == 4.0 - np.testing.assert_array_equal(out.input, [False, True]) - - -# --------------------------------------------------------------------------- -# numpy.resolve_expression -# --------------------------------------------------------------------------- - - -class TestResolveExpression: - def test_no_substitution_returns_verbatim(self) -> None: - sample = Sample(input=None, target=None, metadata={}) - assert np_ops.resolve_expression("hello", sample) == "hello" - assert np_ops.resolve_expression("5.5", sample) == "5.5" - - def test_metadata_substitution(self) -> None: - sample = Sample(input=None, target=None, metadata={"snr": -30.5, "drone": "yz"}) - assert np_ops.resolve_expression("{snr}", sample) == "-30.5" - assert np_ops.resolve_expression("-{snr}", sample) == "--30.5" - assert np_ops.resolve_expression("{drone}", sample) == "yz" - - def test_env_substitution(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("REF_SNR", "12.5") - sample = Sample(input=None, target=None, metadata={}) - assert np_ops.resolve_expression("$REF_SNR", sample) == "12.5" - assert np_ops.resolve_expression("-$REF_SNR", sample) == "-12.5" - - def test_missing_metadata_key_raises(self) -> None: - sample = Sample(input=None, target=None, metadata={}) - with pytest.raises(KeyError, match="metadata key 'nope' missing"): - np_ops.resolve_expression("{nope}", sample) - - def test_missing_env_var_raises(self) -> None: - os.environ.pop("SAMPLEFLUX_TEST_NOPE", None) - sample = Sample(input=None, target=None, metadata={}) - with pytest.raises(KeyError, match="environment variable 'SAMPLEFLUX_TEST_NOPE'"): - np_ops.resolve_expression("$SAMPLEFLUX_TEST_NOPE", sample) - - -# --------------------------------------------------------------------------- -# ThresholdOp -# --------------------------------------------------------------------------- - - -class TestThresholdOp: - def test_numeric_low_level(self) -> None: - arr = np.array([0.0, 1.0, 2.0, 3.0]) - out = np_ops.ThresholdOp(low_level=1.5)(Sample(input=arr, metadata={})) - np.testing.assert_array_equal(out.input, [False, False, True, True]) - assert out.meta["threshold_low"] == 1.5 - assert "threshold_high" not in out.meta - - def test_numeric_high_level(self) -> None: - arr = np.array([0.0, 1.0, 2.0, 3.0]) - out = np_ops.ThresholdOp(high_level=1.5)(Sample(input=arr, metadata={})) - np.testing.assert_array_equal(out.input, [True, True, False, False]) - assert out.meta["threshold_high"] == 1.5 - assert "threshold_low" not in out.meta - - def test_band_low_and_high(self) -> None: - arr = np.array([0.0, 1.0, 2.0, 3.0, 4.0]) - out = np_ops.ThresholdOp(low_level=1.0, high_level=3.0)(Sample(input=arr, metadata={})) - # strictly between 1.0 and 3.0 (default open interval: > and <) - np.testing.assert_array_equal(out.input, [False, False, True, False, False]) - assert out.meta["threshold_low"] == 1.0 - assert out.meta["threshold_high"] == 3.0 - - def test_low_level_inclusive(self) -> None: - arr = np.array([0.0, 1.0, 2.0]) - # ">" excludes the boundary; ">=" includes it. - strict = np_ops.ThresholdOp(low_level=1.0)(Sample(input=arr, metadata={})) - np.testing.assert_array_equal(strict.input, [False, False, True]) - inclusive = np_ops.ThresholdOp(low_level=1.0, low_op=">=")(Sample(input=arr, metadata={})) - np.testing.assert_array_equal(inclusive.input, [False, True, True]) - - def test_high_level_inclusive(self) -> None: - arr = np.array([1.0, 2.0, 3.0]) - # "<" excludes the boundary; "<=" includes it. - strict = np_ops.ThresholdOp(high_level=2.0)(Sample(input=arr, metadata={})) - np.testing.assert_array_equal(strict.input, [True, False, False]) - inclusive = np_ops.ThresholdOp(high_level=2.0, high_op="<=")(Sample(input=arr, metadata={})) - np.testing.assert_array_equal(inclusive.input, [True, True, False]) - - def test_closed_band(self) -> None: - arr = np.array([0.0, 1.0, 2.0, 3.0, 4.0]) - out = np_ops.ThresholdOp(low_level=1.0, high_level=3.0, low_op=">=", high_op="<=")( - Sample(input=arr, metadata={}) - ) - # closed interval [1.0, 3.0]: both boundaries kept - np.testing.assert_array_equal(out.input, [False, True, True, True, False]) - - def test_invalid_operator_rejected(self) -> None: - # ``low_op`` is a closed ``Literal[">", ">="]`` — confluid's pydantic - # validation rejects anything else before the body runs. - from pydantic import ValidationError - - with pytest.raises((ValueError, ValidationError)): - np_ops.ThresholdOp(low_level=1.0, low_op=">>") # type: ignore[arg-type] - - def test_string_numeric(self) -> None: - arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(low_level="1.5")(Sample(input=arr)) - np.testing.assert_array_equal(out.input, [False, False, True]) - - def test_metadata_lookup(self) -> None: - arr = np.array([-50.0, -30.0, -10.0]) - sample = Sample(input=arr, target=None, metadata={"reference_snr_level": -25.0}) - out = np_ops.ThresholdOp(low_level="{reference_snr_level}")(sample) - np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.meta["threshold_low"] == -25.0 - - def test_metadata_lookup_with_negation(self) -> None: - arr = np.array([-50.0, -30.0, -10.0]) - sample = Sample(input=arr, target=None, metadata={"reference_snr_level": 30.0}) - out = np_ops.ThresholdOp(low_level="-{reference_snr_level}")(sample) - np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.meta["threshold_low"] == -30.0 - - def test_env_lookup(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SAMPLEFLUX_TEST_THRESHOLD", "1.0") - arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(low_level="$SAMPLEFLUX_TEST_THRESHOLD")(Sample(input=arr)) - np.testing.assert_array_equal(out.input, [False, False, True]) - - def test_high_level_expression(self) -> None: - arr = np.array([-50.0, -30.0, -10.0]) - sample = Sample(input=arr, target=None, metadata={"ceiling": -20.0}) - out = np_ops.ThresholdOp(high_level="{ceiling}")(sample) - np.testing.assert_array_equal(out.input, [True, True, False]) - assert out.meta["threshold_high"] == -20.0 - - def test_raises_when_no_bounds(self) -> None: - op = np_ops.ThresholdOp() # lazy: construction succeeds (zero-arg) - with pytest.raises(ValueError, match="at least one of 'low_level' / 'high_level'"): - op(Sample(input=np.zeros(3))) - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ThresholdOp expects an np.ndarray"): - np_ops.ThresholdOp(low_level=0.0)(Sample(input=[1.0, 2.0])) - - def test_raises_on_non_numeric_resolution(self) -> None: - sample = Sample(input=np.array([0.0]), target=None, metadata={"drone": "yz"}) - with pytest.raises(ValueError, match="not a number"): - np_ops.ThresholdOp(low_level="{drone}")(sample) - - def test_raises_on_bad_value_type(self) -> None: - # Confluid's ``@configurable`` validates kwargs against the - # auto-generated pydantic schema before the body runs. ``low_level`` is - # typed as ``float | int | str | None``, so a list is rejected at the - # validation layer first; the body's hand-rolled ``TypeError`` - # remains as a safety net. - from pydantic import ValidationError - - with pytest.raises((TypeError, ValidationError)): - np_ops.ThresholdOp(low_level=[1, 2])(Sample(input=np.array([0.0]))) # type: ignore[arg-type] - - def test_numpy_scalar_and_zero_d_array_bounds_accepted(self) -> None: - # A value chain (MaxOp → FormulaOp → ConfigureOp) injects a NumPy scalar / 0-d array into a - # bound via setattr, bypassing the pydantic ctor. np.float64 SUBCLASSES Python float (so it - # slipped through the old `isinstance(bound, (int, float))`), but np.float32 does NOT — - # _resolve must accept anything float() accepts. Live regression for a float32 spectrogram: - # "ThresholdOp bounds must be a number or expression string; got float32". - arr = np.array([0.0, 1.0, 2.0], dtype=np.float32) - for bound in (np.float32(2.0), np.array(2.0)): - op = np_ops.ThresholdOp(low_op=">=") - op.low_level = bound # type: ignore[assignment] # post-construction injection (ConfigureOp does this) - out = op(Sample(input=arr)) - np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.meta["threshold_low"] == 2.0 - - -def test_threshold_comparison_maps_match_literals() -> None: - # The operator-dispatch dicts must stay in lockstep with their closed - # Literals (one source of truth) — a new operator added to the Literal but - # not the map (or vice versa) is a bug this pins. - from typing import get_args - - assert set(np_ops._LOW_COMPARISONS) == set(get_args(np_ops.LowComparison)) - assert set(np_ops._HIGH_COMPARISONS) == set(get_args(np_ops.HighComparison)) - - -# --------------------------------------------------------------------------- -# ConnectedComponentsOp -# --------------------------------------------------------------------------- - - -class TestConnectedComponentsOp: - def test_two_separate_blobs(self) -> None: - mask = np.zeros((10, 10), dtype=bool) - mask[1:3, 1:3] = True # 2x2 blob at (1,1) - mask[6:9, 6:9] = True # 3x3 blob at (6,6) - out = np_ops.ConnectedComponentsOp(min_area_bins=1, connectivity=4)(Sample(input=mask)) - assert sorted(out.input) == [(1, 2, 1, 2), (6, 8, 6, 8)] - - def test_min_area_drops_small_components(self) -> None: - mask = np.zeros((10, 10), dtype=bool) - mask[0, 0] = True # area 1 - mask[5:7, 5:7] = True # area 4 - out = np_ops.ConnectedComponentsOp(min_area_bins=2, connectivity=4)(Sample(input=mask)) - assert out.input == [(5, 6, 5, 6)] - - def test_connectivity_4_keeps_diagonals_separate(self) -> None: - mask = np.zeros((4, 4), dtype=bool) - mask[0, 0] = True - mask[1, 1] = True - mask[2, 2] = True - out = np_ops.ConnectedComponentsOp(min_area_bins=1, connectivity=4)(Sample(input=mask)) - assert len(out.input) == 3 - - def test_connectivity_8_merges_diagonals(self) -> None: - mask = np.zeros((4, 4), dtype=bool) - mask[0, 0] = True - mask[1, 1] = True - mask[2, 2] = True - out = np_ops.ConnectedComponentsOp(min_area_bins=1, connectivity=8)(Sample(input=mask)) - assert len(out.input) == 1 - assert out.input[0] == (0, 2, 0, 2) - - def test_empty_mask_returns_empty_list(self) -> None: - mask = np.zeros((5, 5), dtype=bool) - out = np_ops.ConnectedComponentsOp(connectivity=4)(Sample(input=mask)) - assert out.input == [] - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ConnectedComponentsOp expects an np.ndarray"): - np_ops.ConnectedComponentsOp()(Sample(input=[[True, False]])) - - def test_raises_on_non_2d(self) -> None: - with pytest.raises(ValueError, match="expects a 2-D mask"): - np_ops.ConnectedComponentsOp()(Sample(input=np.array([True, False]))) - - def test_validation_rejects_bad_min_area(self) -> None: - op = np_ops.ConnectedComponentsOp(min_area_bins=0) # lazy: construction succeeds - with pytest.raises(ValueError, match="min_area_bins must be >= 1"): - op(Sample(input=np.zeros((2, 2), dtype=bool))) - - def test_validation_rejects_bad_connectivity(self) -> None: - op = np_ops.ConnectedComponentsOp(connectivity=6) - with pytest.raises(ValueError, match="connectivity must be 4 or 8"): - op(Sample(input=np.zeros((2, 2), dtype=bool))) - - -# --------------------------------------------------------------------------- -# Torch SqueezeOp -# --------------------------------------------------------------------------- - - -class TestTorchSqueezeOp: - """Tests for torch SqueezeOp.""" - - def test_squeeze_all_size1_dims(self) -> None: - tensor = torch.zeros(1, 3, 1, 4) - result = SqueezeOp()(Sample(input=tensor)) - assert result.input.shape == (3, 4) - - def test_squeeze_specific_dim(self) -> None: - tensor = torch.zeros(1, 3, 4) - result = SqueezeOp(dim=0)(Sample(input=tensor)) - assert result.input.shape == (3, 4) - - def test_squeeze_non_unit_dim_is_noop(self) -> None: - # torch.squeeze leaves non-size-1 dims unchanged - tensor = torch.zeros(2, 3) - result = SqueezeOp(dim=0)(Sample(input=tensor)) - assert result.input.shape == (2, 3) - - def test_preserves_target_and_metadata(self) -> None: - tensor = torch.zeros(1, 4) - result = SqueezeOp()(Sample(input=tensor, target=7, metadata={"k": "v"})) - assert result.target == 7 - assert result.meta == {"k": "v"} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="SqueezeOp expects a torch.Tensor"): - SqueezeOp()(Sample(input=np.zeros((1, 3)))) - - def test_zero_arg_construction(self) -> None: - op = SqueezeOp() - assert op.dim is None - - def test_negative_dim(self) -> None: - tensor = torch.zeros(3, 1) - result = SqueezeOp(dim=-1)(Sample(input=tensor)) - assert result.input.shape == (3,) - - -# --------------------------------------------------------------------------- -# Torch UnsqueezeOp -# --------------------------------------------------------------------------- - - -class TestTorchUnsqueezeOp: - """Tests for torch UnsqueezeOp.""" - - def test_unsqueeze_at_dim0(self) -> None: - tensor = torch.zeros(3, 4) - result = UnsqueezeOp(dim=0)(Sample(input=tensor)) - assert result.input.shape == (1, 3, 4) - - def test_unsqueeze_at_dim1(self) -> None: - tensor = torch.zeros(3, 4) - result = UnsqueezeOp(dim=1)(Sample(input=tensor)) - assert result.input.shape == (3, 1, 4) - - def test_unsqueeze_at_last_dim(self) -> None: - tensor = torch.zeros(3, 4) - result = UnsqueezeOp(dim=-1)(Sample(input=tensor)) - assert result.input.shape == (3, 4, 1) - - def test_default_dim_is_zero(self) -> None: - tensor = torch.zeros(5) - result = UnsqueezeOp()(Sample(input=tensor)) - assert result.input.shape == (1, 5) - - def test_preserves_target_and_metadata(self) -> None: - tensor = torch.zeros(4) - result = UnsqueezeOp()(Sample(input=tensor, target=2, metadata={"x": 1})) - assert result.target == 2 - assert result.meta == {"x": 1} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="UnsqueezeOp expects a torch.Tensor"): - UnsqueezeOp()(Sample(input=np.zeros(3))) - - def test_roundtrip_squeeze_unsqueeze(self) -> None: - tensor = torch.zeros(3, 4) - squeezed = UnsqueezeOp(dim=0)(Sample(input=tensor)) - restored = SqueezeOp(dim=0)(squeezed) - assert restored.input.shape == tensor.shape - - -# --------------------------------------------------------------------------- -# Numpy SqueezeOp -# --------------------------------------------------------------------------- - - -class TestNumpySqueezeOp: - """Tests for numpy SqueezeOp.""" - - def test_squeeze_all_size1_axes(self) -> None: - arr = np.zeros((1, 3, 1, 4)) - result = np_ops.SqueezeOp()(Sample(input=arr)) - assert result.input.shape == (3, 4) - - def test_squeeze_specific_axis(self) -> None: - arr = np.zeros((1, 3, 4)) - result = np_ops.SqueezeOp(axis=0)(Sample(input=arr)) - assert result.input.shape == (3, 4) - - def test_squeeze_non_unit_axis_raises(self) -> None: - arr = np.zeros((2, 3)) - with pytest.raises(ValueError): - np_ops.SqueezeOp(axis=0)(Sample(input=arr)) - - def test_preserves_target_and_metadata(self) -> None: - arr = np.zeros((1, 4)) - result = np_ops.SqueezeOp()(Sample(input=arr, target=7, metadata={"k": "v"})) - assert result.target == 7 - assert result.meta == {"k": "v"} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="SqueezeOp expects an np.ndarray"): - np_ops.SqueezeOp()(Sample(input=torch.zeros(1, 3))) - - def test_zero_arg_construction(self) -> None: - op = np_ops.SqueezeOp() - assert op.axis is None - - def test_negative_axis(self) -> None: - arr = np.zeros((3, 1)) - result = np_ops.SqueezeOp(axis=-1)(Sample(input=arr)) - assert result.input.shape == (3,) - - -# --------------------------------------------------------------------------- -# Numpy UnsqueezeOp -# --------------------------------------------------------------------------- - - -class TestNumpyUnsqueezeOp: - """Tests for numpy UnsqueezeOp.""" - - def test_unsqueeze_at_axis0(self) -> None: - arr = np.zeros((3, 4)) - result = np_ops.UnsqueezeOp(axis=0)(Sample(input=arr)) - assert result.input.shape == (1, 3, 4) - - def test_unsqueeze_at_axis1(self) -> None: - arr = np.zeros((3, 4)) - result = np_ops.UnsqueezeOp(axis=1)(Sample(input=arr)) - assert result.input.shape == (3, 1, 4) - - def test_unsqueeze_at_last_axis(self) -> None: - arr = np.zeros((3, 4)) - result = np_ops.UnsqueezeOp(axis=-1)(Sample(input=arr)) - assert result.input.shape == (3, 4, 1) - - def test_default_axis_is_zero(self) -> None: - arr = np.zeros(5) - result = np_ops.UnsqueezeOp()(Sample(input=arr)) - assert result.input.shape == (1, 5) - - def test_preserves_target_and_metadata(self) -> None: - arr = np.zeros(4) - result = np_ops.UnsqueezeOp()(Sample(input=arr, target=2, metadata={"x": 1})) - assert result.target == 2 - assert result.meta == {"x": 1} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="UnsqueezeOp expects an np.ndarray"): - np_ops.UnsqueezeOp()(Sample(input=torch.zeros(3))) - - def test_roundtrip_squeeze_unsqueeze(self) -> None: - arr = np.zeros((3, 4)) - unsqueezed = np_ops.UnsqueezeOp(axis=0)(Sample(input=arr)) - restored = np_ops.SqueezeOp(axis=0)(unsqueezed) - assert restored.input.shape == arr.shape - - -# --------------------------------------------------------------------------- -# DropMetadataOp -# --------------------------------------------------------------------------- -class TestDropMetadataOp: - def test_literal_exclude_drops_exact_keys(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - # A pattern with no wildcards is an EXACT key match; a missing key is ignored. - sample = Sample(input=np.zeros(2), target=None, metadata={"keep": 1, "drop_me": 2, "also": 3}) - out = DropMetadataOp(exclude=["drop_me", "also", "nope"])(sample) - assert out.meta == {"keep": 1} - - def test_glob_star_drops_all_matching(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - meta = {"real": 1, "__taidal_stash_456:input": [1j], "__taidal_stash_456:target": [2j]} - out = DropMetadataOp(exclude=["__taidal_stash*"])(Sample(input=np.zeros(2), metadata=meta)) - assert out.meta == {"real": 1} - - def test_glob_mid_wildcard_is_specific(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - # `__taidal_stash_456:*input` drops ONLY node 456's input stash — keeps its target and - # other nodes' inputs. - meta = { - "__taidal_stash_456:input": 1, - "__taidal_stash_456:target": 2, - "__taidal_stash_99:input": 3, - } - out = DropMetadataOp(exclude=["__taidal_stash_456:*input"])(Sample(input=np.zeros(2), metadata=meta)) - assert out.meta == {"__taidal_stash_456:target": 2, "__taidal_stash_99:input": 3} - - def test_multiple_exclude_patterns_any_match(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - meta = {"a": 1, "b": 2, "__t_x": 3, "__t_y": 4} - out = DropMetadataOp(exclude=["a", "__t_*"])(Sample(input=np.zeros(2), metadata=meta)) - assert out.meta == {"b": 2} - - def test_question_mark_and_set_globs(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - meta = {"img0": 1, "img1": 2, "imgX": 3, "image": 4} - out = DropMetadataOp(exclude=["img[0-9]"])(Sample(input=np.zeros(2), metadata=meta)) - assert out.meta == {"imgX": 3, "image": 4} # only single-digit img0/img1 dropped - - def test_matching_is_case_sensitive(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - out = DropMetadataOp(exclude=["key"])(Sample(input=np.zeros(2), metadata={"Key": 1, "key": 2})) - assert out.meta == {"Key": 1} - - def test_include_protects_keys_from_exclude(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - # include WINS: drop every stash key EXCEPT node 456's (carved out by include). - meta = { - "real": 1, - "__taidal_stash_456:input": 2, - "__taidal_stash_456:target": 3, - "__taidal_stash_99:input": 4, - } - out = DropMetadataOp(exclude=["__taidal_stash*"], include=["__taidal_stash_456:*"])( - Sample(input=np.zeros(2), metadata=meta) - ) - assert out.meta == {"real": 1, "__taidal_stash_456:input": 2, "__taidal_stash_456:target": 3} - - def test_include_without_exclude_drops_nothing(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - meta = {"a": 1, "b": 2} - out = DropMetadataOp(include=["a"])(Sample(input=np.zeros(2), metadata=meta)) - assert out.meta == {"a": 1, "b": 2} # include only protects against exclude - - def test_zero_arg_is_identity_metadata(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - meta = {"a": 1, "b": 2} - out = DropMetadataOp()(Sample(input=np.zeros(2), metadata=meta)) - assert out.meta == {"a": 1, "b": 2} - - def test_copy_on_write_does_not_mutate_original(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - original = {"a": 1, "drop": 2} - out = DropMetadataOp(exclude=["drop"])(Sample(input=np.zeros(2), metadata=original)) - assert original == {"a": 1, "drop": 2} # untouched - assert out.meta == {"a": 1} - - def test_input_and_target_untouched(self) -> None: - from sampleflux.ops.metadata import DropMetadataOp - - arr = np.arange(3) - out = DropMetadataOp(exclude=["x"])(Sample(input=arr, target=7, metadata={"x": 1, "y": 2})) - np.testing.assert_array_equal(out.input, arr) - assert out.target == 7 - - -# --------------------------------------------------------------------------- -# PrintSampleOp -# --------------------------------------------------------------------------- -class TestPrintSampleOp: - def test_returns_sample_unchanged(self) -> None: - from sampleflux.ops.debug import PrintSampleOp - - sample = Sample(input=np.zeros(3), target=1, metadata={"a": 1}) - out = PrintSampleOp(to_console=False)(sample) - assert out is sample - - def test_prints_to_console(self, capsys: pytest.CaptureFixture) -> None: - from sampleflux.ops.debug import PrintSampleOp - - PrintSampleOp(label="probe")(Sample(input=np.zeros((2, 3)), target=None, metadata={"k": 1})) - captured = capsys.readouterr().out - assert "[probe #0]" in captured - assert "shape=(2, 3)" in captured # input summary - assert "'k'" in captured # metadata key - - def test_summarizes_large_array_metadata_without_dumping(self, capsys: pytest.CaptureFixture) -> None: - from sampleflux.ops.debug import PrintSampleOp - - big = np.arange(100000, dtype=np.complex64) # would flood / not be reprable in full - PrintSampleOp(label="p")(Sample(input=np.zeros(2), metadata={"iq": big})) - out = capsys.readouterr().out - assert "shape=(100000,)" in out and "complex64" in out - assert "..." in out and "50000" not in out # values elided, not dumped in full - - def test_prints_small_array_values(self, capsys: pytest.CaptureFixture) -> None: - from sampleflux.ops.debug import PrintSampleOp - - PrintSampleOp(label="p")(Sample(input=np.array([1, 2, 3]), target=None, metadata={})) - out = capsys.readouterr().out - assert "values=[1, 2, 3]" in out # actual values shown for a small array - - def test_limit_caps_emissions_but_passes_all(self, capsys: pytest.CaptureFixture) -> None: - from sampleflux.ops.debug import PrintSampleOp - - op = PrintSampleOp(label="p", limit=2) - for _ in range(5): - assert op(Sample(input=np.zeros(1), metadata={})) is not None # all pass through - lines = [ln for ln in capsys.readouterr().out.splitlines() if ln.startswith("[p #")] - assert len(lines) == 2 # only the first 2 printed - - def test_to_console_false_is_silent_on_stdout(self, capsys: pytest.CaptureFixture) -> None: - from sampleflux.ops.debug import PrintSampleOp - - PrintSampleOp(to_console=False)(Sample(input=np.zeros(1), metadata={})) - assert capsys.readouterr().out == "" diff --git a/tests/test_parallel.py b/tests/test_parallel.py index a9fc68c..7b72113 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -2,6 +2,8 @@ import numpy as np +from sampleflux import Image, Sample +from sampleflux.bag.items import item_data from sampleflux.core import Flux @@ -11,7 +13,7 @@ def heavy_op(x: np.ndarray) -> np.ndarray: def test_parallel_execution() -> None: - source = [np.array([i]) for i in range(10)] + source = [Sample({"x": Image(np.array([i]))}, roles={"x": "input"}) for i in range(10)] start = time.time() # Use a real top-level function for pickling @@ -20,16 +22,12 @@ def test_parallel_execution() -> None: duration = time.time() - start assert len(results) == 10 - # In my previous run it was i*2 - assert results[0].input == 0 - assert results[9].input == 18 - # 10 items of 0.1s sequentially = 1.0s. 4 workers ≈ 0.3s + spawn overhead. - # Threshold is generous because GH Actions runners are noticeably slower - # than local dev machines (observed >5s on cold runners). We only assert - # the pipeline completes — this isn't a benchmark. + assert int(item_data(results[0]["x"])[0]) == 0 + assert int(item_data(results[9]["x"])[0]) == 18 + # We only assert the pipeline completes — this isn't a benchmark. assert duration < 15.0 def test_parallel_with_joint() -> None: - # Already tested in test_joint.py, but helps coverage here too + # Already tested elsewhere; helps coverage here too. pass diff --git a/tests/test_parallel_op.py b/tests/test_parallel_op.py deleted file mode 100644 index 7547ae6..0000000 --- a/tests/test_parallel_op.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tests for :class:`sampleflux.ops.parallel.Parallel`.""" - -from __future__ import annotations - -import time -from typing import Iterable, Iterator, List, Optional - -import numpy as np - -from sampleflux.core import Flux -from sampleflux.ops.parallel import Parallel -from sampleflux.sample import Sample - -# Top-level functions/classes — workers must be able to pickle these. - - -def double_input(sample: Sample) -> Sample: - return sample._replace(input=sample.input * 2) - - -def slow_double_input(sample: Sample) -> Sample: - time.sleep(0.05) - return sample._replace(input=sample.input * 2) - - -def drop_odd(sample: Sample) -> Optional[Sample]: - return sample if int(sample.input.item()) % 2 == 0 else None - - -class _TrackingSource: - """Iterable that records the maximum number of items pulled before any - are consumed downstream — used to verify bounded prefetch.""" - - def __init__(self, n: int, sleep_per_item: float = 0.0) -> None: - self.n = n - self.sleep_per_item = sleep_per_item - self.pulled = 0 - self.consumed = 0 - self.max_outstanding = 0 - - def __iter__(self) -> Iterator[np.ndarray]: - for i in range(self.n): - self.pulled += 1 - self.max_outstanding = max(self.max_outstanding, self.pulled - self.consumed) - if self.sleep_per_item: - time.sleep(self.sleep_per_item) - yield np.array([i]) - - def __len__(self) -> int: - return self.n - - -def test_parallel_inline_call_applies_ops_sequentially() -> None: - """``Parallel.__call__`` (used by ``Flux.__getitem__``) must apply its - inner ops sequentially in the calling process — same result as if the - ops were a plain list.""" - op = Parallel(ops=[double_input, double_input], workers=4) - sample = Sample(input=np.array([3])) - out = op(sample) - assert out is not None - assert out.input == np.array([12]) # 3 * 2 * 2 - - -def test_parallel_stream_yields_in_source_order() -> None: - source = [np.array([i]) for i in range(20)] - flux = Flux(source, ops=[Parallel(ops=[slow_double_input], workers=4)]) - results = list(flux) - assert [int(r.input.item()) for r in results] == [i * 2 for i in range(20)] - - -def test_parallel_stream_bounds_prefetch() -> None: - """With ``workers=4`` and a slow inner op, the source iterator must not - be drained more than ``2*workers + 1 = 9`` items ahead of consumption.""" - workers = 4 - expected_limit = 2 * workers + 1 - source = _TrackingSource(n=50) - - def consume(stream: Iterable[Sample]) -> List[Sample]: - results: List[Sample] = [] - for sample in stream: - source.consumed += 1 - results.append(sample) - return results - - flux = Flux(source, ops=[Parallel(ops=[slow_double_input], workers=workers)]) - results = consume(flux) - - assert len(results) == 50 - assert ( - source.max_outstanding <= expected_limit - ), f"prefetch unbounded: max_outstanding={source.max_outstanding} > {expected_limit}" - - -def test_parallel_filter_drops_nones_and_preserves_order() -> None: - source = [np.array([i]) for i in range(10)] - flux = Flux(source, ops=[Parallel(ops=[drop_odd, double_input], workers=3)]) - results = list(flux) - assert [int(r.input.item()) for r in results] == [0, 4, 8, 12, 16] - - -def test_parallel_workers_must_be_positive() -> None: - import pytest - - op = Parallel(ops=[double_input], workers=0) # lazy: construction succeeds - with pytest.raises(ValueError, match="workers="): - list(op.stream([])) - - -def test_parallel_close_propagates_to_inner_ops() -> None: - closed: List[str] = [] - - class ClosableOp: - def __init__(self, name: str) -> None: - self.name = name - - def __call__(self, s: Sample) -> Sample: - return s - - def close(self) -> None: - closed.append(self.name) - - op = Parallel(ops=[ClosableOp("a"), ClosableOp("b")], workers=2) - op.close() - assert closed == ["a", "b"] diff --git a/tests/test_processing.py b/tests/test_processing.py deleted file mode 100644 index a3bf8c0..0000000 --- a/tests/test_processing.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Tests for :class:`sampleflux.processing.DatasetProcessor` — progress-bar toggle.""" - -from typing import Any, Iterator, List -from unittest.mock import patch - -import confluid -import pytest -from confluid import configurable - -from sampleflux.core import Flux -from sampleflux.processing import DatasetProcessor -from sampleflux.sample import Sample - - -@configurable -class _SizedSource: - """Minimal Sized source yielding ``n`` trivial Samples.""" - - def __init__(self, n: int) -> None: - self.n = n - - def __len__(self) -> int: - return self.n - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.n): - yield Sample(input=i, target=None, metadata={}) - - -@configurable -class _UnsizedSource: - """Source that supports __iter__ but not __len__ — e.g. a streaming glob.""" - - def __init__(self, n: int) -> None: - self.n = n - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.n): - yield Sample(input=i, target=None, metadata={}) - - -class _RecordingSink: - """DataSink stand-in: records writes and flush, so we can assert order.""" - - def __init__(self) -> None: - self.written: List[Sample] = [] - self.flushed = False - - def write(self, sample: Sample) -> None: - self.written.append(sample) - - def flush(self) -> None: - self.flushed = True - - -def test_run_without_progress_uses_raw_flux() -> None: - flux = Flux(source=_SizedSource(3)) - sink = _RecordingSink() - with patch("sampleflux.processing.Progress") as mock_progress: - DatasetProcessor(flux=flux, sink=sink).run() - mock_progress.assert_not_called() - assert len(sink.written) == 3 - assert sink.flushed - - -def test_run_with_progress_sized_source_sets_total() -> None: - flux = Flux(source=_SizedSource(5)) - sink = _RecordingSink() - with patch("sampleflux.processing.Progress") as mock_progress: - DatasetProcessor(flux=flux, sink=sink, show_progress=True).run() - mock_progress.assert_called_once() - progress = mock_progress.return_value.__enter__.return_value - progress.add_task.assert_called_once() - call = progress.add_task.call_args - assert call.args[0] == "DatasetProcessor" - assert call.kwargs["total"] == 5 - assert len(sink.written) == 5 - - -def test_run_with_progress_unsized_source_falls_back_to_none() -> None: - flux = Flux(source=_UnsizedSource(4)) - sink = _RecordingSink() - with patch("sampleflux.processing.Progress") as mock_progress: - DatasetProcessor(flux=flux, sink=sink, show_progress=True).run() - progress = mock_progress.return_value.__enter__.return_value - progress.add_task.assert_called_once() - assert progress.add_task.call_args.kwargs["total"] is None - - -def test_progress_desc_overrides_default() -> None: - flux = Flux(source=_SizedSource(1)) - sink = _RecordingSink() - with patch("sampleflux.processing.Progress") as mock_progress: - DatasetProcessor(flux=flux, sink=sink, show_progress=True, progress_desc="my-run").run() - progress = mock_progress.return_value.__enter__.return_value - assert progress.add_task.call_args.args[0] == "my-run" - - -def test_no_sink_materializes_in_memory() -> None: - """Progress wrapper also exercised on the sinkless path.""" - flux = Flux(source=_SizedSource(2)) - with patch("sampleflux.processing.Progress") as mock_progress: - DatasetProcessor(flux=flux, show_progress=True).run() - progress = mock_progress.return_value.__enter__.return_value - assert progress.add_task.call_args.kwargs["total"] == 2 - - -def test_progress_bar_updates_per_sample() -> None: - """Asserts progress.update() fires exactly once per emitted sample.""" - flux = Flux(source=_SizedSource(3)) - sink = _RecordingSink() - with patch("sampleflux.processing.Progress") as mock_progress: - progress = mock_progress.return_value.__enter__.return_value - DatasetProcessor(flux=flux, sink=sink, show_progress=True).run() - assert progress.update.call_count == 3 - - -def test_yaml_roundtrip_preserves_progress_flags() -> None: - flux = Flux(source=_SizedSource(1)) - proc = DatasetProcessor(flux=flux, show_progress=True, progress_desc="from-yaml") - state = confluid.dump(proc) - restored: Any = confluid.load(state) - assert restored.show_progress is True - assert restored.progress_desc == "from-yaml" - - -def test_yaml_roundtrip_default_is_off() -> None: - proc = DatasetProcessor(flux=Flux(source=_SizedSource(1))) - restored: Any = confluid.load(confluid.dump(proc)) - assert restored.show_progress is False - assert restored.progress_desc is None - - -def test_progress_callback_fires_per_sample_without_console_bar() -> None: - """The executor's (FluxStudio) progress callback fires per sample even when show_progress is OFF. - - The native ComfyUI bar is independent of the rich console bar — set_progress_callback drives it - regardless of ``show_progress``. - """ - flux = Flux(source=_SizedSource(3)) - sink = _RecordingSink() - reports: List[tuple] = [] - proc = DatasetProcessor(flux=flux, sink=sink) # show_progress defaults to False - proc.set_progress_callback(lambda value, total, desc: reports.append((value, total, desc))) - proc.run() - # One report per emitted sample, with a monotonically increasing value and the sized total. - assert [v for v, _, _ in reports] == [1.0, 2.0, 3.0] - assert all(total == 3.0 for _, total, _ in reports) - assert all(desc == "DatasetProcessor" for *_, desc in reports) - - -def test_progress_callback_uses_progress_desc() -> None: - flux = Flux(source=_SizedSource(1)) - reports: List[tuple] = [] - proc = DatasetProcessor(flux=flux, progress_desc="my-run") # sinkless path - proc.set_progress_callback(lambda value, total, desc: reports.append((value, total, desc))) - proc.run() - assert reports == [(1.0, 1.0, "my-run")] - - -def test_progress_callback_noop_for_unsized_source() -> None: - """An unsized source has no total — the executor bar stays indeterminate (callback never fires).""" - flux = Flux(source=_UnsizedSource(4)) - reports: List[tuple] = [] - proc = DatasetProcessor(flux=flux) - proc.set_progress_callback(lambda value, total, desc: reports.append((value, total, desc))) - proc.run() - assert reports == [] - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) diff --git a/tests/test_projection.py b/tests/test_projection.py deleted file mode 100644 index cca4468..0000000 --- a/tests/test_projection.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Tests for the field-projection primitive and the num_classes helper.""" - -import itertools -from typing import Collection, Iterator, List, get_args - -import numpy as np -import pytest -import torch - -from sampleflux.core import Flux -from sampleflux.projection import ( - _FIELDS, - INPUT, - TARGET, - ProjectionField, - SupportsProjection, - _to_int, - iter_inputs, - iter_targets, - num_classes, - project, -) -from sampleflux.sample import Sample - -# --------------------------------------------------------------------------- # -# ProjectionField is a closed Literal a UI / form-spec can enumerate -# --------------------------------------------------------------------------- # - - -def test_projection_field_literal_enumerates_the_field_set() -> None: - # The whole point of the Literal (vs a bare ``str``): callers — UIs, MCP - # schemas, form-spec builders — read the allowed values from the annotation. - assert get_args(ProjectionField) == ("input", "target", "metadata") - - -def test_fields_constant_is_derived_from_the_literal() -> None: - # Single source of truth: the runtime-validation tuple comes FROM the Literal, - # so the two can never drift. - assert _FIELDS == get_args(ProjectionField) - - -# --------------------------------------------------------------------------- # -# Fallback path (sources that do NOT implement SupportsProjection) -# --------------------------------------------------------------------------- # - - -def _plain_source() -> List[Sample]: - return [ - Sample(input=np.array([1, 2]), target=0, metadata={"i": 0}), - Sample(input=np.array([3, 4]), target=2, metadata={"i": 1}), - ] - - -def test_project_fallback_nulls_unrequested_fields() -> None: - out = list(project(_plain_source(), (TARGET,))) - assert [s.target for s in out] == [0, 2] - assert all(s.input is None for s in out) - assert all(s.meta == {} for s in out) - - -def test_project_fallback_input_only() -> None: - out = list(project(_plain_source(), (INPUT,))) - assert all(s.target is None for s in out) - assert np.array_equal(out[0].input, np.array([1, 2])) - - -def test_project_rejects_unknown_field() -> None: - # An off-type value reaches the runtime guard (the Literal is a static hint, - # not a runtime gate). mypy rightly objects — ignore it; that's the point. - with pytest.raises(ValueError, match="Unknown projection field"): - list(project(_plain_source(), ("bogus",))) # type: ignore[arg-type] - - -def test_iter_helpers() -> None: - assert list(iter_targets(_plain_source())) == [0, 2] - inputs = list(iter_inputs(_plain_source())) - assert np.array_equal(inputs[1], np.array([3, 4])) - - -# --------------------------------------------------------------------------- # -# Efficient path (sources that DO implement SupportsProjection) -# --------------------------------------------------------------------------- # - - -class _ProjectableSource: - """A source that records which fields were requested and only builds those. - - Building the input increments ``input_builds`` — the test asserts a - target-only walk never touches it, proving the efficient path skips - unrequested-field construction. - """ - - def __init__(self, targets: List[int]) -> None: - self.targets = targets - self.input_builds = 0 - - def _build_input(self, i: int) -> np.ndarray: - self.input_builds += 1 - return np.full((2, 2), i) - - def __len__(self) -> int: - return len(self.targets) - - def __iter__(self) -> Iterator[Sample]: - for i, t in enumerate(self.targets): - yield Sample(input=self._build_input(i), target=t, metadata={}) - - def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: - want = frozenset(fields) - for i, t in enumerate(self.targets): - yield Sample( - input=self._build_input(i) if "input" in want else None, - target=t if "target" in want else None, - metadata={} if "metadata" not in want else {"i": i}, - ) - - -def test_projectable_source_is_recognized_by_protocol() -> None: - src = _ProjectableSource([0, 1]) - assert isinstance(src, SupportsProjection) - - -def test_efficient_target_only_skips_input_construction() -> None: - src = _ProjectableSource([0, 1, 2]) - targets = list(iter_targets(src)) - assert targets == [0, 1, 2] - assert src.input_builds == 0 # never decoded an input - - -# --------------------------------------------------------------------------- # -# num_classes -# --------------------------------------------------------------------------- # - - -def test_num_classes_int_targets_is_max_plus_one() -> None: - # max id 2 even though id 1 absent from this "split" -> head size 3. - assert num_classes(_ProjectableSource([0, 2, 0, 2])) == 3 - - -def test_num_classes_walks_via_projection_without_inputs() -> None: - src = _ProjectableSource([0, 1, 2, 3]) - assert num_classes(src) == 4 - assert src.input_builds == 0 - - -def test_num_classes_torch_scalar_tensor_targets() -> None: - src = [Sample(input=None, target=torch.tensor(k, dtype=torch.int64)) for k in (0, 4, 1)] - assert num_classes(src) == 5 - - -def test_num_classes_numpy_scalar_targets() -> None: - src = [Sample(input=None, target=np.int64(k)) for k in (0, 1, 2)] - assert num_classes(src) == 3 - - -def test_num_classes_empty_source_raises() -> None: - with pytest.raises(ValueError, match="no targets"): - num_classes([]) - - -def test_num_classes_none_target_raises() -> None: - with pytest.raises(ValueError, match="no target"): - num_classes([Sample(input=np.array([1]), target=None)]) - - -# --------------------------------------------------------------------------- # -# _to_int coercion -# --------------------------------------------------------------------------- # - - -def test_to_int_rejects_bool() -> None: - with pytest.raises(TypeError, match="bool"): - _to_int(True) - - -def test_to_int_rejects_non_scalar() -> None: - with pytest.raises(TypeError): - _to_int("3") - with pytest.raises(TypeError): - _to_int(torch.tensor([1, 2, 3])) # .item() on a multi-element tensor raises - - -def test_to_int_accepts_integer_valued_float_tensor() -> None: - assert _to_int(torch.tensor(2.0)) == 2 - - -# --------------------------------------------------------------------------- # -# Laziness -# --------------------------------------------------------------------------- # - - -def test_project_is_lazy() -> None: - def infinite() -> Iterator[Sample]: - for i in itertools.count(): - yield Sample(input=np.array([i]), target=i) - - first_two = list(itertools.islice(iter_targets(infinite()), 2)) - assert first_two == [0, 1] # never exhausts the infinite source - - -# --------------------------------------------------------------------------- # -# Flux.project -# --------------------------------------------------------------------------- # - - -def test_flux_project_runs_pipeline_then_drops_fields() -> None: - flux = Flux([Sample(input=np.array([1]), target=7, metadata={"k": "v"})]) - assert isinstance(flux, SupportsProjection) - out = list(flux.project((TARGET,))) - assert out[0].target == 7 - assert out[0].input is None - # routed through the module-level project() too - assert list(iter_targets(flux)) == [7] - - -def test_flux_num_classes_via_helper() -> None: - flux = Flux([Sample(input=np.array([1]), target=t) for t in (0, 1, 2, 1)]) - assert num_classes(flux) == 3 diff --git a/tests/test_query.py b/tests/test_query.py deleted file mode 100644 index cca78ea..0000000 --- a/tests/test_query.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Tests for the metadata query layer (`sampleflux.storage.query`).""" - -from pathlib import Path - -import numpy as np -import pytest - -from sampleflux.sample import Sample -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.query import MetadataFilterSource, SupportsMetadataScan, scan_hdf5_metadata -from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource - - -class TestMetadataQuery: - def _write_hdf5(self, path: Path) -> Path: - sink = HDF5Sink(path=path) - for i in range(4): - sink.write( - Sample( - input=np.ones(3) * i, - metadata={"snr_db": float(i * 5), "drone": "DJI" if i % 2 else "Parrot", "mask": np.ones((2, 2))}, - ) - ) - sink.flush() - sink.close() - return path - - def test_hdf5_scan_reads_no_arrays(self, tmp_path: Path) -> None: - path = self._write_hdf5(tmp_path / "d.h5") - scanned = list(scan_hdf5_metadata(path)) - assert len(scanned) == 4 - _key, meta = scanned[2] - assert meta["snr_db"] == 10.0 - assert meta["mask"].startswith(" None: - path = self._write_hdf5(tmp_path / "d.h5") - assert isinstance(HDF5Source(path=path), SupportsMetadataScan) - assert isinstance(ZarrGroupSource(path=str(tmp_path / "z")), SupportsMetadataScan) - - def test_filter_source_where_expression_on_hdf5(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - view = MetadataFilterSource(source=source, where="snr_db >= 10") - assert len(view) == 2 - assert [s.meta["snr_db"] for s in view] == [10.0, 15.0] - assert view[0].meta["snr_db"] == 10.0 # random access into matches - - def test_filter_source_string_and_predicate_compose(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - view = MetadataFilterSource(source=source, where="drone == 'DJI'", predicate=lambda m: m["snr_db"] > 5) - assert [s.meta["snr_db"] for s in view] == [15.0] - - def test_missing_key_is_non_matching_not_fatal(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - assert len(MetadataFilterSource(source=source, where="no_such_key > 1")) == 0 - - def test_malformed_expression_fails_loudly(self, tmp_path: Path) -> None: - source = HDF5Source(path=self._write_hdf5(tmp_path / "d.h5")) - with pytest.raises(ValueError, match="failed"): - len(MetadataFilterSource(source=source, where="snr_db +* 2")) - - def test_empty_filter_is_rejected(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="empty filter"): - len(MetadataFilterSource(source=[Sample(1)])) - - def test_fallback_full_iteration_for_plain_sources(self) -> None: - plain = [Sample(input=i, metadata={"v": i}) for i in range(5)] - view = MetadataFilterSource(source=plain, where="v % 2 == 0") - assert [s.input for s in view] == [0, 2, 4] - - def test_zarr_scan_and_filter(self, tmp_path: Path) -> None: - sink = ZarrGroupSink(path=str(tmp_path / "z")) - for i in range(3): - sink.write(Sample(input=np.ones(2) * i, metadata={"v": i})) - sink.flush() - source = ZarrGroupSource(path=str(tmp_path / "z")) - view = MetadataFilterSource(source=source, where="v == 1") - assert len(view) == 1 and view[0].meta["v"] == 1 diff --git a/tests/test_random_apply.py b/tests/test_random_apply.py deleted file mode 100644 index 3f166c9..0000000 --- a/tests/test_random_apply.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Tests for sampleflux.ops.random_apply.RandomApply.""" - -import pytest - -from sampleflux.ops.random_apply import RandomApply -from sampleflux.sample import Sample - - -def _s(v: int = 0) -> Sample: - return Sample(input=v, metadata={}) - - -class _BumpOp: - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + 1) - - -def test_zero_arg_construction() -> None: - assert RandomApply() is not None - - -def _run(op: RandomApply, sample: Sample) -> Sample: - """Apply and narrow: these tests never exercise the drop (None) path.""" - out = op(sample) - assert out is not None - return out - - -def test_probability_zero_never_applies() -> None: - op = RandomApply(op=_BumpOp(), probability=0.0) - for _ in range(20): - out = _run(op, _s(0)) - assert out.input == 0 - - -def test_probability_one_always_applies() -> None: - op = RandomApply(op=_BumpOp(), probability=1.0) - for _ in range(20): - out = _run(op, _s(0)) - assert out.input == 1 - - -def test_raises_when_op_is_none() -> None: - op = RandomApply(probability=1.0) - with pytest.raises(ValueError, match="op"): - op(_s()) - - -def test_is_marked_random() -> None: - assert getattr(RandomApply, "__confluid_random__", False) is True - - -def test_is_registered_configurable() -> None: - from confluid.registry import resolve_class # type: ignore[import-not-found] - - path = f"{RandomApply.__module__}.{RandomApply.__qualname__}" - assert resolve_class(path) is RandomApply - - -def test_flows_confluid_fluid_op_lazily() -> None: - from confluid import configurable - from confluid.fluid import Class - - @configurable - class _Inner: - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + 10) - - fluid_op = Class(_Inner) - op = RandomApply(op=fluid_op, probability=1.0) - out = _run(op, _s(5)) - assert out.input == 15 - # second call reuses the cached flowed op - out2 = _run(op, _s(5)) - assert out2.input == 15 - - -def test_sample_passes_through_unchanged_when_skipped() -> None: - s = _s(42) - op = RandomApply(op=_BumpOp(), probability=0.0) - assert op(s) is s - - -# --------------------------------------------------------------------------- -# random_state / reproducibility -# --------------------------------------------------------------------------- - - -def test_random_state_stored_on_instance() -> None: - op = RandomApply(op=_BumpOp(), probability=0.5, random_state=42) - assert op.random_state == 42 - - -def test_random_state_none_is_default() -> None: - op = RandomApply(op=_BumpOp()) - assert op.random_state is None - - -def test_gate_reproducible_with_seed() -> None: - """Two RandomApply instances with the same seed must make identical gate decisions.""" - s = _s(0) - op_a = RandomApply(op=_BumpOp(), probability=0.5, random_state=7) - op_b = RandomApply(op=_BumpOp(), probability=0.5, random_state=7) - results_a = [_run(op_a, s).input for _ in range(30)] - results_b = [_run(op_b, s).input for _ in range(30)] - assert results_a == results_b - - -def test_gate_different_seeds_produce_different_sequences() -> None: - s = _s(0) - op_a = RandomApply(op=_BumpOp(), probability=0.5, random_state=1) - op_b = RandomApply(op=_BumpOp(), probability=0.5, random_state=2) - results_a = [_run(op_a, s).input for _ in range(50)] - results_b = [_run(op_b, s).input for _ in range(50)] - assert results_a != results_b - - -def test_zero_arg_construction_with_random_state_none() -> None: - op = RandomApply() - assert op.random_state is None - assert op._gate_rng is None # lazily initialized on first call diff --git a/tests/test_sample.py b/tests/test_sample.py deleted file mode 100644 index 0d7df67..0000000 --- a/tests/test_sample.py +++ /dev/null @@ -1,87 +0,0 @@ -from typing import Any, cast - -import numpy as np -import pytest - -from sampleflux.sample import Sample - - -def test_sample_from_any() -> None: - # 1. From dict - d = {"input": np.array([1, 2]), "target": 1, "metadata": {"id": "test"}} - s = Sample.from_any(d) - assert np.array_equal(s.input, cast(Any, d["input"])) - assert s.target == 1 - assert s.meta["id"] == "test" - - # 2. From tuple (input, target) - t = (np.array([3, 4]), 0) - s2 = Sample.from_any(t) - assert np.array_equal(s2.input, t[0]) - assert s2.target == 0 - - # 3. From tuple (input,) - hits line 22 - t3 = (np.array([7, 8]),) - s3 = Sample.from_any(t3) - assert np.array_equal(s3.input, t3[0]) - assert s3.target is None - - # 4. From single item (input only) - val = np.array([5, 6]) - s4 = Sample.from_any(val) - assert np.array_equal(s4.input, val) - assert s4.target is None - - -def test_sample_to_tuple() -> None: - s = Sample(input=1, target=2, metadata={"a": 3}) - t = s.to_tuple() - assert t == (1, 2, {"a": 3}) - - -# --- Batch vs single metadata (Union schema) ------------------------------- - - -def test_single_sample_metadata_is_a_dict() -> None: - s = Sample(input=1, target=0, metadata={"id": "a"}) - assert s.is_batched is False - assert s.meta == {"id": "a"} - assert s.meta["id"] == "a" - - -def test_batched_sample_metadata_is_a_list_of_dicts() -> None: - # The collate form: one Sample carrying N stacked items + per-item metadata dicts. - batch = Sample(input=[1, 2], target=[0, 1], metadata=[{"id": "a"}, {"id": "b"}]) - assert batch.is_batched is True - assert batch.batch_meta == [{"id": "a"}, {"id": "b"}] - assert [m["id"] for m in batch.batch_meta] == ["a", "b"] - - -def test_meta_accessor_raises_on_a_batch() -> None: - batch = Sample(input=[1], target=[0], metadata=[{"id": "a"}]) - with pytest.raises(TypeError, match="batched"): - _ = batch.meta - - -def test_batch_meta_accessor_raises_on_a_single_sample() -> None: - single = Sample(input=1, target=0, metadata={"id": "a"}) - with pytest.raises(TypeError, match="single"): - _ = single.batch_meta - - -def test_describe_falls_back_to_inference_on_a_batch() -> None: - # A batched sample carries no per-reserved-key stored type -> describe infers (no crash on the list). - batch = Sample(input=np.zeros((2, 4)), target=np.array([0, 1]), metadata=[{}, {}]) - assert batch.describe() is not None - - -def test_with_type_rejects_a_batch() -> None: - from sampleflux.typespec import infer_sample_type - - single = Sample(input=np.zeros((4,)), target=0, metadata={}) - typed = single.with_type(infer_sample_type(single)) # single sample: OK - assert typed.describe() is not None - - batch = Sample(input=np.zeros((2, 4)), target=np.array([0, 1]), metadata=[{}, {}]) - with pytest.raises(TypeError, match="batched"): - batch.with_type(infer_sample_type(single)) diff --git a/tests/test_sources.py b/tests/test_sources.py deleted file mode 100644 index 1569641..0000000 --- a/tests/test_sources.py +++ /dev/null @@ -1,563 +0,0 @@ -"""Tests for SampleFlux sources: DatasetSplit (+ cached split views), RangeSource, ConcatSource.""" - -import inspect -from typing import Any, Iterator, List - -import confluid # type: ignore[import-not-found] -import pytest - -from sampleflux.core import Flux -from sampleflux.sample import Sample -from sampleflux.sources import ConcatSource, DatasetSplit, RangeSource - - -@confluid.configurable -class IndexedSource: - """A configurable indexable source for the tests. - - Stores a list of integers; each `__getitem__` returns ``Sample(input=i)``. - """ - - def __init__(self, size: int = 100) -> None: - self.size = size - - def __len__(self) -> int: - return self.size - - def __getitem__(self, index: int) -> Sample: - return Sample(input=index, target=None, metadata={"idx": index}) - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.size): - yield self[i] - - -# --------------------------------------------------------------------------- -# DatasetSplit — select-one API (split=) -# --------------------------------------------------------------------------- - - -def test_fraction_mode_partitions_cleanly() -> None: - """split='train' + split='val' cover the full source with no overlap.""" - source = IndexedSource(size=100) - train = DatasetSplit(source=source, split="train", val_fraction=0.1, seed=42) - val = DatasetSplit(source=source, split="val", val_fraction=0.1, seed=42) - - train_idx = {s.input for s in train} - val_idx = {s.input for s in val} - - assert len(train) == 90 - assert len(val) == 10 - assert train_idx.isdisjoint(val_idx) - assert train_idx | val_idx == set(range(100)) - - -def test_fraction_mode_is_deterministic_across_instances() -> None: - """Same seed + source -> identical shuffle -> stable split.""" - source = IndexedSource(size=50) - first = list(DatasetSplit(source=source, split="val", val_fraction=0.2, seed=7)) - second = list(DatasetSplit(source=source, split="val", val_fraction=0.2, seed=7)) - assert [s.input for s in first] == [s.input for s in second] - - -def test_fraction_mode_different_seeds_differ() -> None: - source = IndexedSource(size=50) - a = {s.input for s in DatasetSplit(source=source, split="val", val_fraction=0.2, seed=1)} - b = {s.input for s in DatasetSplit(source=source, split="val", val_fraction=0.2, seed=2)} - # Overwhelmingly likely to differ on 50 elements - assert a != b - - -def test_fraction_mode_requires_seed() -> None: - split = DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.1) # lazy ctor - with pytest.raises(ValueError, match="seed"): - _ = split.train - - -def test_fraction_mode_rejects_invalid_split() -> None: - with pytest.raises(ValueError, match="split"): - DatasetSplit(source=IndexedSource(size=10), split="holdout", val_fraction=0.1, seed=0) # type: ignore[arg-type] - - -def test_fraction_mode_rejects_out_of_range_fraction() -> None: - src = IndexedSource(size=10) - split = DatasetSplit(source=src, split="train", val_fraction=1.5, seed=0) # lazy ctor - with pytest.raises(ValueError, match="val_fraction"): - _ = split.train - - -# --------------------------------------------------------------------------- -# DatasetSplit — three-way (select-one) -# --------------------------------------------------------------------------- - - -def test_three_way_split_partitions_cleanly() -> None: - """split='train'/'val'/'test' cover the full source with no overlap.""" - source = IndexedSource(size=100) - kw = dict(source=source, val_fraction=0.2, test_fraction=0.1, seed=42) - train = DatasetSplit(split="train", **kw) # type: ignore[arg-type] - val = DatasetSplit(split="val", **kw) # type: ignore[arg-type] - test = DatasetSplit(split="test", **kw) # type: ignore[arg-type] - - train_idx = {s.input for s in train} - val_idx = {s.input for s in val} - test_idx = {s.input for s in test} - - assert len(val) == 20 - assert len(test) == 10 - assert len(train) == 70 - assert train_idx.isdisjoint(val_idx) - assert train_idx.isdisjoint(test_idx) - assert val_idx.isdisjoint(test_idx) - assert train_idx | val_idx | test_idx == set(range(100)) - - -def test_test_fraction_out_of_range_rejected() -> None: - split = DatasetSplit(source=IndexedSource(size=10), split="test", test_fraction=1.5, seed=0) # lazy ctor - with pytest.raises(ValueError, match="test_fraction"): - _ = split.test - - -def test_val_plus_test_fraction_must_be_under_one() -> None: - split = DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.6, test_fraction=0.5, seed=0) - with pytest.raises(ValueError, match="must be < 1"): - _ = split.train - - -# --------------------------------------------------------------------------- -# DatasetSplit — cached property API (.train / .val / .test) -# --------------------------------------------------------------------------- - - -def test_property_api_partitions_cleanly() -> None: - """A single DatasetSplit exposes the three disjoint, complementary views.""" - source = IndexedSource(size=100) - split = DatasetSplit(source=source, val_fraction=0.2, test_fraction=0.1, seed=42) - - train_idx = {s.input for s in split.train} - val_idx = {s.input for s in split.val} - test_idx = {s.input for s in split.test} - - assert len(split.train) == 70 and len(split.val) == 20 and len(split.test) == 10 - assert train_idx.isdisjoint(val_idx) - assert train_idx.isdisjoint(test_idx) - assert val_idx.isdisjoint(test_idx) - assert train_idx | val_idx | test_idx == set(range(100)) - - -def test_property_views_are_cached() -> None: - split = DatasetSplit(source=IndexedSource(size=20), val_fraction=0.25, seed=1) - assert split.train is split.train # memoized — same object each access - assert split.val is split.val - assert split.test is split.test - - -def test_property_and_select_one_agree() -> None: - """``split='val'`` (select-one) yields the same indices as the ``.val`` property.""" - source = IndexedSource(size=40) - selected = DatasetSplit(source=source, split="val", val_fraction=0.25, seed=3) - split = DatasetSplit(source=source, val_fraction=0.25, seed=3) - assert [s.input for s in selected] == [s.input for s in split.val] - - -def test_no_fractions_train_is_full_val_test_empty() -> None: - """No fractions (and no seed needed) → train is the whole source (unshuffled), val/test empty.""" - source = IndexedSource(size=8) - split = DatasetSplit(source=source) - assert [s.input for s in split.train] == list(range(8)) - assert len(split.val) == 0 - assert len(split.test) == 0 - - -def test_default_iteration_is_train() -> None: - """Iterating a DatasetSplit with no ``split`` delegates to the ``train`` view.""" - split = DatasetSplit(source=IndexedSource(size=10), val_fraction=0.2, seed=1) - assert [s.input for s in split] == [s.input for s in split.train] - - -def test_datasetsplit_dropped_range_params() -> None: - """Range mode moved to RangeSource — DatasetSplit no longer accepts start/end.""" - params = set(inspect.signature(DatasetSplit).parameters) - assert {"source", "split", "val_fraction", "test_fraction", "seed"} == params - - -def test_invalid_source_type() -> None: - class _Plain: - pass - - split = DatasetSplit(source=_Plain()) # lazy: construction succeeds - with pytest.raises(TypeError, match="__len__"): - _ = split.train - - -# --------------------------------------------------------------------------- -# RangeSource (the extracted contiguous-slice mode) -# --------------------------------------------------------------------------- - - -def test_range_source_slices() -> None: - source = IndexedSource(size=20) - view = RangeSource(source=source, start=5, end=15) - assert [s.input for s in view] == list(range(5, 15)) - assert len(view) == 10 - - -def test_range_source_open_ended() -> None: - source = IndexedSource(size=20) - head = RangeSource(source=source, end=10) - tail = RangeSource(source=source, start=10) - assert [s.input for s in head] == list(range(10)) - assert [s.input for s in tail] == list(range(10, 20)) - - -def test_range_source_clamps_out_of_bounds() -> None: - view = RangeSource(source=IndexedSource(size=5), start=-10, end=100) - assert len(view) == 5 - - -def test_range_source_getitem_resolves_through_underlying_source() -> None: - view = RangeSource(source=IndexedSource(size=30), start=10, end=20) - assert view[0].input == 10 - assert view[-1].input == 19 # Python list indexing supports negatives - - -def test_range_source_invalid_source_type() -> None: - class _Plain: - pass - - rng = RangeSource(source=_Plain()) # lazy: construction succeeds - with pytest.raises(TypeError, match="__len__"): - len(rng) - - -def _scale(value: int, factor: int = 1) -> int: - return value * factor - - -def test_range_source_inside_flux() -> None: - view = RangeSource(source=IndexedSource(size=20), start=0, end=5) - flux = Flux(source=view).map(_scale, factor=10) - results: List[Sample] = flux.collect() - assert [s.input for s in results] == [0, 10, 20, 30, 40] - - -# --------------------------------------------------------------------------- -# ConcatSource (indexable join of multiple sources) -# --------------------------------------------------------------------------- - - -def test_concat_source_len_and_iter() -> None: - cat = ConcatSource(sources=[IndexedSource(size=3), IndexedSource(size=2)]) - assert len(cat) == 5 - # Each sub-source yields its own 0..n-1 inputs, walked in order. - assert [s.input for s in cat] == [0, 1, 2, 0, 1] - - -def test_concat_source_getitem_maps_to_subsource() -> None: - cat = ConcatSource(sources=[IndexedSource(size=3), IndexedSource(size=2)]) - assert [cat[i].input for i in range(5)] == [0, 1, 2, 0, 1] # 3 from src0, 2 from src1 - assert cat[-1].input == 1 # last item of src1 - - -def test_concat_source_out_of_bounds() -> None: - cat = ConcatSource(sources=[IndexedSource(size=3)]) - with pytest.raises(IndexError): - cat[3] - - -def test_concat_source_empty() -> None: - cat = ConcatSource(sources=[]) - assert len(cat) == 0 - with pytest.raises(IndexError): - cat[0] - - -def test_concat_source_rejects_non_indexable() -> None: - cat = ConcatSource(sources=[object()]) # lazy: construction succeeds - with pytest.raises(TypeError, match="__len__"): - len(cat) - - -def test_concat_source_is_splittable() -> None: - """A ConcatSource is indexable, so DatasetSplit can partition the joined sources.""" - cat = ConcatSource(sources=[IndexedSource(size=30), IndexedSource(size=20)]) - split = DatasetSplit(source=cat, val_fraction=0.2, seed=1) - assert len(split.train) == 40 and len(split.val) == 10 - assert len(split.train) + len(split.val) == 50 - - -# --------------------------------------------------------------------------- -# Confluid serialization round-trips -# --------------------------------------------------------------------------- - - -def test_serialization_roundtrip_preserves_split() -> None: - source = IndexedSource(size=40) - split = DatasetSplit(source=source, split="val", val_fraction=0.25, seed=123) - yaml_state = confluid.dump(split) - assert "!class:DatasetSplit" in yaml_state - assert "val_fraction: 0.25" in yaml_state - assert "seed: 123" in yaml_state - - restored: Any = confluid.load(yaml_state) - # Reloaded source is a fresh IndexedSource with size=40 — feed it identically. - assert len(restored) == len(split) - assert [s.input for s in restored] == [s.input for s in split] - - -def test_serialization_roundtrip_preserves_three_way_split() -> None: - source = IndexedSource(size=60) - split = DatasetSplit(source=source, split="test", val_fraction=0.2, test_fraction=0.1, seed=5) - yaml_state = confluid.dump(split) - assert "!class:DatasetSplit" in yaml_state - assert "test_fraction: 0.1" in yaml_state - - restored: Any = confluid.load(yaml_state) - assert len(restored) == len(split) - assert [s.input for s in restored] == [s.input for s in split] - - -def test_property_split_roundtrip_via_views() -> None: - """A property-style DatasetSplit (no ``split``) round-trips; views recompute identically.""" - source = IndexedSource(size=40) - split = DatasetSplit(source=source, val_fraction=0.25, seed=123) - yaml_state = confluid.dump(split) - assert "!class:DatasetSplit" in yaml_state - restored: Any = confluid.load(yaml_state) - assert [s.input for s in restored.val] == [s.input for s in split.val] - assert [s.input for s in restored.train] == [s.input for s in split.train] - - -def test_property_refs_share_one_instance_and_partition_cleanly() -> None: - """ONE DatasetSplit referenced via ``!ref:my_split.train`` / ``.val`` — the new pattern. - - Both attribute-refs resolve from the SAME flowed DatasetSplit (Confluid's dotted-ref now reuses - the materialized instance — see confluid ``test_dotted_attribute_ref_reuses_single_instance``), - so the cached views share the single underlying source — it is the document's ``hf`` instance, - loaded exactly once — and form a disjoint, complementary partition. - """ - yaml_state = """ -hf: !class:IndexedSource() - size: 50 - -my_split: !class:DatasetSplit() - source: !ref:hf - val_fraction: 0.2 - seed: 9 - -train_set: !class:sampleflux.core.Flux() - source: !ref:my_split.train - -val_set: !class:sampleflux.core.Flux() - source: !ref:my_split.val -""" - state: Any = confluid.load(yaml_state) - train_flux = state["train_set"] - val_flux = state["val_set"] - - # Both Flux sources are views off the SAME DatasetSplit, wrapping the SINGLE ``hf`` instance - # (one load), and the shared split's cached property IS the view the Flux received. - assert train_flux.source.source is val_flux.source.source - assert train_flux.source.source is state["hf"] - assert state["my_split"].train is train_flux.source - - train_idx = {s.input for s in train_flux} - val_idx = {s.input for s in val_flux} - assert len(val_idx) == 10 - assert train_idx.isdisjoint(val_idx) - assert train_idx | val_idx == set(range(50)) - - -def test_select_one_refs_share_source_and_partition_cleanly() -> None: - """Select-one pattern: two DatasetSplits over a shared ``!ref:source`` — a single load. - - Both ``!ref:hf_train`` resolve (by Confluid's instance memo) to the SAME source instance, so - the source is materialized exactly once and the two views partition it cleanly. This is the - load-once-guaranteed pattern when a single shared instance matters (e.g. ``HuggingFaceSource``). - """ - yaml_state = """ -hf_train: !class:IndexedSource() - size: 50 - -train_set: !class:DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.2 - seed: 9 - -val_set: !class:DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.2 - seed: 9 -""" - state: Any = confluid.load(yaml_state) - train = state["train_set"] - val = state["val_set"] - - assert train.source is val.source - assert train.source is state["hf_train"] - - train_idx = {s.input for s in train} - val_idx = {s.input for s in val} - assert train_idx.isdisjoint(val_idx) - assert train_idx | val_idx == set(range(50)) - - -def test_concat_source_roundtrip() -> None: - cat = ConcatSource(sources=[IndexedSource(size=3), IndexedSource(size=4)]) - yaml_state = confluid.dump(cat) - assert "!class:ConcatSource" in yaml_state - restored: Any = confluid.load(yaml_state) - assert len(restored) == 7 - assert [s.input for s in restored] == [s.input for s in cat] - - -# --------------------------------------------------------------------------- -# HuggingFaceSource lazy / zero-arg construction (no network in __init__) -# --------------------------------------------------------------------------- - - -def test_hf_source_zero_arg_construction_does_no_work() -> None: - # Per the lazy / zero-arg convention: building the source must not touch the network and - # must succeed with no constructor arguments. Nothing is materialized until first use. - from sampleflux.sources import HuggingFaceSource - - src = HuggingFaceSource() - assert src._dataset is None # nothing loaded at construction time - # Even a fully-configured source stays unmaterialized until the dataset is accessed. - configured = HuggingFaceSource(path="some/dataset", split="test", count=7) - assert configured._dataset is None - assert configured.path == "some/dataset" and configured.split == "test" and configured.count == 7 - - -def test_hf_source_dataset_without_path_raises() -> None: - # The zero-arg constructor allows an unconfigured source, but materializing one without a - # dataset id cannot succeed — the error surfaces lazily, at the `dataset` property, not in __init__. - from sampleflux.sources import HuggingFaceSource - - src = HuggingFaceSource() - with pytest.raises(ValueError, match="path is empty"): - _ = src.dataset - - -# --------------------------------------------------------------------------- -# HuggingFaceSource.__len__ / count semantics (lazy `_dataset` pre-seeded, no network) -# --------------------------------------------------------------------------- - - -def _hf_source_with_count(count: Any, dataset_len: int = 13) -> Any: - """Build a HuggingFaceSource and pre-seed its lazy cache so `dataset` never hits the network.""" - from sampleflux.sources import HuggingFaceSource - - src: Any = HuggingFaceSource(count=count) - src._dataset = list(range(dataset_len)) # short-circuits the lazy load in the `dataset` property - return src - - -def test_hf_source_len_count_zero_means_all() -> None: - # Regression: count=0 must report the full length (matching __iter__'s ``count or len``), - # not 0 — otherwise a len()-based stepper (e.g. FluxStudio's WalkDataset) sees an empty - # source even though iteration yields every sample. - assert len(_hf_source_with_count(0)) == 13 - - -def test_hf_source_len_count_none_means_all() -> None: - assert len(_hf_source_with_count(None)) == 13 - - -def test_hf_source_len_positive_count_caps() -> None: - assert len(_hf_source_with_count(5)) == 5 - - -# --------------------------------------------------------------------------- -# HuggingFaceSource.metadata_features resolution ("*" sentinel = the rest) -# --------------------------------------------------------------------------- - - -def test_resolve_metadata_features_none_and_empty_mean_none() -> None: - from sampleflux.sources import _resolve_metadata_features - - cols = ["image", "label", "id", "source_file"] - assert _resolve_metadata_features(None, cols, "image", "label") == [] - assert _resolve_metadata_features([], cols, "image", "label") == [] - - -def test_resolve_metadata_features_explicit_list_verbatim() -> None: - from sampleflux.sources import _resolve_metadata_features - - cols = ["image", "label", "id", "source_file"] - assert _resolve_metadata_features(["id"], cols, "image", "label") == ["id"] - # used verbatim — names need not exist in column_names (caller's choice) - assert _resolve_metadata_features(["id", "extra"], cols, "image", "label") == ["id", "extra"] - - -def test_resolve_metadata_features_star_is_the_rest() -> None: - from sampleflux.sources import _resolve_metadata_features - - cols = ["image", "label", "id", "source_file"] - # the rest = every column except input/target, order preserved - assert _resolve_metadata_features(["*"], cols, "image", "label") == ["id", "source_file"] - # bare string form accepted (YAML users may write `metadata_features: "*"`) - assert _resolve_metadata_features("*", cols, "image", "label") == ["id", "source_file"] - - -def test_resolve_metadata_features_star_plus_extras_union() -> None: - from sampleflux.sources import _resolve_metadata_features - - cols = ["image", "label", "id"] - # "*" plus a name already in the rest -> no duplicate; an out-of-columns extra is appended - assert _resolve_metadata_features(["*", "id", "note"], cols, "image", "label") == ["id", "note"] - - -def test_resolve_metadata_features_star_without_columns_degrades() -> None: - from sampleflux.sources import _resolve_metadata_features - - # No column_names available (e.g. a non-Dataset backing) -> "*" yields just the extras. - assert _resolve_metadata_features(["*"], None, "image", "label") == [] - assert _resolve_metadata_features(["*", "note"], None, "image", "label") == ["note"] - - -class _StubHFDataset(list): - """A list of row-dicts that also exposes ``column_names`` like a real ``datasets.Dataset``. - - Lets the lazy ``resolved_metadata_features`` property expand the ``"*"`` sentinel against the - backing columns without touching the network — iterable + indexable + ``len``-able for free. - """ - - def __init__(self, rows: List[Any], column_names: List[str]) -> None: - super().__init__(rows) - self.column_names = column_names - - -def test_hf_source_iter_metadata_features_star_expands_on_real_dataset() -> None: - # End-to-end through __iter__: a dataset with extra columns + metadata_features="*" carries - # every non-input/target column onto its OWN aux Label field (plus the synthetic hf_path/hf_split). - # The "*" expansion is now lazy (resolved_metadata_features reads dataset.column_names). - from sampleflux import Image, Label, TypedSample - from sampleflux.sources import HuggingFaceSource - - rows = [{"image": i, "label": i % 2, "id": f"r{i}", "src": "a"} for i in range(3)] - src = HuggingFaceSource(path="fake/ds", split="train", metadata_features=["*"]) - src._dataset = _StubHFDataset(rows, ["image", "label", "id", "src"]) # pre-seed: no network - - samples = list(src) - assert all(isinstance(s, TypedSample) for s in samples) - - # input_feature -> "image" Image (role input); target_feature -> "class" Label (role target). - assert [int(s["image"]) for s in samples] == [0, 1, 2] - assert all(isinstance(s["image"], Image) and s.role_of("image") == "input" for s in samples) - assert [s["class"].value for s in samples] == [0, 1, 0] # label = i % 2 - assert all(isinstance(s["class"], Label) and s.role_of("class") == "target" for s in samples) - - # Each metadata column rides its own aux Label field, keyed by the column name. - s0 = samples[0] - assert s0["id"].value == "r0" and s0["src"].value == "a" - assert s0.role_of("id") == "aux" and s0.role_of("src") == "aux" - assert isinstance(s0["id"], Label) and isinstance(s0["src"], Label) - # input/target features are NOT duplicated as aux metadata fields. - assert s0.role_of("image") == "input" and s0.role_of("class") == "target" - - # Source provenance rides aux Label fields too. - assert s0["hf_path"].value == "fake/ds" and s0["hf_split"].value == "train" - assert s0.role_of("hf_path") == "aux" and s0.role_of("hf_split") == "aux" diff --git a/tests/test_storage.py b/tests/test_storage.py deleted file mode 100644 index e784674..0000000 --- a/tests/test_storage.py +++ /dev/null @@ -1,263 +0,0 @@ -from pathlib import Path -from typing import cast - -import confluid -import numpy as np -import torch - -from sampleflux.core import Flux -from sampleflux.sample import Sample -from sampleflux.storage.directory import DirectorySink -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource - - -def test_hdf5_storage(tmp_path: Path) -> None: - h5_path = tmp_path / "test.h5" - samples = [ - Sample(input=torch.randn(10), target=torch.tensor([1])), - Sample(input=torch.randn(10), target=torch.tensor([0])), - ] - - # Write - sink = HDF5Sink(h5_path, overwrite=True) - Flux(samples).to_sink(sink) - sink.close() - - # Read - source = HDF5Source(h5_path) - loaded = list(source) - assert len(loaded) == 2 - assert torch.allclose(loaded[0].input, samples[0].input) - assert loaded[0].target == samples[0].target - assert len(source) == 2 - source.close() - - -def test_hdf5_array_metadata_roundtrip(tmp_path: Path) -> None: - """Array-valued metadata (e.g. a segmentation mask) survives the HDF5 round-trip. - - Regression test: writing such a value as an HDF5 *attribute* overflows the attribute - size limit and the old str() fallback silently truncated it. It must now be stored as a - dataset under the per-sample ``{prefix}_meta/`` group and read back byte-exact, while - scalar metadata continues to round-trip via attributes. - """ - h5_path = tmp_path / "meta.h5" - mask = np.random.randint(0, 2, size=(128, 128), dtype=np.uint8) - samples = [ - Sample( - input=torch.randn(10), - target=torch.tensor([1]), - metadata={"id": "a", "samplerate": 100.0, "mask": mask}, - ), - Sample(input=torch.randn(10), metadata={"id": "b"}), - ] - - sink = HDF5Sink(h5_path, overwrite=True) - Flux(samples).to_sink(sink) - sink.close() - - source = HDF5Source(h5_path) - loaded = list(source) - source.close() - - assert len(loaded) == 2 - # Scalar metadata round-trips via attributes. - assert loaded[0].meta["id"] == "a" - assert loaded[0].meta["samplerate"] == 100.0 - assert loaded[1].meta["id"] == "b" - # Array metadata round-trips exactly (no truncation). - assert np.array_equal(loaded[0].meta["mask"], mask) - # The sample without array metadata has no spurious mask key. - assert "mask" not in loaded[1].meta - - -def test_hdf5_array_metadata_no_compression(tmp_path: Path) -> None: - """Array metadata is stored as a dataset even when compression is disabled.""" - h5_path = tmp_path / "meta_nc.h5" - mask = np.arange(16, dtype=np.uint8).reshape(4, 4) - - sink = HDF5Sink(h5_path, compression=None, overwrite=True) - Flux([Sample(input=np.array([1.0]), metadata={"mask": mask})]).to_sink(sink) - sink.close() - - loaded = list(HDF5Source(h5_path)) - assert np.array_equal(loaded[0].meta["mask"], mask) - - -def test_zarr_group_storage(tmp_path: Path) -> None: - zarr_path = tmp_path / "test.zarr" - samples = [ - Sample(input=np.random.randn(5), metadata={"id": "a"}), - Sample(input=np.random.randn(10), metadata={"id": "b"}), - ] - - sink = ZarrGroupSink(zarr_path, overwrite=True) - Flux(samples).to_sink(sink) - - # Verification (ZarrGroupSink doesn't have a Source yet, but we check files) - assert zarr_path.exists() - assert (zarr_path / "sample_000000").exists() - assert (zarr_path / "sample_000001").exists() - - -def test_zarr_batch_storage(tmp_path: Path) -> None: - zarr_path = tmp_path / "batch.zarr" - samples = [Sample(input=np.ones((10, 10), dtype=np.float32)) for _ in range(5)] - - sink = ZarrBatchSink(zarr_path, shape=[10, 10], overwrite=True) - Flux(samples).to_sink(sink) - - # Check if data was written - import zarr - - z = zarr.open_array(store=f"{zarr_path}/data", mode="r") - assert z.shape == (5, 10, 10) - assert np.all(z[:] == 1.0) - - -def test_directory_storage(tmp_path: Path) -> None: - dir_path = tmp_path / "out_dir" - samples = [ - Sample(input=np.array([1, 2]), metadata={"name": "first"}), - Sample(input=np.array([3, 4]), metadata={"name": "second"}), - ] - - sink = DirectorySink(dir_path, overwrite=True) - Flux(samples).to_sink(sink) - - -def test_directory_storage_separate(tmp_path: Path) -> None: - dir_path = tmp_path / "out_dir_sep" - samples = [ - Sample(input=np.array([1, 2]), target=np.array([0])), - ] - - # use_npz=False hits lines 52-54 - sink = DirectorySink(dir_path, overwrite=True, use_npz=False) - Flux(samples).to_sink(sink) - - assert (dir_path / "000000" / "data.npy").exists() - assert (dir_path / "000000" / "target.npy").exists() - - -def test_hdf5_to_numpy_direct() -> None: - from sampleflux.storage.hdf5 import to_numpy - - # Hits line 19 - assert to_numpy(123) == 123 - - -def test_hdf5_flush(tmp_path: Path) -> None: - h5_path = tmp_path / "flush.h5" - sink = HDF5Sink(h5_path) - sink.open() - sink.flush() # Hits lines 107-110 - sink.close() - - -def test_hdf5_overwrite(tmp_path: Path) -> None: - h5_path = tmp_path / "over.h5" - s1 = [Sample(input=np.array([1]))] - s2 = [Sample(input=np.array([2]))] - - # 1. Write first - sink1 = HDF5Sink(h5_path, overwrite=True) - Flux(s1).to_sink(sink1) - sink1.close() - - # 2. Overwrite - sink2 = HDF5Sink(h5_path, overwrite=True) - Flux(s2).to_sink(sink2) - sink2.close() - - # 3. Verify only s2 exists - source = HDF5Source(h5_path) - loaded = list(source) - assert len(loaded) == 1 - assert loaded[0].input == 2 - - -def test_zarr_group_with_target(tmp_path: Path) -> None: - zarr_path = tmp_path / "target.zarr" - samples = [Sample(input=np.array([1]), target=np.array([0]))] - - sink = ZarrGroupSink(zarr_path, overwrite=True) - Flux(samples).to_sink(sink) - - import zarr - - z = zarr.open_group(str(zarr_path), mode="r") - grp = cast(zarr.Group, z["sample_000000"]) - assert "target" in grp - - -def test_zarr_group_source_roundtrip(tmp_path: Path) -> None: - zarr_path = tmp_path / "group_rt.zarr" - samples = [ - Sample(input=np.arange(5, dtype="float32"), target=np.array([1]), metadata={"id": "a"}), - Sample(input=np.arange(3, dtype="float32"), metadata={"id": "b"}), - ] - - Flux(samples).to_sink(ZarrGroupSink(zarr_path, overwrite=True)) - - source = ZarrGroupSource(zarr_path) - loaded = list(source) - assert len(loaded) == 2 - assert len(source) == 2 - # Input is returned as a tensor (matches HDF5Source); order matches write order. - assert torch.equal(loaded[0].input, torch.arange(5, dtype=torch.float32)) - assert torch.equal(loaded[1].input, torch.arange(3, dtype=torch.float32)) - # Target round-trips; absent target stays None. - assert np.array_equal(loaded[0].target, np.array([1])) - assert loaded[1].target is None - # Metadata round-trips via group attributes. - assert loaded[0].meta["id"] == "a" - assert loaded[1].meta["id"] == "b" - source.close() - - -def test_zarr_group_sink_handles_torch_tensors(tmp_path: Path) -> None: - """ZarrGroupSink writes torch-tensor input/target (e.g. streamed from HDF5Source). - - Regression: zarr's ``create_array`` can't read a torch tensor's dtype, so the sink - must convert via ``to_numpy`` first — otherwise a torch-tensor sample raises - ``TypeError: Cannot interpret 'torch.float32' as a data type``. - """ - zarr_path = tmp_path / "torch.zarr" - samples = [Sample(input=torch.arange(5, dtype=torch.float32), target=torch.tensor([1]))] - Flux(samples).to_sink(ZarrGroupSink(zarr_path, overwrite=True)) - - loaded = list(ZarrGroupSource(zarr_path)) - assert len(loaded) == 1 - assert torch.equal(loaded[0].input, torch.arange(5, dtype=torch.float32)) - assert np.array_equal(loaded[0].target, np.array([1])) - - -def test_zarr_batch_source_roundtrip(tmp_path: Path) -> None: - zarr_path = tmp_path / "batch_rt.zarr" - samples = [Sample(input=np.full((4,), i, dtype=np.float32)) for i in range(3)] - - Flux(samples).to_sink(ZarrBatchSink(zarr_path, shape=[4], overwrite=True)) - - source = ZarrBatchSource(zarr_path) - loaded = list(source) - assert len(loaded) == 3 - assert len(source) == 3 - # Batch sink stores input only — one Sample per row of the leading axis. - assert [int(s.input[0]) for s in loaded] == [0, 1, 2] - assert all(s.target is None for s in loaded) - source.close() - - -def test_zarr_sources_configurable_roundtrip(tmp_path: Path) -> None: - group_src = ZarrGroupSource(tmp_path / "g.zarr", target_key="label") - restored_group = confluid.load(confluid.dump(group_src)) - assert isinstance(restored_group, ZarrGroupSource) - assert restored_group.path == group_src.path - assert restored_group.target_key == "label" - - batch_src = ZarrBatchSource(tmp_path / "b.zarr") - restored_batch = confluid.load(confluid.dump(batch_src)) - assert isinstance(restored_batch, ZarrBatchSource) - assert restored_batch.path == batch_src.path diff --git a/tests/test_structure_ops.py b/tests/test_structure_ops.py index a9fa405..e173d3b 100644 --- a/tests/test_structure_ops.py +++ b/tests/test_structure_ops.py @@ -3,12 +3,12 @@ import numpy as np import pytest -from sampleflux import Image, Label, Regions, TypedSample, primary +from sampleflux import Image, Label, Regions, Sample, primary from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole -def _sample() -> TypedSample: - return TypedSample( +def _sample() -> Sample: + return Sample( {"image": Image(np.zeros((2, 2, 3))), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")}, roles={"regions": "target", "class": "target"}, ) @@ -99,15 +99,15 @@ def test_primary_missing_role_raises(self) -> None: primary(_sample(), "pred") def test_merge_union_last_wins(self) -> None: - a = TypedSample({"x": Label("a"), "shared": Label("from_a")}) - b = TypedSample({"y": Label("b"), "shared": Label("from_b")}, roles={"shared": "target"}) - m = TypedSample.merge(a, b) + a = Sample({"x": Label("a"), "shared": Label("from_a")}) + b = Sample({"y": Label("b"), "shared": Label("from_b")}, roles={"shared": "target"}) + m = Sample.merge(a, b) assert list(m.keys()) == ["x", "shared", "y"] # union keeps first-seen position assert m["shared"].value == "from_b" and m.role_of("shared") == "target" # last wins, role travels def test_merge_rejects_non_sample(self) -> None: - with pytest.raises(TypeError, match="expected TypedSample"): - TypedSample.merge(_sample(), "nope") # type: ignore[arg-type] + with pytest.raises(TypeError, match="expected Sample"): + Sample.merge(_sample(), "nope") # type: ignore[arg-type] def test_configurable_marks(self) -> None: for cls in (SetRole, RenameField, DropField, CopyField, SelectFields): diff --git a/tests/test_target_ops.py b/tests/test_target_ops.py deleted file mode 100644 index 0196c37..0000000 --- a/tests/test_target_ops.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Tests for the target movers / encoders (``sampleflux.ops.target``).""" - -import numpy as np -import pytest -from PIL import Image - -from sampleflux.ops.target import ( - CocoToTorchVisionDetectionOp, - DecodeTargetOp, - EncodeTargetOp, - MasksToDetectionBoxesOp, - MetadataToTargetOp, -) -from sampleflux.sample import Sample - - -# --------------------------------------------------------------------------- # -# MetadataToTargetOp -# --------------------------------------------------------------------------- # -def test_metadata_to_target_moves_value() -> None: - out = MetadataToTargetOp(key="drone")(Sample(input=0, metadata={"drone": "DJI MINI3"})) - assert out.target == "DJI MINI3" - - -def test_metadata_to_target_leaves_metadata_untouched_without_target_key() -> None: - sample = Sample(input=0, metadata={"drone": "DJI MINI3"}) - out = MetadataToTargetOp(key="drone")(sample) - assert set(out.meta) == {"drone"} - - -def test_metadata_to_target_copies_to_target_key() -> None: - sample = Sample(input=0, metadata={"drone": "DJI MINI3"}) - out = MetadataToTargetOp(key="drone", target_key="raw_label")(sample) - assert out.target == "DJI MINI3" - assert out.meta["raw_label"] == "DJI MINI3" - - -def test_metadata_to_target_missing_key_raises() -> None: - with pytest.raises(KeyError, match="no key 'drone'"): - MetadataToTargetOp(key="drone")(Sample(input=0, metadata={"other": 1})) - - -# --------------------------------------------------------------------------- # -# EncodeTargetOp -# --------------------------------------------------------------------------- # -def test_encode_target_maps_known_value() -> None: - op = EncodeTargetOp(mapping={"DJI AVATA2": 2, "DJI MINI3": 5}) - assert op(Sample(input=0, target="DJI MINI3")).target == 5 - - -def test_encode_target_class_zero_allowed() -> None: - op = EncodeTargetOp(mapping={"first": 0, "second": 1}) - assert op(Sample(input=0, target="first")).target == 0 - - -def test_encode_target_unknown_raises() -> None: - op = EncodeTargetOp(mapping={"a": 1}) - with pytest.raises(KeyError, match="not in mapping"): - op(Sample(input=0, target="missing")) - - -def test_encode_target_unknown_substitutes_default_when_ignored() -> None: - op = EncodeTargetOp(mapping={"a": 1}, ignore_unknown=True, default=7) - assert op(Sample(input=0, target="missing")).target == 7 - - -def test_encode_target_empty_mapping_rejected() -> None: - op = EncodeTargetOp(mapping={}) # lazy: construction succeeds - with pytest.raises(ValueError, match="at least one entry"): - op(Sample(input=0, target="x")) - - -# --------------------------------------------------------------------------- # -# DecodeTargetOp -# --------------------------------------------------------------------------- # -def test_decode_target_inverts_encode() -> None: - mapping = {"DJI AVATA2": 2, "DJI MINI3": 5} - inverse = {v: k for k, v in mapping.items()} - sample = Sample(input=0, target="DJI MINI3") - encoded = EncodeTargetOp(mapping=mapping)(sample) - decoded = DecodeTargetOp(mapping=inverse)(encoded) - assert decoded.target == "DJI MINI3" - - -def test_decode_target_unknown_default_is_none() -> None: - op = DecodeTargetOp(mapping={1: "a"}, ignore_unknown=True) - assert op(Sample(input=0, target=999)).target is None - - -def test_decode_target_empty_mapping_rejected() -> None: - op = DecodeTargetOp(mapping={}) # lazy: construction succeeds - with pytest.raises(ValueError, match="at least one entry"): - op(Sample(input=0, target=1)) - - -# --------------------------------------------------------------------------- # -# Composed chain (the decomposed classification label path) -# --------------------------------------------------------------------------- # -def test_metadata_to_target_then_encode() -> None: - label_to_index = {"DJI AVATA2": 2, "DJI MINI3": 5} - sample = Sample(input=0, metadata={"drone": "DJI AVATA2"}) - sample = MetadataToTargetOp(key="drone", target_key="raw_label")(sample) - sample = EncodeTargetOp(mapping=label_to_index)(sample) - assert sample.target == 2 - # raw label preserved for decode/reporting - assert sample.meta["raw_label"] == "DJI AVATA2" - - -# --------------------------------------------------------------------------- # -# CocoToTorchVisionDetectionOp (HF / COCO objects -> {boxes xyxy, labels}) -# --------------------------------------------------------------------------- # -def _objects_sample(bbox: object, category: object) -> Sample: - """A Sample shaped like ``HuggingFaceSource(target_feature='objects')`` output.""" - return Sample(input="img", target={"bbox": bbox, "category": category}, metadata={}) - - -def test_objects_to_boxes_xywh_to_xyxy_and_dtypes() -> None: - op = CocoToTorchVisionDetectionOp() # default bbox_format="xywh" - out = op(_objects_sample([[10, 20, 30, 40]], [2])) - # COCO [x,y,w,h]=[10,20,30,40] -> xyxy [10,20,40,60] - assert out.target["boxes"].tolist() == [[10.0, 20.0, 40.0, 60.0]] - assert out.target["labels"].tolist() == [2] - assert str(out.target["boxes"].dtype) == "torch.float32" - assert str(out.target["labels"].dtype) == "torch.int64" - - -def test_objects_to_boxes_label_offset_for_background_class() -> None: - # label_offset=1 shifts 0-indexed dataset categories to torchvision foreground ids 1..K. - op = CocoToTorchVisionDetectionOp(label_offset=1) - out = op(_objects_sample([[0, 0, 4, 4], [1, 1, 2, 2]], [0, 3])) - assert out.target["labels"].tolist() == [1, 4] - - -def test_objects_to_boxes_xyxy_passthrough() -> None: - op = CocoToTorchVisionDetectionOp(bbox_format="xyxy") - out = op(_objects_sample([[1, 2, 3, 4]], [0])) - assert out.target["boxes"].tolist() == [[1.0, 2.0, 3.0, 4.0]] - - -def test_objects_to_boxes_cxcywh() -> None: - op = CocoToTorchVisionDetectionOp(bbox_format="cxcywh") - # center (50,50), size (20,40) -> [40,30,60,70] - out = op(_objects_sample([[50, 50, 20, 40]], [1])) - assert out.target["boxes"].tolist() == [[40.0, 30.0, 60.0, 70.0]] - - -def test_objects_to_boxes_empty_annotation_yields_empty_tensors() -> None: - op = CocoToTorchVisionDetectionOp() - out = op(_objects_sample([], [])) - assert tuple(out.target["boxes"].shape) == (0, 4) - assert tuple(out.target["labels"].shape) == (0,) - - -def test_objects_to_boxes_custom_keys() -> None: - op = CocoToTorchVisionDetectionOp(bbox_key="boxes", category_key="labels") - out = op(Sample(input="i", target={"boxes": [[0, 0, 2, 2]], "labels": [5]}, metadata={})) - assert out.target["boxes"].tolist() == [[0.0, 0.0, 2.0, 2.0]] - assert out.target["labels"].tolist() == [5] - - -def test_objects_to_boxes_rejects_non_mapping_target() -> None: - with pytest.raises(TypeError, match="objects mapping"): - CocoToTorchVisionDetectionOp()(Sample(input="i", target=[1, 2, 3], metadata={})) - - -# --------------------------------------------------------------------------- # -# MasksToDetectionBoxesOp (segmentation mask -> {boxes xyxy, labels}) -# --------------------------------------------------------------------------- # -def _instance_mask() -> np.ndarray: - """Two objects: instance id 1 at rows 2-4/cols 1-3, id 2 at rows 6-8/cols 7-10.""" - m = np.zeros((10, 12), dtype=np.uint8) - m[2:5, 1:4] = 1 - m[6:9, 7:11] = 2 - return m - - -def test_masks_instance_mode_per_id_bbox_and_dtypes() -> None: - op = MasksToDetectionBoxesOp(label=1) # default connected=False - out = op(Sample(input="img", target=Image.fromarray(_instance_mask(), mode="L"), metadata={})) - # row/col extents → xyxy with exclusive far edge. - assert sorted(out.target["boxes"].tolist()) == [[1.0, 2.0, 4.0, 5.0], [7.0, 6.0, 11.0, 9.0]] - assert out.target["labels"].tolist() == [1, 1] - assert str(out.target["boxes"].dtype) == "torch.float32" - assert str(out.target["labels"].dtype) == "torch.int64" - - -def test_masks_label_assigns_class_id() -> None: - out = MasksToDetectionBoxesOp(label=3)(Sample(input="i", target=_instance_mask(), metadata={})) - assert out.target["labels"].tolist() == [3, 3] - - -def test_masks_connected_mode_splits_semantic_blobs() -> None: - # A SEMANTIC mask (both objects = 1) — connected components separate the two blobs. - sem = (_instance_mask() > 0).astype(np.uint8) - out = MasksToDetectionBoxesOp(connected=True)(Sample(input="i", target=sem, metadata={})) - assert sorted(out.target["boxes"].tolist()) == [[1.0, 2.0, 4.0, 5.0], [7.0, 6.0, 11.0, 9.0]] - assert out.target["labels"].tolist() == [1, 1] - - -def test_masks_min_area_drops_small_instances() -> None: - m = np.zeros((8, 8), dtype=np.uint8) - m[0, 0] = 1 # area 1 - m[4:7, 4:7] = 2 # area 9 - out = MasksToDetectionBoxesOp(min_area=2)(Sample(input="i", target=m, metadata={})) - assert out.target["boxes"].tolist() == [[4.0, 4.0, 7.0, 7.0]] - - -def test_masks_empty_mask_yields_empty_tensors() -> None: - out = MasksToDetectionBoxesOp()(Sample(input="i", target=np.zeros((5, 5), dtype=np.uint8), metadata={})) - assert tuple(out.target["boxes"].shape) == (0, 4) - assert tuple(out.target["labels"].shape) == (0,) - - -def test_masks_rejects_non_2d_target() -> None: - with pytest.raises(TypeError, match="2-D segmentation mask"): - MasksToDetectionBoxesOp()(Sample(input="i", target=np.zeros((4, 4, 3), dtype=np.uint8), metadata={})) diff --git a/tests/test_transform_chain.py b/tests/test_transform_chain.py deleted file mode 100644 index 73fb7a1..0000000 --- a/tests/test_transform_chain.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Tests for :class:`sampleflux.ops.transform_chain.TransformChain`.""" - -from typing import List, Optional - -from sampleflux.ops.transform_chain import TransformChain -from sampleflux.sample import Sample - - -def _s(v: int = 0) -> Sample: - return Sample(input=v, target=None, metadata={}) - - -class _AddOp: - """Increment sample.input by a fixed delta.""" - - def __init__(self, delta: int = 1) -> None: - self.delta = delta - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + self.delta) - - -class _TagOp: - """Append a string tag to metadata['tags'].""" - - def __init__(self, tag: str) -> None: - self.tag = tag - - def __call__(self, sample: Sample) -> Sample: - new_meta = dict(sample.meta) - new_meta["tags"] = new_meta.get("tags", []) + [self.tag] - return sample._replace(metadata=new_meta) - - -class _DropOp: - """Always returns None — simulates a filter op.""" - - def __call__(self, sample: Sample) -> Optional[Sample]: - return None - - -class _ClosableOp: - def __init__(self, name: str, log: List[str]) -> None: - self.name = name - self.log = log - - def __call__(self, sample: Sample) -> Sample: - return sample - - def close(self) -> None: - self.log.append(self.name) - - -# --------------------------------------------------------------------------- -# Core behaviour -# --------------------------------------------------------------------------- - - -def test_zero_arg_construction() -> None: - """TransformChain() must construct with no arguments (lazy convention).""" - chain = TransformChain() - assert chain.ops == [] - - -def test_empty_chain_is_identity() -> None: - """An empty TransformChain passes the sample through unchanged.""" - chain = TransformChain() - s = _s(42) - out = chain(s) - assert out is s - - -def test_happy_path_multiple_ops_applied_in_order() -> None: - """Ops fire left-to-right; each op sees the output of the previous one.""" - chain = TransformChain(ops=[_AddOp(1), _AddOp(2), _AddOp(3)]) - out = chain(_s(0)) - assert out is not None - assert out.input == 6 # 0 + 1 + 2 + 3 - - -def test_ops_applied_in_declared_order_via_metadata_tags() -> None: - """Ordering is visible: tags accumulate in declaration order.""" - chain = TransformChain(ops=[_TagOp("a"), _TagOp("b"), _TagOp("c")]) - out = chain(_s()) - assert out is not None - assert out.meta["tags"] == ["a", "b", "c"] - - -def test_none_propagation_stops_chain_early() -> None: - """If any op returns None the chain stops and propagates None.""" - called: List[str] = [] - - class _RecordOp: - def __init__(self, tag: str) -> None: - self.tag = tag - - def __call__(self, sample: Sample) -> Sample: - called.append(self.tag) - return sample - - chain = TransformChain(ops=[_RecordOp("before"), _DropOp(), _RecordOp("after")]) - out = chain(_s()) - assert out is None - assert called == ["before"] # "after" must NOT have fired - - -def test_none_propagation_from_first_op() -> None: - """None returned by the very first op also short-circuits the chain.""" - chain = TransformChain(ops=[_DropOp(), _AddOp(99)]) - out = chain(_s(0)) - assert out is None - - -def test_fluid_resolution_lazy_and_cached() -> None: - """Confluid Fluid markers inside ops are resolved on first call and cached.""" - from confluid import configurable - from confluid.fluid import Class, Fluid - - @configurable - class _Inner: - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + 10) - - fluid_op = Class(_Inner) - chain = TransformChain(ops=[fluid_op]) - - out1 = chain(_s(5)) - assert out1 is not None - assert out1.input == 15 - - # Slot must now hold the resolved instance, not a Fluid. - assert not isinstance(chain.ops[0], Fluid) - - out2 = chain(_s(5)) - assert out2 is not None - assert out2.input == 15 - - -def test_close_propagates_to_all_inner_ops() -> None: - """close() forwards to every inner op that implements it.""" - log: List[str] = [] - chain = TransformChain(ops=[_ClosableOp("x", log), _ClosableOp("y", log)]) - chain.close() - assert log == ["x", "y"] - - -def test_close_on_empty_chain_is_safe() -> None: - """close() on an empty chain must not raise.""" - TransformChain().close() # must not raise - - -def test_close_skips_ops_without_close_method() -> None: - """close() only calls close on ops that have it — no AttributeError on plain callables.""" - log: List[str] = [] - chain = TransformChain(ops=[_AddOp(1), _ClosableOp("z", log)]) - chain.close() - assert log == ["z"] diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py index 798ef45..0b92f0f 100644 --- a/tests/test_typed_collate.py +++ b/tests/test_typed_collate.py @@ -1,4 +1,4 @@ -"""The typed collate — batched TypedSample convention (golden shapes consumers rely on).""" +"""The typed collate — batched Sample convention (golden shapes consumers rely on).""" from dataclasses import dataclass @@ -6,7 +6,7 @@ import pytest import torch -from sampleflux import Image, Label, Mask, TypedSample, collate, get_collate, register_item +from sampleflux import Image, Label, Mask, Sample, collate, get_collate, register_item @register_item @@ -16,8 +16,8 @@ class _CollateBlob: rate: float = 1.0 -def _sample(i: int) -> TypedSample: - return TypedSample( +def _sample(i: int) -> Sample: + return Sample( { "image": Image(np.full((4, 5, 3), float(i), dtype=np.float32)), "mask": Mask(np.full((4, 5), i, dtype=np.int64)), @@ -29,10 +29,10 @@ def _sample(i: int) -> TypedSample: class TestTypedCollate: def test_golden_shapes(self) -> None: - # THE batch convention consumers rely on: batched TypedSample, payloads stacked + # THE batch convention consumers rely on: batched Sample, payloads stacked # per field, per-item attrs as lists, roles preserved. batch = collate([_sample(0), _sample(1), _sample(2)]) - assert isinstance(batch, TypedSample) + assert isinstance(batch, Sample) assert np.asarray(batch["image"]).shape == (3, 4, 5, 3) # stacked payload assert np.asarray(batch["mask"]).shape == (3, 4, 5) assert batch["class"].value == [0, 1, 2] # per-item attrs become lists @@ -41,22 +41,21 @@ def test_golden_shapes(self) -> None: def test_auto_dispatch_and_explicit_key(self) -> None: samples = [_sample(0), _sample(1)] - auto = collate(samples) # TypedSample batch routes to "typed" automatically + auto = collate(samples) # Sample batch routes to "typed" automatically explicit = get_collate("typed")(samples) - assert isinstance(auto, TypedSample) and isinstance(explicit, TypedSample) + assert isinstance(auto, Sample) and isinstance(explicit, Sample) assert np.array_equal(np.asarray(auto["image"]), np.asarray(explicit["image"])) def test_torch_payloads_stack_to_tensor(self) -> None: samples = [ - TypedSample({"sig": _CollateBlob(torch.ones(8) * i, rate=float(i))}, roles={"sig": "input"}) - for i in range(2) + Sample({"sig": _CollateBlob(torch.ones(8) * i, rate=float(i))}, roles={"sig": "input"}) for i in range(2) ] batch = collate(samples) assert isinstance(batch["sig"].data, torch.Tensor) and batch["sig"].data.shape == (2, 8) assert batch["sig"].rate == [0.0, 1.0] def test_heterogeneous_batch_raises(self) -> None: - odd = TypedSample({"other": Label("x")}) + odd = Sample({"other": Label("x")}) with pytest.raises(ValueError, match="do not match the batch fields"): collate([_sample(0), odd]) @@ -65,5 +64,5 @@ def test_empty_batch_raises(self) -> None: get_collate("typed")([]) def test_non_typed_items_raise(self) -> None: - with pytest.raises(TypeError, match="expected TypedSample"): + with pytest.raises(TypeError, match="expected Sample"): get_collate("typed")([1, 2, 3]) diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py index a0c7273..ca61cfa 100644 --- a/tests/test_typed_detection_target_ops.py +++ b/tests/test_typed_detection_target_ops.py @@ -1,6 +1,6 @@ """Typed-bag TWINS of the two detection target-shaping ops. -Pins the native typed transforms that let a ``TypedSample`` detection pipeline build its +Pins the native typed transforms that let a ``Sample`` detection pipeline build its torchvision-style ``{boxes, labels}`` target as a :class:`~sampleflux.Regions` item without the legacy ``Sample`` path: @@ -18,15 +18,14 @@ import torch from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Label, Mask, Regions, TypedSample +from sampleflux import Image, Label, Mask, Regions, Sample from sampleflux.collate import typed_collate from sampleflux.ops.target import ( CocoToTorchVisionDetection, - CocoToTorchVisionDetectionOp, MasksToDetectionBoxes, - MasksToDetectionBoxesOp, + coco_to_detection, + masks_to_detection, ) -from sampleflux.sample import Sample # A COCO / HF objects annotation: two boxes in [x, y, w, h] pixels + integer categories. _OBJECTS = {"bbox": [[10.0, 20.0, 30.0, 40.0], [5.0, 6.0, 7.0, 8.0]], "category": [1, 3]} @@ -46,7 +45,7 @@ def _instance_mask() -> np.ndarray: # --------------------------------------------------------------------------- # class TestCocoToTorchVisionDetection: def test_produces_target_regions(self) -> None: - s = TypedSample({"objects": Label(_OBJECTS)}, roles={"objects": "aux"}) + s = Sample({"objects": Label(_OBJECTS)}, roles={"objects": "aux"}) out = CocoToTorchVisionDetection(field="objects")(s) regions = out["target"] assert isinstance(regions, Regions) @@ -56,35 +55,33 @@ def test_produces_target_regions(self) -> None: assert regions.boxes.shape == (2, 4) assert regions.labels.shape == (2,) - def test_parity_with_legacy(self) -> None: - typed = CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label(_OBJECTS)})) - legacy = CocoToTorchVisionDetectionOp()(Sample(input=None, target=_OBJECTS, metadata={})).target - assert torch.equal(typed["target"].boxes, legacy["boxes"]) - assert torch.equal(typed["target"].labels, legacy["labels"]) + def test_parity_with_helper(self) -> None: + typed = CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label(_OBJECTS)})) + expected = coco_to_detection(_OBJECTS) + assert torch.equal(typed["target"].boxes, expected["boxes"]) + assert torch.equal(typed["target"].labels, expected["labels"]) def test_parity_xyxy_and_label_offset(self) -> None: objects = {"bbox": [[10.0, 20.0, 40.0, 60.0]], "category": [2]} typed = CocoToTorchVisionDetection(field="objects", bbox_format="xyxy", label_offset=1)( - TypedSample({"objects": Label(objects)}) + Sample({"objects": Label(objects)}) ) - legacy = CocoToTorchVisionDetectionOp(bbox_format="xyxy", label_offset=1)( - Sample(input=None, target=objects, metadata={}) - ).target - assert torch.equal(typed["target"].boxes, legacy["boxes"]) - assert torch.equal(typed["target"].labels, legacy["labels"]) + expected = coco_to_detection(objects, bbox_format="xyxy", label_offset=1) + assert torch.equal(typed["target"].boxes, expected["boxes"]) + assert torch.equal(typed["target"].labels, expected["labels"]) def test_empty_annotation_yields_empty_tensors(self) -> None: - out = CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label({"bbox": [], "category": []})})) + out = CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label({"bbox": [], "category": []})})) assert out["target"].boxes.shape == (0, 4) assert out["target"].labels.shape == (0,) def test_default_picks_first_label(self) -> None: - s = TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "objects": Label(_OBJECTS)}) + s = Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "objects": Label(_OBJECTS)}) out = CocoToTorchVisionDetection()(s) assert out["target"].boxes.shape == (2, 4) def test_new_output_field_keeps_source(self) -> None: - s = TypedSample({"objects": Label(_OBJECTS)}) + s = Sample({"objects": Label(_OBJECTS)}) out = CocoToTorchVisionDetection(field="objects", output="det")(s) assert isinstance(out["det"], Regions) assert out.role_of("det") == "target" @@ -92,16 +89,16 @@ def test_new_output_field_keeps_source(self) -> None: def test_missing_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - CocoToTorchVisionDetection(field="nope")(TypedSample({"objects": Label(_OBJECTS)})) + CocoToTorchVisionDetection(field="nope")(Sample({"objects": Label(_OBJECTS)})) def test_empty_sample_raises(self) -> None: with pytest.raises(ValueError, match="sample is empty"): - CocoToTorchVisionDetection()(TypedSample({})) + CocoToTorchVisionDetection()(Sample({})) def test_non_dict_source_raises(self) -> None: # The reused legacy op rejects a non-objects-shaped value loudly. with pytest.raises(TypeError, match="objects mapping"): - CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label("not a dict")})) + CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label("not a dict")})) # --------------------------------------------------------------------------- # @@ -109,7 +106,7 @@ def test_non_dict_source_raises(self) -> None: # --------------------------------------------------------------------------- # class TestMasksToDetectionBoxes: def test_instance_mask_produces_target_regions(self) -> None: - s = TypedSample({"mask": Mask(_instance_mask())}, roles={"mask": "aux"}) + s = Sample({"mask": Mask(_instance_mask())}, roles={"mask": "aux"}) out = MasksToDetectionBoxes(field="mask")(s) regions = out["target"] assert isinstance(regions, Regions) @@ -119,40 +116,40 @@ def test_instance_mask_produces_target_regions(self) -> None: assert regions.boxes.shape == (3, 4) # three instances assert regions.labels.tolist() == [1, 1, 1] # every box → foreground class 1 - def test_instance_parity_with_legacy(self) -> None: + def test_instance_parity_with_helper(self) -> None: mask = _instance_mask() - typed = MasksToDetectionBoxes(field="mask")(TypedSample({"mask": Mask(mask)})) - legacy = MasksToDetectionBoxesOp()(Sample(input=None, target=mask, metadata={})).target - assert torch.equal(typed["target"].boxes, legacy["boxes"]) - assert torch.equal(typed["target"].labels, legacy["labels"]) + typed = MasksToDetectionBoxes(field="mask")(Sample({"mask": Mask(mask)})) + expected = masks_to_detection(mask) + assert torch.equal(typed["target"].boxes, expected["boxes"]) + assert torch.equal(typed["target"].labels, expected["labels"]) def test_connected_components_parity(self) -> None: # A binary/semantic mask (all objects share value 1): connected=True splits into blobs. binary = (_instance_mask() != 0).astype(np.uint8) - typed = MasksToDetectionBoxes(field="mask", connected=True, label=2)(TypedSample({"mask": Mask(binary)})) - legacy = MasksToDetectionBoxesOp(connected=True, label=2)(Sample(input=None, target=binary, metadata={})).target + typed = MasksToDetectionBoxes(field="mask", connected=True, label=2)(Sample({"mask": Mask(binary)})) + expected = masks_to_detection(binary, connected=True, label=2) assert typed["target"].boxes.shape[0] == 3 # three connected blobs - assert torch.equal(typed["target"].boxes, legacy["boxes"]) - assert torch.equal(typed["target"].labels, legacy["labels"]) + assert torch.equal(typed["target"].boxes, expected["boxes"]) + assert torch.equal(typed["target"].labels, expected["labels"]) def test_min_area_drops_small_instances(self) -> None: mask = _instance_mask() - typed = MasksToDetectionBoxes(field="mask", min_area=10)(TypedSample({"mask": Mask(mask)})) - legacy = MasksToDetectionBoxesOp(min_area=10)(Sample(input=None, target=mask, metadata={})).target - assert torch.equal(typed["target"].boxes, legacy["boxes"]) + typed = MasksToDetectionBoxes(field="mask", min_area=10)(Sample({"mask": Mask(mask)})) + expected = masks_to_detection(mask, min_area=10) + assert torch.equal(typed["target"].boxes, expected["boxes"]) def test_empty_mask_yields_empty_tensors(self) -> None: - out = MasksToDetectionBoxes(field="mask")(TypedSample({"mask": Mask(np.zeros((4, 4), dtype=np.uint8))})) + out = MasksToDetectionBoxes(field="mask")(Sample({"mask": Mask(np.zeros((4, 4), dtype=np.uint8))})) assert out["target"].boxes.shape == (0, 4) assert out["target"].labels.shape == (0,) def test_default_picks_first_mask(self) -> None: - s = TypedSample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "seg": Mask(_instance_mask())}) + s = Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "seg": Mask(_instance_mask())}) out = MasksToDetectionBoxes()(s) assert out["target"].boxes.shape == (3, 4) def test_new_output_field_keeps_source(self) -> None: - s = TypedSample({"mask": Mask(_instance_mask())}) + s = Sample({"mask": Mask(_instance_mask())}) out = MasksToDetectionBoxes(field="mask", output="det")(s) assert isinstance(out["det"], Regions) assert out.role_of("det") == "target" @@ -160,20 +157,20 @@ def test_new_output_field_keeps_source(self) -> None: def test_missing_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - MasksToDetectionBoxes(field="nope")(TypedSample({"mask": Mask(_instance_mask())})) + MasksToDetectionBoxes(field="nope")(Sample({"mask": Mask(_instance_mask())})) def test_no_mask_or_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no Mask or array-bearing field"): - MasksToDetectionBoxes()(TypedSample({"lbl": Label("x")})) + MasksToDetectionBoxes()(Sample({"lbl": Label("x")})) # --------------------------------------------------------------------------- # # Collate — per-sample Regions gather into a list of detection targets. # --------------------------------------------------------------------------- # def test_typed_collate_gathers_regions_as_list() -> None: - a = CocoToTorchVisionDetection(field="objects")(TypedSample({"objects": Label(_OBJECTS)})) + a = CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label(_OBJECTS)})) c = CocoToTorchVisionDetection(field="objects")( - TypedSample({"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [5]})}) + Sample({"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [5]})}) ) batch = typed_collate([a, c]) # Variable-N boxes can't be stacked → the collate gathers them as a per-sample list of tensors. diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index 1b66b9e..174e1c0 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from sampleflux import FlowGraph, Flux, Image, Label, Mask, Transform, TypedSample, to_ops +from sampleflux import FlowGraph, Flux, Image, Label, Mask, Sample, Transform, to_ops from sampleflux.flow import from_ops, parse_flow from sampleflux.ops.context import MergeFields from sampleflux.ops.structure import RenameField, SetRole @@ -20,7 +20,7 @@ def __init__(self, offset: float = 0.0, only: Optional[List[str]] = None) -> Non super().__init__(only=only) self.offset = offset - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: out = sample for key, item in sample.items(): if isinstance(item, Image) and (self.only is None or key in self.only): @@ -31,14 +31,14 @@ def __call__(self, sample: TypedSample) -> TypedSample: class _MakeMask(Transform): """Derives a Mask field from the first Image (a branch producer).""" - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: image = next(item for item in sample.fields.values() if isinstance(item, Image)) out = sample.replace_field("mask", Mask(np.asarray(image)[..., 0] > 0.5)) return out.set_role("mask", "target") -def _seed(value: float = 0.0) -> TypedSample: - return TypedSample( +def _seed(value: float = 0.0) -> Sample: + return Sample( {"image": Image(np.full((2, 3, 3), value, dtype=np.float32)), "label": Label("x")}, roles={"label": "target"}, ) @@ -48,7 +48,7 @@ class TestTypedFlowGraph: def test_linear_typed_flow(self) -> None: graph = FlowGraph(source=[_seed(1.0)], flow={"plus": _AddOffset(offset=2.0)}) (out,) = list(graph) - assert isinstance(out, TypedSample) and np.allclose(np.asarray(out["image"]), 3.0) + assert isinstance(out, Sample) and np.allclose(np.asarray(out["image"]), 3.0) def test_merge_from_union(self) -> None: # Fork: derive a mask on a branch, SELECT the new field, union it back into the main @@ -100,7 +100,7 @@ def __init__(self, item: Any = None) -> None: super().__init__() self.item = item - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: offset = float(np.asarray(self.item).mean()) out = sample for key, value in sample.items(): @@ -122,7 +122,7 @@ def __init__(self, item: Any = None) -> None: super().__init__() self.item = item - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: assert isinstance(self.item, Image) # primary input-role field of the bound step return sample @@ -132,21 +132,23 @@ def __call__(self, sample: TypedSample) -> TypedSample: "final": {"op": _CapturePrimary(), "from": "start", "bind": {"item": "probe"}}, } (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) - assert isinstance(out, TypedSample) + assert isinstance(out, Sample) - def test_typed_step_with_legacy_fanin_raises(self) -> None: + def test_legacy_fanin_key_removed(self) -> None: + # target_from / metadata_from (the legacy Sample fan-in) were purged; they are now + # unknown step keys — a flow document using one fails loudly at parse. flow = { "start": {}, "a": {"op": _AddOffset(offset=1.0), "from": "start"}, "out": {"from": "a", "target_from": "start"}, } graph = FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out") - with pytest.raises(TypeError, match="LEGACY fan-in"): + with pytest.raises(ValueError, match="unknown step key"): list(graph) - def test_merge_and_legacy_fanin_mutually_exclusive(self) -> None: - with pytest.raises(ValueError, match="mutually exclusive"): - parse_flow({"a": {}, "b": {"from": "a", "merge_from": ["a"], "target_from": "a"}}) + def test_metadata_from_key_removed(self) -> None: + with pytest.raises(ValueError, match="unknown step key"): + parse_flow({"a": {}, "b": {"from": "a", "merge_from": ["a"], "metadata_from": "a"}}) def test_merge_from_forward_ref_raises(self) -> None: with pytest.raises(ValueError, match="EARLIER step"): @@ -188,7 +190,7 @@ def __init__(self, item: Any = None) -> None: super().__init__() self.item = item - def __call__(self, sample: TypedSample) -> TypedSample: + def __call__(self, sample: Sample) -> Sample: return sample.replace_field("echo", self.item) flow = { @@ -208,10 +210,10 @@ def __call__(self, sample: TypedSample) -> TypedSample: class TestTypedThroughFlux: def test_default_flux_carries_typed_verbatim(self) -> None: - # No native=True needed: a TypedSample source item is NEVER coerced to legacy Sample. + # No native=True needed: a Sample source item is NEVER coerced to legacy Sample. flux = Flux(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) (out,) = list(flux) - assert isinstance(out, TypedSample) and np.allclose(np.asarray(out["image"]), 2.0) + assert isinstance(out, Sample) and np.allclose(np.asarray(out["image"]), 2.0) def test_getitem_typed(self) -> None: flux = Flux(source=[_seed(1.0), _seed(2.0)], ops=[SetRole(key="image", role="aux")]) diff --git a/tests/test_typed_generic_ops.py b/tests/test_typed_generic_ops.py index d63aa50..b6870dd 100644 --- a/tests/test_typed_generic_ops.py +++ b/tests/test_typed_generic_ops.py @@ -1,6 +1,6 @@ """Typed-bag TWINS of the generic array→Image→Mask→Regions ops. -Pins the three native typed transforms that let a ``TypedSample`` pipeline run the +Pins the three native typed transforms that let a ``Sample`` pipeline run the detection/segmentation front-end without the legacy ``Sample`` path: * :class:`sampleflux.ops.image.ConvertToImage` — array-bearing field → ``Image`` item; @@ -15,10 +15,9 @@ import pytest from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Mask, Regions, TypedSample -from sampleflux.ops.image import ConvertToImage, ConvertToImageOp -from sampleflux.ops.numpy import ConnectedComponents, ConnectedComponentsOp, Threshold, ThresholdOp -from sampleflux.sample import Sample +from sampleflux import Image, Mask, Regions, Sample +from sampleflux.ops.image import ConvertToImage, _bound_longest_side, _render_rgb +from sampleflux.ops.numpy import ConnectedComponents, Threshold, connected_component_bboxes, threshold_array def _ramp_2d() -> np.ndarray: @@ -37,7 +36,7 @@ def _blob_mask() -> np.ndarray: # --------------------------------------------------------------------------- # class TestConvertToImage: def test_produces_image_item_shape_dtype_role(self) -> None: - out = ConvertToImage(colormap="gray")(TypedSample({"spec": Mask(_ramp_2d())})) + out = ConvertToImage(colormap="gray")(Sample({"spec": Mask(_ramp_2d())})) assert "image" in out img = out["image"] assert isinstance(img, Image) @@ -48,41 +47,35 @@ def test_produces_image_item_shape_dtype_role(self) -> None: # source field untouched assert isinstance(out["spec"], Mask) - def test_parity_with_legacy_default_sizing(self) -> None: + def test_parity_with_render_helper_default_sizing(self) -> None: arr = _ramp_2d() - typed = ConvertToImage(colormap="viridis")(TypedSample({"spec": Mask(arr)})) - legacy = ConvertToImageOp(colormap="viridis")(Sample(input=arr, target=None, metadata={})) - assert np.array_equal(np.array(legacy.input), np.asarray(typed["image"])) + typed = ConvertToImage(colormap="viridis")(Sample({"spec": Mask(arr)})) + expected = _bound_longest_side(_render_rgb(arr, "viridis"), 512) + assert np.array_equal(expected, np.asarray(typed["image"])) - def test_parity_with_legacy_exact_resize_and_flip(self) -> None: + def test_exact_resize_and_flip(self) -> None: arr = _ramp_2d() - typed = ConvertToImage(colormap="gray", width=20, height=16, flip_vertical=True)( - TypedSample({"spec": Mask(arr)}) - ) - legacy = ConvertToImageOp(colormap="gray", width=20, height=16, flip_vertical=True)( - Sample(input=arr, target=None, metadata={}) - ) + typed = ConvertToImage(colormap="gray", width=20, height=16, flip_vertical=True)(Sample({"spec": Mask(arr)})) assert np.asarray(typed["image"]).shape == (16, 20, 3) - assert np.array_equal(np.array(legacy.input), np.asarray(typed["image"])) def test_explicit_field_and_custom_output(self) -> None: - s = TypedSample({"a": Mask(_ramp_2d()), "b": Mask(np.zeros((4, 4), dtype=np.float32))}) + s = Sample({"a": Mask(_ramp_2d()), "b": Mask(np.zeros((4, 4), dtype=np.float32))}) out = ConvertToImage(field="b", output="preview")(s) assert np.asarray(out["preview"]).shape == (4, 4, 3) def test_does_not_publish_image_dims_metadata(self) -> None: # There is no shared metadata dict in the typed model; the Image SHAPE carries the dims. - out = ConvertToImage()(TypedSample({"spec": Mask(_ramp_2d())})) + out = ConvertToImage()(Sample({"spec": Mask(_ramp_2d())})) assert set(out.keys()) == {"spec", "image"} # no image_width_px / image_height_px field assert np.asarray(out["image"]).shape[:2] == (8, 10) def test_missing_explicit_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - ConvertToImage(field="nope")(TypedSample({"spec": Mask(_ramp_2d())})) + ConvertToImage(field="nope")(Sample({"spec": Mask(_ramp_2d())})) def test_no_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - ConvertToImage()(TypedSample({"lbl": Regions(boxes=[[0, 0, 1, 1]])})) + ConvertToImage()(Sample({"lbl": Regions(boxes=[[0, 0, 1, 1]])})) # --------------------------------------------------------------------------- # @@ -91,60 +84,58 @@ def test_no_array_field_raises(self) -> None: class TestThreshold: def test_produces_mask_parity_role(self) -> None: arr = _ramp_2d() - typed = Threshold(low_level=20.0)(TypedSample({"spec": Mask(arr)})) + typed = Threshold(low_level=20.0)(Sample({"spec": Mask(arr)})) assert isinstance(typed["mask"], Mask) assert np.asarray(typed["mask"]).dtype == np.bool_ assert typed.role_of("mask") == "aux" - legacy = ThresholdOp(low_level=20.0)(Sample(input=arr, target=None, metadata={})).input - assert np.array_equal(np.asarray(typed["mask"]), legacy) + expected = threshold_array(arr, low_level=20.0) + assert np.array_equal(np.asarray(typed["mask"]), expected) def test_string_literal_bound(self) -> None: arr = _ramp_2d() - typed = Threshold(low_level="20")(TypedSample({"spec": Mask(arr)})) - legacy = ThresholdOp(low_level="20")(Sample(input=arr, target=None, metadata={})).input - assert np.array_equal(np.asarray(typed["mask"]), legacy) + typed = Threshold(low_level="20")(Sample({"spec": Mask(arr)})) + expected = threshold_array(arr, low_level="20") + assert np.array_equal(np.asarray(typed["mask"]), expected) def test_env_var_expression_bound(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("TEST_THRESH_LEVEL", "20") arr = _ramp_2d() - typed = Threshold(low_level="$TEST_THRESH_LEVEL")(TypedSample({"spec": Mask(arr)})) + typed = Threshold(low_level="$TEST_THRESH_LEVEL")(Sample({"spec": Mask(arr)})) assert np.array_equal(np.asarray(typed["mask"]), arr > 20.0) def test_meta_key_expression_has_no_typed_source(self) -> None: # {key} expressions have no typed metadata home -> loud KeyError (documented). with pytest.raises(KeyError): - Threshold(low_level="{some_key}")(TypedSample({"spec": Mask(_ramp_2d())})) + Threshold(low_level="{some_key}")(Sample({"spec": Mask(_ramp_2d())})) def test_band_pass_both_bounds_and_ops(self) -> None: arr = _ramp_2d() - typed = Threshold(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")(TypedSample({"spec": Mask(arr)})) - legacy = ThresholdOp(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")( - Sample(input=arr, target=None, metadata={}) - ).input - assert np.array_equal(np.asarray(typed["mask"]), legacy) + typed = Threshold(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")(Sample({"spec": Mask(arr)})) + expected = threshold_array(arr, low_level=20.0, high_level=60.0, low_op=">=", high_op="<=") + assert np.array_equal(np.asarray(typed["mask"]), expected) assert np.array_equal(np.asarray(typed["mask"]), (arr >= 20.0) & (arr <= 60.0)) def test_no_bound_raises(self) -> None: with pytest.raises(ValueError, match="at least one"): - Threshold()(TypedSample({"spec": Mask(_ramp_2d())})) + Threshold()(Sample({"spec": Mask(_ramp_2d())})) def test_default_field_picks_first_array(self) -> None: # No explicit field: first array-bearing item (insertion order). - s = TypedSample({"raw": Mask(_ramp_2d()), "other": Regions(boxes=[])}) + s = Sample({"raw": Mask(_ramp_2d()), "other": Regions(boxes=[])}) out = Threshold(low_level=20.0)(s) assert np.array_equal(np.asarray(out["mask"]), _ramp_2d() > 20.0) def test_missing_explicit_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - Threshold(low_level=1.0, field="nope")(TypedSample({"spec": Mask(_ramp_2d())})) + Threshold(low_level=1.0, field="nope")(Sample({"spec": Mask(_ramp_2d())})) def test_non_array_field_raises(self) -> None: with pytest.raises(TypeError, match="expected an array"): - Threshold(low_level=1.0, field="reg")(TypedSample({"reg": Regions(boxes=[])})) + Threshold(low_level=1.0, field="reg")(Sample({"reg": Regions(boxes=[])})) def test_no_array_field_default_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - Threshold(low_level=1.0)(TypedSample({"reg": Regions(boxes=[])})) + Threshold(low_level=1.0)(Sample({"reg": Regions(boxes=[])})) # --------------------------------------------------------------------------- # @@ -152,7 +143,7 @@ def test_no_array_field_default_raises(self) -> None: # --------------------------------------------------------------------------- # class TestConnectedComponents: def test_produces_regions_bin_box_contract_and_role(self) -> None: - out = ConnectedComponents()(TypedSample({"m": Mask(_blob_mask())})) + out = ConnectedComponents()(Sample({"m": Mask(_blob_mask())})) regions = out["boxes"] assert isinstance(regions, Regions) assert out.role_of("boxes") == "aux" @@ -161,15 +152,15 @@ def test_produces_regions_bin_box_contract_and_role(self) -> None: def test_parity_with_legacy(self) -> None: mask = _blob_mask() - typed = ConnectedComponents()(TypedSample({"m": Mask(mask)})) - legacy = ConnectedComponentsOp()(Sample(input=mask, target=None, metadata={})).input - assert typed["boxes"].boxes == legacy + typed = ConnectedComponents()(Sample({"m": Mask(mask)})) + expected = connected_component_bboxes(mask) + assert typed["boxes"].boxes == expected def test_min_area_bins_filters_small_blobs(self) -> None: m = np.zeros((6, 6), dtype=bool) m[0:2, 0:2] = True # area 4 m[5, 5] = True # area 1 -> dropped when min_area_bins=2 - out = ConnectedComponents(min_area_bins=2)(TypedSample({"m": Mask(m)})) + out = ConnectedComponents(min_area_bins=2)(Sample({"m": Mask(m)})) assert out["boxes"].boxes == [(0, 1, 0, 1)] def test_connectivity_parity(self) -> None: @@ -177,33 +168,33 @@ def test_connectivity_parity(self) -> None: m = np.zeros((4, 4), dtype=bool) m[0, 0] = True m[1, 1] = True - four = ConnectedComponents(connectivity=4)(TypedSample({"m": Mask(m)})) - eight = ConnectedComponents(connectivity=8)(TypedSample({"m": Mask(m)})) + four = ConnectedComponents(connectivity=4)(Sample({"m": Mask(m)})) + eight = ConnectedComponents(connectivity=8)(Sample({"m": Mask(m)})) assert len(four["boxes"].boxes) == 2 assert len(eight["boxes"].boxes) == 1 def test_default_prefers_mask_over_other_array(self) -> None: # An Image is inserted first, but a Mask is preferred by the default resolver. - s = TypedSample({"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())}) + s = Sample({"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())}) out = ConnectedComponents()(s) assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] def test_falls_back_to_first_array_when_no_mask(self) -> None: # No Mask item — a 2-D array item is used. - out = ConnectedComponents()(TypedSample({"m": Image(_blob_mask())})) + out = ConnectedComponents()(Sample({"m": Image(_blob_mask())})) assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] def test_non_2d_mask_raises(self) -> None: with pytest.raises(ValueError, match="2-D mask"): - ConnectedComponents()(TypedSample({"m": Mask(np.zeros((2, 2, 2), dtype=bool))})) + ConnectedComponents()(Sample({"m": Mask(np.zeros((2, 2, 2), dtype=bool))})) def test_missing_explicit_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - ConnectedComponents(field="nope")(TypedSample({"m": Mask(_blob_mask())})) + ConnectedComponents(field="nope")(Sample({"m": Mask(_blob_mask())})) def test_no_mask_or_array_raises(self) -> None: with pytest.raises(ValueError, match="no Mask or array-bearing field"): - ConnectedComponents()(TypedSample({"reg": Regions(boxes=[])})) + ConnectedComponents()(Sample({"reg": Regions(boxes=[])})) # --------------------------------------------------------------------------- # @@ -211,7 +202,7 @@ def test_no_mask_or_array_raises(self) -> None: # --------------------------------------------------------------------------- # def test_array_to_image_to_mask_to_regions_chain() -> None: arr = _ramp_2d() - sample = TypedSample({"spec": Mask(arr)}) + sample = Sample({"spec": Mask(arr)}) out = ConnectedComponents(field="mask")(Threshold(field="spec", low_level=20.0)(ConvertToImage()(sample))) # Every stage produced its typed field. assert isinstance(out["image"], Image) diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py index 0186f45..eec08c9 100644 --- a/tests/test_typed_storage.py +++ b/tests/test_typed_storage.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from sampleflux import Image, Label, Regions, Sample, TypedSample, register_item +from sampleflux import Image, Label, Regions, Sample, register_item from sampleflux.storage.base import restore_attrs, split_attrs from sampleflux.storage.directory import DirectorySink, DirectorySource from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source @@ -26,7 +26,7 @@ class _StoreSig: def _samples() -> list: # Ragged across samples: different box counts, one field with an array-valued attr. - s0 = TypedSample( + s0 = Sample( { "image": Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW"), "sig": _StoreSig(np.arange(8, dtype=np.float32), samplerate=20e6, mask=np.array([1, 0, 1], dtype=np.uint8)), @@ -35,7 +35,7 @@ def _samples() -> list: }, roles={"regions": "target", "label": "target", "sig": "aux"}, ) - s1 = TypedSample( + s1 = Sample( { "image": Image(np.ones((2, 2, 3), dtype=np.float32)), "sig": _StoreSig(np.zeros(4, dtype=np.float32), samplerate=1e6, mask=np.array([0], dtype=np.uint8)), @@ -102,36 +102,13 @@ def test_round_trip(self, tmp_path: Path) -> None: assert source.is_typed and len(source) == 2 _assert_round_trip(list(source), _samples()) - def test_carrier_guards_both_directions(self, tmp_path: Path) -> None: - typed_path = tmp_path / "typed.h5" - sink = HDF5Sink(path=typed_path, overwrite=True) + def test_non_sample_write_raises(self, tmp_path: Path) -> None: + # The sink only accepts a typed Sample bag; a bare array is rejected loudly. + sink = HDF5Sink(path=tmp_path / "typed.h5", overwrite=True) with sink: sink.write(_samples()[0]) - appender = HDF5Sink(path=typed_path) - with appender: - with pytest.raises(TypeError, match="typed field-group layout"): - appender.write(Sample(input=np.zeros(3))) - - legacy_path = tmp_path / "legacy.h5" - legacy = HDF5Sink(path=legacy_path, overwrite=True) - with legacy: - legacy.write(Sample(input=np.zeros(3), metadata={"k": 1})) - appender2 = HDF5Sink(path=legacy_path) - with appender2: - with pytest.raises(TypeError, match="legacy Sample layout"): - appender2.write(_samples()[0]) - - def test_legacy_path_unchanged(self, tmp_path: Path) -> None: - path = tmp_path / "legacy.h5" - sink = HDF5Sink(path=path, overwrite=True) - with sink: - sink.write(Sample(input=np.arange(4, dtype=np.float32), target=1, metadata={"snr_db": 12.0})) - sink.flush() - source = HDF5Source(path=path) - with source: - assert not source.is_typed - (back,) = list(source) - assert isinstance(back, Sample) and back.meta["snr_db"] == 12.0 + with pytest.raises(TypeError, match="expected a Sample bag"): + sink.write(np.zeros(3)) class TestZarrTyped: @@ -145,13 +122,13 @@ def test_group_round_trip(self, tmp_path: Path) -> None: assert source.is_typed and len(source) == 2 _assert_round_trip(list(source), _samples()) - def test_group_carrier_guard(self, tmp_path: Path) -> None: + def test_group_non_sample_write_raises(self, tmp_path: Path) -> None: path = str(tmp_path / "g.zarr") sink = ZarrGroupSink(path=path) sink.open() sink.write(_samples()[0]) - with pytest.raises(TypeError, match="typed field-group layout"): - sink.write(Sample(input=np.zeros(3))) + with pytest.raises(TypeError, match="expected a Sample bag"): + sink.write(np.zeros(3)) def test_batch_typed_rows(self, tmp_path: Path) -> None: path = str(tmp_path / "b.zarr") @@ -161,7 +138,7 @@ def test_batch_typed_rows(self, tmp_path: Path) -> None: sink.write(s) # appends the PRIMARY input field's payload source = ZarrBatchSource(path=path) rows = list(source) - assert len(rows) == 2 and all(isinstance(r, TypedSample) for r in rows) + assert len(rows) == 2 and all(isinstance(r, Sample) for r in rows) assert isinstance(rows[0]["image"], Image) and rows[0]["image"].layout == "CHW" # uniform template assert np.asarray(rows[1]["image"]).shape == (2, 2, 3) @@ -216,10 +193,10 @@ def test_where_field_attr_expression(self, tmp_path: Path) -> None: fast = MetadataFilterSource(source=source, where="sig.samplerate > 1e7") assert len(fast) == 1 (match,) = list(fast) - assert isinstance(match, TypedSample) and match["sig"].samplerate == 20e6 + assert isinstance(match, Sample) and match["sig"].samplerate == 20e6 def test_full_iteration_fallback_on_typed_samples(self) -> None: - # A plain list source (no iter_metadata protocol) of TypedSamples still filters. + # A plain list source (no iter_metadata protocol) of Samples still filters. filt = MetadataFilterSource(source=_samples(), where="image.layout == 'CHW'") assert len(filt) == 1 diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py index 30006b7..8856e06 100644 --- a/tests/test_typed_target_ops.py +++ b/tests/test_typed_target_ops.py @@ -1,6 +1,6 @@ """Typed-bag TWINS of the tensorization + target-shaping ops. -Pins the native typed transforms that let a ``TypedSample`` classification pipeline build its +Pins the native typed transforms that let a ``Sample`` classification pipeline build its model INPUT tensor and its encoded TARGET ``Label`` without the legacy ``Sample`` path: * :class:`sampleflux.ops.torch.ToTensor` — array-bearing field → CHW-float ``Image`` item; @@ -13,15 +13,13 @@ import numpy as np import pytest -import torch from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Label, Mask, TypedSample +from sampleflux import Image, Label, Mask, Sample from sampleflux.collate import typed_collate from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.target import DecodeTarget, DecodeTargetOp, EncodeTarget, EncodeTargetOp, MetadataToTarget -from sampleflux.ops.torch import ToTensor, ToTensorOp -from sampleflux.sample import Sample +from sampleflux.ops.target import DecodeTarget, EncodeTarget, MetadataToTarget +from sampleflux.ops.torch import ToTensor, to_tensor _MAP = {"cat": 0, "dog": 1, "fox": 2} _INV = {0: "cat", 1: "dog", 2: "fox"} @@ -37,7 +35,7 @@ def _hwc_uint8() -> np.ndarray: class TestToTensor: def test_produces_chw_float_image_role_preserved(self) -> None: arr = _hwc_uint8() - out = ToTensor()(TypedSample({"image": Image(arr)}, roles={"image": "input"})) + out = ToTensor()(Sample({"image": Image(arr)}, roles={"image": "input"})) img = out["image"] assert isinstance(img, Image) assert img.layout == "CHW" @@ -47,57 +45,56 @@ def test_produces_chw_float_image_role_preserved(self) -> None: assert payload.max() <= 1.0 # normalized assert out.role_of("image") == "input" # replaced in place -> role preserved - def test_parity_with_legacy_tensor(self) -> None: + def test_parity_with_to_tensor_helper(self) -> None: arr = _hwc_uint8() - typed = ToTensor()(TypedSample({"image": Image(arr)})) - legacy = ToTensorOp()(Sample(input=arr, target=None, metadata={})).input - assert isinstance(legacy, torch.Tensor) - assert np.array_equal(np.asarray(typed["image"]), legacy.numpy()) + typed = ToTensor()(Sample({"image": Image(arr)})) + expected = to_tensor(arr).numpy() + assert np.array_equal(np.asarray(typed["image"]), expected) def test_parity_no_normalize(self) -> None: arr = _hwc_uint8() - typed = ToTensor(normalize=False)(TypedSample({"image": Image(arr)})) - legacy = ToTensorOp(normalize=False)(Sample(input=arr, target=None, metadata={})).input - assert np.array_equal(np.asarray(typed["image"]), legacy.numpy()) + typed = ToTensor(normalize=False)(Sample({"image": Image(arr)})) + expected = to_tensor(arr, normalize=False).numpy() + assert np.array_equal(np.asarray(typed["image"]), expected) def test_payload_is_numpy_not_live_tensor(self) -> None: # NDArrayItem coerces its payload via np.asarray, so an Image CANNOT hold a live tensor; # the stored CHW-float payload is a numpy array (values identical to the legacy tensor). from sampleflux.bag.items import item_data - out = ToTensor()(TypedSample({"image": Image(_hwc_uint8())})) + out = ToTensor()(Sample({"image": Image(_hwc_uint8())})) assert isinstance(item_data(out["image"]), np.ndarray) def test_new_output_field_tagged_input(self) -> None: arr = _hwc_uint8() - out = ToTensor(output="tensor")(TypedSample({"image": Image(arr)}, roles={"image": "input"})) + out = ToTensor(output="tensor")(Sample({"image": Image(arr)}, roles={"image": "input"})) assert np.asarray(out["tensor"]).shape == (3, 4, 5) assert out.role_of("tensor") == "input" # original field left as-is (HWC uint8) assert np.asarray(out["image"]).shape == (4, 5, 3) def test_explicit_field(self) -> None: - s = TypedSample({"a": Mask(np.zeros((2, 2), dtype=np.uint8)), "b": Image(_hwc_uint8())}) + s = Sample({"a": Mask(np.zeros((2, 2), dtype=np.uint8)), "b": Image(_hwc_uint8())}) out = ToTensor(field="b")(s) assert np.asarray(out["b"]).shape == (3, 4, 5) def test_default_picks_first_array_field(self) -> None: - s = TypedSample({"lbl": Label("cat"), "image": Image(_hwc_uint8())}) + s = Sample({"lbl": Label("cat"), "image": Image(_hwc_uint8())}) out = ToTensor()(s) assert np.asarray(out["image"]).shape == (3, 4, 5) def test_missing_explicit_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - ToTensor(field="nope")(TypedSample({"image": Image(_hwc_uint8())})) + ToTensor(field="nope")(Sample({"image": Image(_hwc_uint8())})) def test_no_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - ToTensor()(TypedSample({"lbl": Label("cat")})) + ToTensor()(Sample({"lbl": Label("cat")})) def test_typed_collate_stacks_payloads(self) -> None: # The typed collate stacks the CHW-float Image payloads into a batched array. - a = ToTensor()(TypedSample({"image": Image(_hwc_uint8())})) - b = ToTensor()(TypedSample({"image": Image(_hwc_uint8())})) + a = ToTensor()(Sample({"image": Image(_hwc_uint8())})) + b = ToTensor()(Sample({"image": Image(_hwc_uint8())})) batch = typed_collate([a, b]) assert np.asarray(batch["image"]).shape == (2, 3, 4, 5) @@ -107,35 +104,35 @@ def test_typed_collate_stacks_payloads(self) -> None: # --------------------------------------------------------------------------- # class TestMetadataToTarget: def test_promotes_label_value_to_target(self) -> None: - s = TypedSample({"class": Label("cat")}, roles={"class": "aux"}) + s = Sample({"class": Label("cat")}, roles={"class": "aux"}) out = MetadataToTarget(field="class", output="target")(s) assert isinstance(out["target"], Label) assert out["target"].value == "cat" assert out.role_of("target") == "target" def test_default_picks_first_label(self) -> None: - s = TypedSample({"image": Image(_hwc_uint8()), "y": Label("dog")}) + s = Sample({"image": Image(_hwc_uint8()), "y": Label("dog")}) out = MetadataToTarget()(s) assert out["target"].value == "dog" assert out.role_of("target") == "target" def test_read_named_attribute(self) -> None: # Read a carried attribute off a field (a value that rode as item-scoped metadata). - s = TypedSample({"y": Label("cat", classes=["cat", "dog"])}) + s = Sample({"y": Label("cat", classes=["cat", "dog"])}) out = MetadataToTarget(field="y", key="classes", output="vocab")(s) assert out["vocab"].value == ["cat", "dog"] def test_missing_attribute_raises(self) -> None: with pytest.raises(AttributeError, match="no attribute 'nope'"): - MetadataToTarget(field="y", key="nope")(TypedSample({"y": Label("cat")})) + MetadataToTarget(field="y", key="nope")(Sample({"y": Label("cat")})) def test_missing_field_raises(self) -> None: with pytest.raises(ValueError, match="field 'nope' not in sample"): - MetadataToTarget(field="nope")(TypedSample({"y": Label("cat")})) + MetadataToTarget(field="nope")(Sample({"y": Label("cat")})) def test_empty_sample_raises(self) -> None: with pytest.raises(ValueError, match="sample is empty"): - MetadataToTarget()(TypedSample({})) + MetadataToTarget()(Sample({})) # --------------------------------------------------------------------------- # @@ -143,49 +140,47 @@ def test_empty_sample_raises(self) -> None: # --------------------------------------------------------------------------- # class TestEncodeDecodeTarget: def test_encode_name_to_id_role_target(self) -> None: - out = EncodeTarget(mapping=_MAP)(TypedSample({"y": Label("cat")}, roles={"y": "target"})) + out = EncodeTarget(mapping=_MAP)(Sample({"y": Label("cat")}, roles={"y": "target"})) assert isinstance(out["y"], Label) assert out["y"].value == 0 assert out.role_of("y") == "target" - def test_encode_parity_with_legacy(self) -> None: + def test_encode_maps_every_name(self) -> None: for name in _MAP: - typed = EncodeTarget(mapping=_MAP)(TypedSample({"y": Label(name)})) - legacy = EncodeTargetOp(mapping=_MAP)(Sample(input=None, target=name, metadata={})).target - assert typed["y"].value == legacy + typed = EncodeTarget(mapping=_MAP)(Sample({"y": Label(name)})) + assert typed["y"].value == _MAP[name] def test_encode_preserves_classes_vocab(self) -> None: - out = EncodeTarget(mapping=_MAP)(TypedSample({"y": Label("dog", classes=list(_MAP))})) + out = EncodeTarget(mapping=_MAP)(Sample({"y": Label("dog", classes=list(_MAP))})) assert out["y"].value == 1 assert out["y"].classes == list(_MAP) def test_encode_new_output_field(self) -> None: - out = EncodeTarget(mapping=_MAP, output="target_id")(TypedSample({"y": Label("fox")})) + out = EncodeTarget(mapping=_MAP, output="target_id")(Sample({"y": Label("fox")})) assert out["target_id"].value == 2 assert out.role_of("target_id") == "target" assert out["y"].value == "fox" # source left intact def test_encode_ignore_unknown(self) -> None: - out = EncodeTarget(mapping=_MAP, ignore_unknown=True, default=-1)(TypedSample({"y": Label("bird")})) + out = EncodeTarget(mapping=_MAP, ignore_unknown=True, default=-1)(Sample({"y": Label("bird")})) assert out["y"].value == -1 def test_encode_unknown_raises(self) -> None: with pytest.raises(KeyError): - EncodeTarget(mapping=_MAP)(TypedSample({"y": Label("bird")})) + EncodeTarget(mapping=_MAP)(Sample({"y": Label("bird")})) def test_encode_empty_mapping_raises_lazily(self) -> None: op = EncodeTarget() # constructible with no mapping (lazy) with pytest.raises(ValueError, match="at least one entry"): - op(TypedSample({"y": Label("cat")})) + op(Sample({"y": Label("cat")})) - def test_decode_id_to_name_parity(self) -> None: + def test_decode_id_to_name(self) -> None: for cid in _INV: - typed = DecodeTarget(mapping=_INV)(TypedSample({"y": Label(cid)})) - legacy = DecodeTargetOp(mapping=_INV)(Sample(input=None, target=cid, metadata={})).target - assert typed["y"].value == legacy + typed = DecodeTarget(mapping=_INV)(Sample({"y": Label(cid)})) + assert typed["y"].value == _INV[cid] def test_encode_then_decode_round_trip(self) -> None: - s = TypedSample({"y": Label("dog")}) + s = Sample({"y": Label("dog")}) encoded = EncodeTarget(mapping=_MAP)(s) assert encoded["y"].value == 1 decoded = DecodeTarget(mapping=_INV)(encoded) @@ -193,15 +188,15 @@ def test_encode_then_decode_round_trip(self) -> None: def test_decode_empty_mapping_raises_lazily(self) -> None: with pytest.raises(ValueError, match="at least one entry"): - DecodeTarget()(TypedSample({"y": Label(0)})) + DecodeTarget()(Sample({"y": Label(0)})) def test_encode_non_label_field_raises(self) -> None: with pytest.raises(TypeError, match="expected a Label"): - EncodeTarget(mapping=_MAP, field="image")(TypedSample({"image": Image(_hwc_uint8())})) + EncodeTarget(mapping=_MAP, field="image")(Sample({"image": Image(_hwc_uint8())})) def test_encode_no_label_field_raises(self) -> None: with pytest.raises(ValueError, match="no Label field"): - EncodeTarget(mapping=_MAP)(TypedSample({"image": Image(_hwc_uint8())})) + EncodeTarget(mapping=_MAP)(Sample({"image": Image(_hwc_uint8())})) # --------------------------------------------------------------------------- # @@ -209,7 +204,7 @@ def test_encode_no_label_field_raises(self) -> None: # --------------------------------------------------------------------------- # def test_typed_classification_input_and_target_chain() -> None: # Source-shaped bag: an HWC image (role input) + a class-NAME label (role target). - sample = TypedSample( + sample = Sample( {"image": Image(_hwc_uint8()), "class": Label("cat", classes=list(_MAP))}, roles={"image": "input", "class": "target"}, ) @@ -234,7 +229,7 @@ def test_typed_classification_input_and_target_chain() -> None: def test_convert_then_tensor_chain() -> None: # A raw 2-D array field runs ConvertToImage -> ToTensor into a CHW-float input. arr = np.arange(6 * 4).reshape(6, 4).astype(np.float32) - out = ToTensor(field="image")(ConvertToImage(colormap="gray")(TypedSample({"spec": Mask(arr)}))) + out = ToTensor(field="image")(ConvertToImage(colormap="gray")(Sample({"spec": Mask(arr)}))) assert out["image"].layout == "CHW" assert np.asarray(out["image"]).shape == (3, 6, 4) diff --git a/tests/test_typespec.py b/tests/test_typespec.py deleted file mode 100644 index 85114fd..0000000 --- a/tests/test_typespec.py +++ /dev/null @@ -1,658 +0,0 @@ -"""Exhaustive tests for the sampleflux type-spec system (matching, inference, JSON, HF bridge).""" - -from typing import Any, Iterable, List, Tuple, cast, get_args - -import numpy as np -import pytest - -from sampleflux.core import Flux -from sampleflux.sample import FEATURES_KEY, SPEC_KEY, Sample -from sampleflux.typespec import ( - _DTYPE_FAMILIES, - AnyType, - ArrayType, - Dim, - Dtype, - DtypeFamily, - Framework, - ImageLayout, - ListType, - MappingType, - PythonType, - SampleType, - TypeSpec, - UnionType, - accepts, - canonical_dtype, - compatible, - infer_sample_type, - infer_type, - type_from_dict, - typed, -) - - -class _Widget: - """Module-level class so its ``__qualname__`` is the bare name (used by infer_type fallback test).""" - - -# -------------------------------------------------------------------------------------------------- -# Closed Literals (Framework / ImageLayout) a UI / connection-validator can enumerate -# -------------------------------------------------------------------------------------------------- - - -def test_framework_literal_enumerates_supported_frameworks() -> None: - # The point of the Literal over a bare str: choices are readable from the annotation. - assert get_args(Framework) == ("numpy", "torch", "tensorflow") - - -def test_image_layout_literal_enumerates_layouts() -> None: - assert get_args(ImageLayout) == ("CHW", "HWC") - - -def test_dtype_family_literal_matches_family_map() -> None: - # Single source of truth: the DtypeFamily Literal can't drift from the runtime family map. - assert set(get_args(DtypeFamily)) == set(_DTYPE_FAMILIES) - - -def test_dtype_literal_is_exactly_the_family_members() -> None: - # The concrete Dtype Literal is exactly the union of every family's members (no drift). - assert set(get_args(Dtype)) == set().union(*_DTYPE_FAMILIES.values()) - - -# -------------------------------------------------------------------------------------------------- -# Dim -# -------------------------------------------------------------------------------------------------- - - -def test_dim_accepts_strict() -> None: - assert Dim.range(1, 10).accepts(Dim.exact(5)) - assert Dim.range(1, 10).accepts(Dim.range(2, 9)) - assert not Dim.range(1, 10).accepts(Dim.exact(50)) - assert not Dim.range(1, 10).accepts(Dim.exact(0)) - assert Dim.any().accepts(Dim.exact(999)) - assert Dim.any().accepts(Dim.any()) - # unbounded producer against bounded consumer -> strict reject - assert not Dim.range(1, 10).accepts(Dim.any()) - # half-open consumer bounds - assert Dim.range(2, None).accepts(Dim.exact(100)) - assert not Dim.range(2, None).accepts(Dim.exact(1)) - - -def test_dim_compatible_permissive() -> None: - # unbounded producer against bounded consumer -> soft pass - assert Dim.range(1, 10).compatible(Dim.any()) - assert Dim.range(1, 10).compatible(Dim.exact(5)) - # provably disjoint -> reject even when permissive - assert not Dim.range(1, 10).compatible(Dim.exact(50)) - assert not Dim.range(20, 30).compatible(Dim.range(1, 5)) - - -def test_dim_json_roundtrip() -> None: - d = Dim.range(1, 10, "axis") - assert Dim.from_dict(d.to_dict()) == d - - -# -------------------------------------------------------------------------------------------------- -# AnyType -# -------------------------------------------------------------------------------------------------- - - -def test_any_type_matches_everything() -> None: - for producer in (PythonType("dict"), ArrayType(ndim=3), UnionType((AnyType(),))): - assert accepts(AnyType(), producer) - # concrete consumer rejects an Any producer (strict) but accepts it permissively - assert not accepts(ArrayType(ndim=2), AnyType()) - assert compatible(ArrayType(ndim=2), AnyType()) - - -# -------------------------------------------------------------------------------------------------- -# ArrayType -# -------------------------------------------------------------------------------------------------- - - -def test_array_ndim() -> None: - assert accepts(ArrayType(ndim=2), ArrayType(ndim=2)) - assert not accepts(ArrayType(ndim=2), ArrayType(ndim=3)) - assert accepts(ArrayType(ndim=None), ArrayType(ndim=5)) # rank-agnostic consumer - # unknown producer rank: strict reject, permissive pass - assert not accepts(ArrayType(ndim=2), ArrayType(ndim=None)) - assert compatible(ArrayType(ndim=2), ArrayType(ndim=None)) - - -def test_array_shape() -> None: - consumer = ArrayType(shape=(Dim.range(1, 10), Dim.any())) - assert accepts(consumer, ArrayType(shape=(Dim.exact(5), Dim.exact(7)))) - assert not accepts(consumer, ArrayType(shape=(Dim.exact(50), Dim.exact(7)))) - # producer without shape: strict reject, permissive pass - assert not accepts(consumer, ArrayType(ndim=2)) - assert compatible(consumer, ArrayType(ndim=2)) - - -def test_array_dtype_families() -> None: - assert accepts(ArrayType(dtype="floating"), ArrayType(dtype="float32")) - assert accepts(ArrayType(dtype="numeric"), ArrayType(dtype="int64")) - assert not accepts(ArrayType(dtype="floating"), ArrayType(dtype="int64")) - assert accepts(ArrayType(dtype="float32"), ArrayType(dtype="float32")) - assert not accepts(ArrayType(dtype="float32"), ArrayType(dtype="float64")) - # family-accepts-subfamily: numeric accepts the whole floating family - assert accepts(ArrayType(dtype="numeric"), ArrayType(dtype="floating")) - assert not accepts(ArrayType(dtype="floating"), ArrayType(dtype="numeric")) - # producer dtype unknown - assert not accepts(ArrayType(dtype="float32"), ArrayType()) - assert compatible(ArrayType(dtype="float32"), ArrayType()) - - -def test_array_frameworks() -> None: - assert accepts(ArrayType(frameworks={"numpy", "torch"}), ArrayType(frameworks={"torch"})) - assert not accepts(ArrayType(frameworks={"torch"}), ArrayType(frameworks={"numpy"})) - # permissive: overlap is enough - assert compatible(ArrayType(frameworks={"torch"}), ArrayType(frameworks={"torch", "numpy"})) - assert not compatible(ArrayType(frameworks={"torch"}), ArrayType(frameworks={"numpy"})) - # producer framework unknown - assert not accepts(ArrayType(frameworks={"torch"}), ArrayType()) - assert compatible(ArrayType(frameworks={"torch"}), ArrayType()) - - -def test_array_post_init() -> None: - a = ArrayType(shape=(Dim.exact(3), Dim.any())) - assert a.ndim == 2 # derived from shape - assert isinstance(ArrayType(frameworks={"numpy"}).frameworks, frozenset) # set coerced to frozenset - # Off-Literal alias/casing is a *runtime-only* convenience normalized by canonical_dtype; mypy - # rightly flags the non-canonical literal at the call site — that's the point of the Dtype Literal. - assert ArrayType(dtype="FLOAT32").dtype == "float32" # type: ignore[arg-type] # normalized - with pytest.raises(ValueError): - ArrayType(ndim=3, shape=(Dim.any(),)) - - -def test_array_image_constructor() -> None: - chw = ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") - assert chw.ndim == 3 and chw.semantic == "image" - assert chw.shape is not None and chw.shape[0] == Dim.exact(3, "C") - hwc = ArrayType.image("HWC", channels=(1, 4)) - assert hwc.shape is not None and hwc.shape[2] == Dim(1, 4, "C") - with pytest.raises(ValueError): - # Deliberately off-type: the ImageLayout Literal is a static hint, the - # runtime still guards. mypy rightly objects — that's the point. - ArrayType.image("XYZ") # type: ignore[arg-type] - - -def test_array_parse() -> None: - a = ArrayType.parse("3 h w", dtype="float32", framework="torch") - assert a.shape == (Dim.exact(3), Dim.any("h"), Dim.any("w")) - assert a.dtype == "float32" and a.frameworks == frozenset({"torch"}) - ranged = ArrayType.parse("1-10 N") - assert ranged.shape == (Dim.range(1, 10), Dim.any("N")) - assert ArrayType.parse("2-").shape == (Dim.range(2, None),) - with pytest.raises(ValueError): - ArrayType.parse("*batch c h w") - - -# -------------------------------------------------------------------------------------------------- -# PythonType / Union / Mapping / List -# -------------------------------------------------------------------------------------------------- - - -def test_python_type_exact() -> None: - assert accepts(PythonType("dict"), PythonType("dict")) - assert not accepts(PythonType("dict"), PythonType("list")) - # class mismatch - assert not accepts(PythonType("dict"), ArrayType(ndim=1)) - assert not accepts(ArrayType(ndim=1), PythonType("dict")) - - -def test_union_variance() -> None: - consumer = UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image"))) - assert accepts(consumer, ArrayType(dtype="int64")) - assert accepts(consumer, PythonType("PIL.Image.Image")) - assert not accepts(consumer, PythonType("str")) - # producer union: every branch must be accepted - prod = UnionType((ArrayType(dtype="int64"), ArrayType(dtype="float32"))) - assert accepts(ArrayType(dtype="numeric"), prod) - assert not accepts(ArrayType(dtype="floating"), prod) # int64 branch fails - - -def test_mapping_and_list() -> None: - consumer = MappingType.of({"a": ArrayType(ndim=1), "b": PythonType("str")}) - assert accepts(consumer, MappingType.of({"a": ArrayType(ndim=1), "b": PythonType("str"), "c": AnyType()})) - assert not accepts(consumer, MappingType.of({"a": ArrayType(ndim=1)})) # missing 'b' - assert not accepts(consumer, MappingType.of({"a": ArrayType(ndim=2), "b": PythonType("str")})) - assert accepts(ListType(ArrayType(dtype="floating")), ListType(ArrayType(dtype="float32"))) - assert not accepts(ListType(ArrayType(dtype="floating")), ListType(ArrayType(dtype="int64"))) - assert not accepts(MappingType.of({"a": AnyType()}), ListType(AnyType())) # class mismatch - - -# -------------------------------------------------------------------------------------------------- -# SampleType -# -------------------------------------------------------------------------------------------------- - - -def test_sample_type_pair() -> None: - consumer = SampleType(input=ArrayType(ndim=2), target=ArrayType(dtype="int64")) - assert consumer.accepts(SampleType(input=ArrayType(ndim=2), target=ArrayType(dtype="int64"))) - assert not consumer.accepts(SampleType(input=ArrayType(ndim=3), target=ArrayType(dtype="int64"))) - # default target is Any -> input-only constraints accept any target - input_only = SampleType(input=ArrayType(ndim=2)) - assert input_only.accepts(SampleType(input=ArrayType(ndim=2), target=PythonType("dict"))) - assert SampleType() == SampleType() # both AnyType defaults compare equal - - -# -------------------------------------------------------------------------------------------------- -# dtype canonicalization -# -------------------------------------------------------------------------------------------------- - - -def test_canonical_dtype() -> None: - assert canonical_dtype("Float32") == "float32" - assert canonical_dtype("double") == "float64" - assert canonical_dtype("floating") == "floating" # family passes through - assert canonical_dtype(np.dtype("int64")) == "int64" - assert canonical_dtype(np.float32) == "float32" - import torch - - assert canonical_dtype(torch.float32) == "float32" - assert canonical_dtype(torch.int64) == "int64" - - -# -------------------------------------------------------------------------------------------------- -# inference -# -------------------------------------------------------------------------------------------------- - - -def test_infer_type_numpy() -> None: - spec = infer_type(np.zeros((3, 8, 8), dtype=np.float32)) - assert isinstance(spec, ArrayType) - assert spec.ndim == 3 and spec.dtype == "float32" and spec.frameworks == frozenset({"numpy"}) - assert spec.shape == (Dim.exact(3), Dim.exact(8), Dim.exact(8)) - scalar = infer_type(np.int64(7)) - assert isinstance(scalar, ArrayType) and scalar.ndim == 0 and scalar.dtype == "int64" - - -def test_infer_type_torch() -> None: - import torch - - spec = infer_type(torch.zeros(2, 4, dtype=torch.float64)) - assert isinstance(spec, ArrayType) and spec.frameworks == frozenset({"torch"}) - assert spec.ndim == 2 and spec.dtype == "float64" - - -def test_infer_type_pil() -> None: - Image = pytest.importorskip("PIL.Image") - assert infer_type(Image.new("RGB", (4, 4))) == PythonType("PIL.Image.Image") - - -def test_infer_type_python_values() -> None: - assert infer_type(None) == AnyType() - assert infer_type(True) == PythonType("bool") - assert infer_type(3) == PythonType("int") - assert infer_type(2.5) == PythonType("float") - assert infer_type("x") == PythonType("str") - assert infer_type({"a": 1}) == PythonType("dict") - assert infer_type([1, 2]) == PythonType("list") - assert infer_type((1, 2)) == PythonType("tuple") - assert infer_type(_Widget()) == PythonType("_Widget") - - -def test_infer_sample_type() -> None: - st = infer_sample_type(Sample(input=np.zeros((2, 2), dtype=np.float32), target=np.int64(1))) - assert isinstance(st.input, ArrayType) and st.input.ndim == 2 - assert isinstance(st.target, ArrayType) and st.target.ndim == 0 - # unset target -> Any - assert infer_sample_type(Sample(input=np.zeros(2))).target == AnyType() - - -# -------------------------------------------------------------------------------------------------- -# JSON round-trip -# -------------------------------------------------------------------------------------------------- - - -def test_json_roundtrip_all_kinds() -> None: - specs: List[TypeSpec] = [ - AnyType(), - ArrayType.image("CHW", 3, "float32", "torch"), - ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="numeric", frameworks={"numpy"}), - PythonType("PIL.Image.Image"), - UnionType((ArrayType(ndim=1), PythonType("str"), AnyType())), - MappingType.of({"boxes": ArrayType(ndim=2), "labels": ListType(ArrayType(ndim=0, dtype="int64"))}), - ] - for spec in specs: - assert type_from_dict(spec.to_dict()) == spec - st = SampleType(input=specs[1], target=specs[5]) - assert SampleType.from_dict(st.to_dict()) == st - - -def test_type_from_dict_unknown_kind() -> None: - with pytest.raises(ValueError): - type_from_dict({"kind": "bogus"}) - - -# -------------------------------------------------------------------------------------------------- -# datasets.Features bridge -# -------------------------------------------------------------------------------------------------- - - -def test_hf_features_classification_target() -> None: - spec = SampleType( - input=PythonType("PIL.Image.Image"), - target=ArrayType(ndim=0, dtype="int64", frameworks={"torch"}), - ) - features, extras = spec.to_hf_features() - assert set(features.keys()) == {"input", "target"} - rt = SampleType.from_hf_features(features, extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_detection_target() -> None: - spec = SampleType( - target=MappingType.of( - { - "boxes": ArrayType(ndim=2, shape=(Dim.any("N"), Dim.exact(4)), dtype="float32", frameworks={"torch"}), - "labels": ArrayType(ndim=1, shape=(Dim.any("N"),), dtype="int64", frameworks={"torch"}), - } - ) - ) - features, extras = spec.to_hf_features() - rt = SampleType.from_hf_features(features, extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_segmentation_target() -> None: - spec = SampleType( - input=ArrayType.image("CHW", 3, "float32", "torch"), - target=ArrayType(ndim=2, shape=(Dim.any("H"), Dim.any("W")), dtype="int64", frameworks={"torch"}), - ) - features, extras = spec.to_hf_features() - rt = SampleType.from_hf_features(features, extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_non_expressible_falls_back_to_typespec() -> None: - # Any / Union / family-dtype / unknown-rank can't be a concrete Feature -> stored as a typespec blob. - spec = SampleType( - input=UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image"))), - target=ArrayType(dtype="floating"), # family dtype, no rank - ) - features, extras = spec.to_hf_features() - assert "input" not in features and "target" not in features - assert "typespec" in extras["input"] and "typespec" in extras["target"] - rt = SampleType.from_hf_features(features, extras) - assert rt == spec - - -def test_hf_features_from_dict_form() -> None: - spec = SampleType(input=ArrayType(ndim=2, shape=(Dim.any(), Dim.exact(4)), dtype="float32")) - features, extras = spec.to_hf_features() - # describe() passes the Features.to_dict() form (a plain dict), not a Features instance - rt = SampleType.from_hf_features(features.to_dict(), extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_pythontype_and_string_roundtrip() -> None: - # PIL <-> Image, str <-> Value("string"), and a non-Feature python type falls back to a typespec blob. - spec = SampleType(input=PythonType("str"), target=PythonType("dict")) - features, extras = spec.to_hf_features() - assert "input" in features and "target" not in features # str -> Value, dict -> typespec fallback - assert SampleType.from_hf_features(features, extras) == spec - - -def test_hf_features_high_rank_and_nested_fallback() -> None: - # ndim>5 has no ArrayXD feature, and a mapping/list with a non-expressible member falls back whole. - high_rank = SampleType(input=ArrayType(ndim=6, dtype="float32")) - f, e = high_rank.to_hf_features() - assert "input" not in f and SampleType.from_hf_features(f, e) == high_rank - - nested = SampleType(target=MappingType.of({"a": ArrayType(ndim=2, dtype="float32"), "b": AnyType()})) - f, e = nested.to_hf_features() - assert "target" not in f and SampleType.from_hf_features(f, e) == nested - - listed = SampleType(target=ListType(AnyType())) - f, e = listed.to_hf_features() - assert SampleType.from_hf_features(f, e) == listed - - -def test_canonical_dtype_fallback() -> None: - # an object that is neither a str nor a known dtype object falls through to str(x).lower() - assert canonical_dtype(123) == "123" - - -# -------------------------------------------------------------------------------------------------- -# @typed decorator -# -------------------------------------------------------------------------------------------------- - - -def test_typed_decorator() -> None: - acc = SampleType(input=PythonType("PIL.Image.Image")) - prod = SampleType(input=ArrayType.image("CHW", 3, "float32", "torch")) - - @typed(accepts=acc, produces=prod) - class Op: - pass - - assert getattr(Op, "ACCEPTS") == acc and getattr(Op, "PRODUCES") == prod - - with pytest.raises(TypeError): - - @typed(accepts=ArrayType(ndim=1)) # type: ignore[arg-type] - class Bad: - pass - - with pytest.raises(TypeError): - - @typed(produces="nope") # type: ignore[arg-type] - class Bad2: - pass - - -# -------------------------------------------------------------------------------------------------- -# Sample.describe / with_type -# -------------------------------------------------------------------------------------------------- - - -def test_sample_describe_infers_when_unstored() -> None: - s = Sample(input=np.zeros((3, 8, 8), dtype=np.float32)) - st = s.describe() - assert isinstance(st.input, ArrayType) and st.input.ndim == 3 and st.input.frameworks == frozenset({"numpy"}) - - -def test_sample_with_type_and_describe_stored() -> None: - declared = SampleType(input=ArrayType.image("CHW", 3, "float32", "torch")) - s = Sample(input=np.zeros((3, 8, 8), dtype=np.float32), metadata={"id": 7}) - typed_s = s.with_type(declared) - assert FEATURES_KEY in typed_s.meta and SPEC_KEY in typed_s.meta - assert typed_s.meta["id"] == 7 # pre-existing metadata preserved - assert s.meta == {"id": 7} and FEATURES_KEY not in s.meta # copy-on-write: original untouched - rt = typed_s.describe() - assert rt.accepts(declared) and declared.accepts(rt) - - -# -------------------------------------------------------------------------------------------------- -# Pipeline freshness (maintain-if-present) -# -------------------------------------------------------------------------------------------------- - - -class _ToFloat64Op: - PRODUCES = SampleType(input=ArrayType(ndim=1, dtype="float64", frameworks={"numpy"})) - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input.astype("float64")) - - -class _UntypedOp: - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + 1) - - -def _run(sample: Sample, op: Any) -> Sample: - return cast(Sample, list(Flux(source=[sample], ops=[op]))[0]) - - -def test_pipeline_does_not_stamp_untracked_samples() -> None: - out = _run(Sample(input=np.array([1, 2, 3])), _ToFloat64Op()) - assert FEATURES_KEY not in out.meta and SPEC_KEY not in out.meta - - -def test_pipeline_refreshes_stored_type_from_produces() -> None: - stamped = Sample(input=np.array([1, 2, 3])).with_type( - SampleType(input=ArrayType(ndim=1, dtype="int64", frameworks={"numpy"})) - ) - out = _run(stamped, _ToFloat64Op()) - assert FEATURES_KEY in out.meta - assert out.describe().accepts(_ToFloat64Op.PRODUCES) - assert _ToFloat64Op.PRODUCES.accepts(out.describe()) - - -def test_pipeline_drops_stored_type_when_op_has_no_produces() -> None: - stamped = Sample(input=np.array([1, 2, 3])).with_type( - SampleType(input=ArrayType(ndim=1, dtype="int64", frameworks={"numpy"})) - ) - out = _run(stamped, _UntypedOp()) - assert FEATURES_KEY not in out.meta and SPEC_KEY not in out.meta - # describe() falls back to inference -> still correct, just not "stored" - assert isinstance(out.describe().input, ArrayType) - - -# -------------------------------------------------------------------------------------------------- -# Op annotation conformance: declared PRODUCES must accept the real inferred output, and declared -# ACCEPTS must accept the real input (proves the static specs match runtime reality). -# -------------------------------------------------------------------------------------------------- - - -def test_sampleflux_op_spec_conformance() -> None: - import sampleflux.ops.numpy as N - import sampleflux.ops.torch as T - - rgb = (np.random.rand(3, 8, 8) * 255).astype(np.float32) - cases: List[Tuple[Any, Sample]] = [ - (N.StandardizeOp(mean=0.5, std=0.5), Sample(input=rgb.copy())), - (N.RescaleOp(in_min=0, in_max=255), Sample(input=rgb.copy())), - (N.ClipPercentilesOp(), Sample(input=rgb.copy())), - (N.ReplaceNonFiniteOp(), Sample(input=rgb.copy())), - (N.ThresholdOp(low_level=0.5), Sample(input=rgb.copy())), - (N.ConnectedComponentsOp(), Sample(input=(np.random.rand(8, 8) > 0.5))), - (T.RescaleOp(in_min=0, in_max=255), Sample(input=__import__("torch").rand(3, 8, 8) * 255)), - (T.StandardizeOp(mean=0.5, std=0.5), Sample(input=__import__("torch").rand(3, 8, 8))), - ] - for op, sample in cases: - name = type(op).__module__ + "." + type(op).__name__ - assert op.ACCEPTS.accepts(infer_sample_type(sample)), f"{name}: ACCEPTS rejects its real input" - out = op(sample) - assert op.PRODUCES.accepts(infer_sample_type(out)), f"{name}: PRODUCES rejects its real output" - # specs are JSON round-trippable - assert SampleType.from_dict(op.PRODUCES.to_dict()) == op.PRODUCES - - -# -------------------------------------------------------------------------------------------------- -# Dim / ArrayType __str__ and SampleType.explain_mismatch -# -------------------------------------------------------------------------------------------------- - - -class TestDimStr: - def test_unbounded(self) -> None: - assert str(Dim.any()) == "any" - - def test_exact(self) -> None: - assert str(Dim.exact(3)) == "3" - - def test_range(self) -> None: - assert str(Dim(min=2, max=8)) == "2–8" - - def test_open_upper(self) -> None: - assert str(Dim(min=1, max=None)) == "1–∞" - - def test_open_lower(self) -> None: - assert str(Dim(min=None, max=4)) == "0–4" - - -class TestArrayTypeStr: - def test_framework_only(self) -> None: - assert str(ArrayType(frameworks={"torch"})) == "array[torch]" - - def test_framework_and_dtype(self) -> None: - s = str(ArrayType(frameworks={"numpy"}, dtype="float32")) - assert "numpy" in s and "float32" in s - - def test_shape_shown(self) -> None: - s = str(ArrayType(shape=(Dim.exact(3), Dim.any()))) - assert "shape=(3, any)" in s - - def test_rank_without_shape(self) -> None: - s = str(ArrayType(ndim=2)) - assert "rank-2" in s - - def test_empty_is_just_array(self) -> None: - assert str(ArrayType()) == "array" - - -class TestExplainMismatch: - """SampleType.explain_mismatch produces actionable human-readable reasons.""" - - def _make( - self, - frameworks: Iterable[Framework] | None = None, - dtype: Dtype | None = None, - ndim: int | None = None, - ) -> SampleType: - return SampleType( - input=ArrayType( - frameworks=frozenset(cast(Iterable[Framework], frameworks)) if frameworks is not None else None, - dtype=cast(Dtype | None, dtype), - ndim=ndim, - ) - ) - - def test_framework_mismatch_names_both_sides(self) -> None: - consumer = self._make(frameworks={"numpy"}) - producer = self._make(frameworks={"torch"}) - msg = consumer.explain_mismatch(producer) - assert "numpy" in msg and "torch" in msg - assert "framework" in msg - - def test_dtype_mismatch_names_both_dtypes(self) -> None: - consumer = self._make(dtype="float32") - producer = self._make(dtype="complex64") - msg = consumer.explain_mismatch(producer) - assert "float32" in msg and "complex64" in msg - assert "dtype" in msg - - def test_ndim_mismatch_names_both_ranks(self) -> None: - consumer = self._make(ndim=2) - producer = self._make(ndim=3) - msg = consumer.explain_mismatch(producer) - assert "2" in msg and "3" in msg - assert "rank" in msg - - def test_shape_axis_mismatch_names_axis_and_sizes(self) -> None: - consumer = SampleType(input=ArrayType(shape=(Dim.exact(1), Dim.any()))) - producer = SampleType(input=ArrayType(shape=(Dim.exact(2), Dim.any()))) - msg = consumer.explain_mismatch(producer) - assert "axis 0" in msg - assert "1" in msg and "2" in msg - - def test_no_reasons_when_accepts(self) -> None: - consumer = self._make(frameworks={"torch"}) - producer = self._make(frameworks={"torch"}) - # accepts() is True — explain_mismatch should return empty/fallback - msg = consumer.explain_mismatch(producer) - assert msg # always returns a string - - def test_python_type_mismatch(self) -> None: - consumer = SampleType(input=PythonType("PIL.Image.Image")) - producer = SampleType(input=PythonType("dict")) - msg = consumer.explain_mismatch(producer) - assert "PIL.Image.Image" in msg and "dict" in msg - - def test_original_error_scenario(self) -> None: - """The exact case from the bug report: numpy op, torch upstream.""" - consumer = SampleType(input=ArrayType(frameworks=frozenset({"numpy"}))) - producer = SampleType( - input=ArrayType( - ndim=2, - shape=(Dim.exact(1), Dim.exact(1228800)), - dtype="complex64", - frameworks=frozenset({"torch"}), - ) - ) - msg = consumer.explain_mismatch(producer) - assert "numpy" in msg - assert "torch" in msg - assert "framework" in msg From 60cf785765bad17531845ed7609dd80abb8e4cca Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 11:40:55 +0200 Subject: [PATCH 035/102] fix(sampleflux): resolve generated-op annotations to real objects (to_pydantic robustness) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Alb*/Tv* generated ops synthesized __init__ with the library's raw (often PEP-563 string) param annotations referencing names local to the library module (cv2, Literal, …). to_pydantic's get_type_hints then eval'd them against the bridge module and raised IntrospectionError. Resolve each annotation to a real object against the transform's own module at generation time, degrading unresolvable ones to Any. Unblocks navigaitor/fluxstudio discovery. 561 passed. --- sampleflux/ops/_augment_bridge.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/sampleflux/ops/_augment_bridge.py b/sampleflux/ops/_augment_bridge.py index d58209e..eb7691c 100644 --- a/sampleflux/ops/_augment_bridge.py +++ b/sampleflux/ops/_augment_bridge.py @@ -22,7 +22,11 @@ """ import inspect -from typing import Any, Iterable, List, Optional, Tuple, Type +# Callable/Dict/Literal/Sequence/Union are referenced only inside the wrapped transforms' +# STRING annotations, which get_type_hints() evaluates against THIS module's globals when +# to_pydantic introspects a synthesized __init__ — so they must be importable here (flake8 +# can't see string-annotation usage). +from typing import Any, Iterable, List, Optional, Tuple, Type, get_type_hints from confluid import configurable from loggair import get_logger @@ -107,9 +111,25 @@ def __init__(self: Any, **kwargs: Any) -> None: # widgets / parse_param_docs) sees the transform's real parameters, keyword-only, with # the adapter's target/seed appended LAST. Required params are defaulted to None so # zero-arg construction always works (the workspace lazy-init mandate). + # Resolve annotations to REAL objects against the transform's own module. A library's + # param annotations are often strings (PEP 563) referencing names local to that module + # (cv2, Literal, the library's own aliases); left as strings they'd blow up later when + # to_pydantic's get_type_hints evals them against THIS module. Anything that still won't + # resolve degrades to Any so introspection never chokes on a stray name. + try: + _hints = get_type_hints(transform_cls.__init__) + except Exception: + _hints = {} + + def _resolve(p: inspect.Parameter) -> Any: + ann = _hints.get(p.name, p.annotation) + return Any if isinstance(ann, str) else ann + sig_params = [inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD)] for p in specs: - sig_params.append(p.replace(kind=inspect.Parameter.KEYWORD_ONLY, default=defaults[p.name])) + sig_params.append( + p.replace(kind=inspect.Parameter.KEYWORD_ONLY, default=defaults[p.name], annotation=_resolve(p)) + ) sig_params.append( inspect.Parameter("target", inspect.Parameter.KEYWORD_ONLY, default="none", annotation=TargetMode) ) @@ -118,7 +138,7 @@ def __init__(self: Any, **kwargs: Any) -> None: inspect.Parameter("seed", inspect.Parameter.KEYWORD_ONLY, default=None, annotation=Optional[int]) ) __init__.__signature__ = inspect.Signature(sig_params) # type: ignore[attr-defined] - annotations = {p.name: p.annotation for p in specs if p.annotation is not inspect.Parameter.empty} + annotations = {p.name: _resolve(p) for p in specs if p.annotation is not inspect.Parameter.empty} annotations["target"] = TargetMode if seed_param: annotations["seed"] = Optional[int] From 8d355f5ff4f36b88d976ba550dcaeb5bacf6700b Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 23 Jul 2026 17:24:39 +0200 Subject: [PATCH 036/102] =?UTF-8?q?docs:=20finish=20typed-bag=20purge=20?= =?UTF-8?q?=E2=80=94=20rewrite=20mandates/docs=20to=20typed-only,=20drop?= =?UTF-8?q?=20dead=20kinds/typespec=20refs,=20remove=20discovery=20ACCEPTS?= =?UTF-8?q?/PRODUCES=20dead=20branch,=20fix=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 25 ++++--- README.md | 30 ++++----- docs/architecture.md | 90 +++++++++++++------------- docs/augmentation.md | 12 ++-- docs/configure.md | 4 +- docs/graph.md | 6 +- docs/image.md | 4 +- docs/kinds.md | 104 ++++++++++++------------------ docs/projection.md | 2 +- docs/storage.md | 24 +++---- docs/typed-model.md | 72 ++++++++++----------- docs/typespec.md | 37 ----------- sampleflux/context.py | 13 ++-- sampleflux/discovery.py | 22 ++----- sampleflux/ops/_augment_bridge.py | 2 +- sampleflux/ops/albumentations.py | 12 ++-- sampleflux/ops/configure.py | 4 +- sampleflux/ops/context.py | 5 +- sampleflux/ops/formula.py | 8 +-- sampleflux/ops/image.py | 4 +- sampleflux/ops/sink.py | 4 +- sampleflux/ops/torchvision.py | 12 ++-- sampleflux/storage/hdf5.py | 2 +- sampleflux/storage/zarr.py | 4 +- 24 files changed, 216 insertions(+), 286 deletions(-) delete mode 100644 docs/typespec.md diff --git a/AGENTS.md b/AGENTS.md index da710b8..a05dcbd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,33 +3,32 @@ - **The Runnable Protocol Lives Here (`sampleflux.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** sampleflux owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `sampleflux.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `sampleflux.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `sampleflux.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `flux` validated in `run()`). `sampleflux.cli`: the `sampleflux run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `sampleflux-processing`/`sampleflux-workflow` + the `sampleflux` console script + `liquifai.apps`. - **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases). `Tee` threaded the sample through its branches sequentially, making it executionally identical to `TransformChain(ops=[...])` — use `TransformChain` for grouping and the context ops (`Save`/`Use`/`Mix`) for real, isolated fan-out. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom `ConfigureOp(ops=[UnstashInputOp(key)])` is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-sample side-branch (`ops` chain → `metadata[key]` + setattr) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters (fluxstudio export.py AND graphio.py) emit ONLY context ops for wiring; graphio's legacy `__taidal_stash_*` import replay was removed (pre-2026-07 stash-format ops-docs no longer import — re-export from the canvas). Do NOT reintroduce a metadata-bus twin of a context op. -- **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. **Scope (2026-07-21):** this mandate governs the CLASSIC engine (`sampleflux.core` / `sampleflux.ops` / `sampleflux.sample`). The experimental typed-bag redesign `sampleflux.bag` (a coexisting proof of concept — see [[typed-bag-model]] below and `docs/architecture.md`) DELIBERATELY introduces a `Transform` base + typed item classes; it does not relax this rule for the classic engine, whose ops stay plain callables. -- **Typed-Bag Model Is THE Data Model — Migration In Progress (`sampleflux.bag`, promoted 2026-07-21):** `sampleflux.bag` is the redesign that steps away from the `Sample(input, target, metadata)` triple, and the user has PROMOTED it from PoC to THE sampleflux data model. The workspace migration is STAGED (see root `TASKS.md` / the migration plan): the typed world is built beside the legacy engine against the FROZEN top-level API (`from sampleflux import TypedSample, Image, Transform, primary, register_io, ...` — new/migrated code imports ONLY from the package top level, never `sampleflux.bag.*` paths, so the eventual promotion of `bag/*` to the package root changes no consumer), consumers flip one project at a time, and a final purge stage deletes the legacy `Sample`/`kinds`/augment layer and renames `TypedSample` → `Sample` workspace-wide. Until that purge stage the legacy mandates below stay in force for `sampleflux.core`/`ops`/`sample`. Model recap: a `TypedSample` is a NAMED BAG of TYPED ITEMS (each owning its metadata), transforms dispatch on item TYPE via a kernel registry (`sampleflux.bag.dispatch`, the torchvision-v2 pattern) sampling params ONCE per sample (so a flip moves image+mask+boxes together), and external libraries (torchvision `transforms.v2`, albumentations) + user types plug in via `sampleflux.bag.adapters` + one-line `@Transform.kernel(ItemType)` registrations. BARE library transforms drop straight into a `Pipeline` — `coerce_transform` wraps any element via a registered matcher/factory (`register_adapter`); the torchvision/albumentations adapters self-register a matcher (by MRO module name, no eager library import) at package load, so `Pipeline([Fourier(), v2.Normalize(...), A.GaussNoise(...)])` works with no explicit adapter wrapper (use the explicit adapter with `only=` for per-key targeting). Items are HYBRID (array items subclass `np.ndarray` w/ attr-preserving `__array_finalize__`; structured items are dataclass wrappers). `input`/`target` are ROLE TAGS on fields, not tuple positions. **Modality-neutral (mandate above):** sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and ships **NO native augmentation transforms** — geometric/photometric augmentation comes from the libraries via coercion (the former native `HorizontalFlip` was DELETED; its kernels survive only as the `tests/_bag_fixtures.py::FixtureFlip` dispatch/parity fixture), and native `Transform`s exist only where no library covers them; the signal-domain items (`Signal`/`Spectrogram`) and the `Fourier` transform live in `waivefront.bag` and register into the SAME `sampleflux.bag` registries on import — do NOT add signal-domain items/transforms here. **Typed engine primitives (2026-07-21, migration Stage 1):** `primary(sample, role)` (first field of a role — THE "the input" accessor for bind/Apply/engines) + `TypedSample.merge(*samples)` (ordered field/role union, last-listed wins on collision — the typed fan-in that replaces metadata dict-merge) + `TypedSample.rename`; the item CODEC registry `sampleflux.bag.io` (`EncodedItem`/`encode_item`/`decode_item`/`register_io` — storage backends call ONLY the codec, so externally-registered item types serialize with zero storage edits); the structure ops `sampleflux.ops.structure` (`SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, entry point `sampleflux-ops-structure` — the typed replacement for triple-slot plumbing); and the `"typed"` collate (`collate.py::typed_collate`, auto-dispatched for `TypedSample` batches) returning a BATCHED TypedSample (payloads stacked per field, per-item attrs as lists, roles preserved) — the ONE batch convention that replaces both the list-form batched metadata and the `{"per_sample": [...]}` dict-nest. **Typed storage (migration Stage 2):** all three backends write a `TypedSample` in the ONE field-group layout (`sampleflux_format="typedsample-v1"`; per sample one group per FIELD: `__item_type__`/`__role__` + plain attrs natively, payload as `data`, array attrs under `attrs/`, insertion order in `__field_order__`; structured attr values ride the JSON-tagged wire format in `storage/base.py::split_attrs`/`restore_attrs` — tuples SURVIVE) — a store holds ONE carrier (typed↔legacy append raises); backends serialize ONLY through the `bag.io` codec so external item types round-trip with zero storage edits; `DirectorySink` gained its missing matching `DirectorySource` (typed layout); `ZarrBatchSink`'s typed path appends the PRIMARY input payload + a one-time uniform item template; typed metadata scans yield NESTED `{field: {attr: value}}` and `MetadataFilterSource.where` addresses it as `.` (`query.py::_AttrView`; a Python-keyword field name is unaddressable in an expression — use `predicate`). NO legacy readers/converter for old datasets (user decision — regenerate from sources). Pins: `tests/test_typed_storage.py`. **Typed engines (migration Stage 3):** a `TypedSample` is NEVER coerced — `core._as_carrier` passes it verbatim on every Flux route (no `native=True` needed), `core._apply_op`/`_apply_op_native` apply ops to the bag verbatim (the kinds binding + `_refresh_type` are legacy-only paths), and `Use`/`Apply`/`Capture`/`_cell_field` are typed-aware (`_cell_field` on a bag = the `key`-named item or `primary()`; `Apply` gained `key=`). FlowGraph: `FlowStep.merge_from` is the TYPED fan-in (UNION of the named steps' fields+roles via `TypedSample.merge`, slot order, last-write-wins; mutually exclusive with `target_from`/`metadata_from`, which stay legacy-only — cross-carrier use raises), lowered to the new `MergeFields` context op (`ops/context.py`, `sources`/`keys`/`drop`) and lifted back by `from_ops`; `bind:` gained the field form `step[key]` (the named item; bare `step` = the primary input item), lowered to `Apply(key=...)`. The derived-field branch idiom: produce → `SelectFields([new_field])` → `merge_from` (a FULL branch bag would last-wins-overwrite shared keys — deliberate). Pins: `tests/test_typed_flow.py`. The legacy engine is otherwise untouched — the "Functional Purity" / "Sample Triplet" / "Stored Type Is Derived" mandates stay in force for `sampleflux.core`/`ops`/`sample` until the purge stage. The subpackage is named `bag`, NOT `typed`, because `sampleflux.typespec.typed` (the `@typed(...)` contract decorator) is re-exported at the package root as `sampleflux.typed` and a `typed/` submodule would shadow it. `sampleflux.bag` imports without torchvision (adapters lazy-import). Entry point `sampleflux-bag-transform`. Usage: `docs/typed-model.md`; rationale: `docs/architecture.md` → "The typed-bag model"; pins: `tests/test_bag_*.py`, `examples/typed_pipeline.py`. Follow-ups (root TASKS.md): torch-Tensor-subclass item base, confluid-native item discovery, generated `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, the `decode` path. -- **`Sample.metadata` Is `dict` (single) OR `list[dict]` (batch) — Narrow via `.meta` / `.batch_meta`:** The `metadata` field is `Metadata = Union[Dict[str, Any], List[Dict[str, Any]]]`. A **single** item carries one `dict` (the normal pipeline form every source/op produces and consumes); a **batch** carries a `list` of per-item dicts (one per stacked item), produced by the collate functions (`marainer.collate.collate_fn_with_metadata`, `sonair.classification.classification_collate_fn`) when N samples are stacked into one Sample for the model/loss/predictions-sinks. `Sample.is_batched` (= `isinstance(metadata, list)`) is the single source of truth for telling them apart. Per-sample code MUST read/mutate metadata through the narrowing accessor **`sample.meta`** (returns the dict, raises `TypeError` on a batch) — `sample.meta[key]` / `sample.meta[key] = v`; batch consumers use **`sample.batch_meta`** (returns the list, raises on a single). NEVER index the raw `sample.metadata` Union directly (mypy rejects `Union[...][str]`). NOTE the batch convention is per-collate: marainer/sonair stack into the **list** form (`is_batched` True); deltaid's `segmentation_collate_fn` instead nests under a **dict** `metadata={"per_sample": [...]}` (so `is_batched` is False there — use `.meta["per_sample"]`). `describe()`/`with_type()` operate on single samples only (a batch infers / raises). Pins: `tests/test_sample.py` (batch vs single, `.meta`/`.batch_meta` guards). -- **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. (Scope: the CLASSIC engine; the coexisting `sampleflux.bag` PoC replaces the triple with a typed bag — see the "Typed-Bag Redesign" mandate above.) In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane — Never `sample.metadata` (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its `input`, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `Mix` (fan-in; named slots read cells, empty slots keep the incoming sample; metadata merges incoming-first then slot order, `metadata_from` wins last). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches `sample.metadata` — a linear run's metadata is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`), mirroring `UnstashInputOp(copy=True, remove=True)` — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); the stash family's charter is NARROWED to the two jobs cells cannot do — carrying a snapshot ACROSS a `Parallel` boundary (metadata rides the sample; cells raise at the boundary) and deliberately PERSISTING a snapshot into a sink's metadata (2026-07-18 consolidation); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. -- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `target_from`/`metadata_from` (fan-in slots, Mix field semantics), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on item TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `sampleflux.ops` stay plain callables. The `Transform` base is a thin type-dispatch shell (it samples params once per sample, then applies the per-type kernel to each handled field), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. +- **The Typed-Bag Model Is THE Data Model (`Sample`):** A `Sample` is a NAMED BAG of TYPED ITEMS, each item owning its own metadata. Import the whole typed surface from the PACKAGE TOP LEVEL (`from sampleflux import Sample, Image, Mask, Regions, Label, Transform, Pipeline, primary, item_data, typed_collate, register_item, register_kernel, register_adapter, register_io, ...`) — the `sampleflux.bag.*` module path is an internal/transitional home, never the taught import path. Transforms dispatch on item TYPE via a kernel registry (`@Transform.kernel(ItemType)` / `register_kernel`, the torchvision-v2 pattern) sampling params ONCE per sample (so a flip moves image+mask+boxes together), and external libraries (torchvision `transforms.v2`, albumentations) + user types plug in via registered adapters + one-line kernel registrations. BARE library transforms drop straight into a `Pipeline` — `coerce_transform` wraps any element via a registered matcher/factory (`register_adapter`); the torchvision/albumentations adapters self-register a matcher (by MRO module name, no eager library import) at package load, so `Pipeline([Fourier(), v2.Normalize(...), A.GaussNoise(...)])` works with no explicit adapter wrapper (use the explicit adapter with `only=` for per-key targeting). Items are HYBRID (array items subclass `np.ndarray` w/ attr-preserving `__array_finalize__`; structured items are dataclass wrappers). `input`/`target`/`aux` are ROLE TAGS on named fields, not tuple positions. **Modality-neutral (mandate above):** sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and ships **NO native augmentation transforms** — geometric/photometric augmentation comes from the libraries via coercion, and native `Transform`s exist only where no library covers them; the signal-domain items (`Signal`/`Spectrogram`) and the `Fourier` transform live in `waivefront` and register into the SAME sampleflux registries on import — do NOT add signal-domain items/transforms here. **Typed engine primitives:** `primary(sample, role)` (first field of a role — THE "the input" accessor for bind/Apply/engines) + `Sample.merge(*samples)` (ordered field/role union, last-listed wins on collision — the typed fan-in) + `Sample.rename`; the item CODEC registry (`EncodedItem`/`encode_item`/`decode_item`/`register_io` — storage backends call ONLY the codec, so externally-registered item types serialize with zero storage edits); the structure ops `sampleflux.ops.structure` (`SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, entry point `sampleflux-ops-structure` — the typed field-plumbing ops); and `typed_collate` (auto-dispatched for `Sample` batches) returning a BATCHED Sample (payloads stacked per field, per-item attrs as lists, roles preserved) — the ONE batch convention. **Typed storage:** all three backends write a `Sample` in the ONE field-group layout (`sampleflux_format="typedsample-v1"`; per sample one group per FIELD: `__item_type__`/`__role__` + plain attrs natively, payload as `data`, array attrs under `attrs/`, insertion order in `__field_order__`; structured attr values ride the JSON-tagged wire format in `storage/base.py::split_attrs`/`restore_attrs` — tuples SURVIVE); backends serialize ONLY through the `bag.io` codec so external item types round-trip with zero storage edits; `DirectorySink`↔`DirectorySource` (typed layout); `ZarrBatchSink` appends the PRIMARY input payload + a one-time uniform item template; typed metadata scans yield NESTED `{field: {attr: value}}` and `MetadataFilterSource.where` addresses it as `.` (`query.py::_AttrView`; a Python-keyword field name is unaddressable in an expression — use `predicate`). Pins: `tests/test_typed_storage.py`. **Typed engines:** a `Sample` passes through verbatim on every Flux route, `core._apply_op`/`_apply_op_native` apply ops to the bag verbatim, and `Use`/`Apply`/`Capture`/`_cell_field` are typed-aware (`_cell_field` on a bag = the `key`-named item or `primary()`; `Apply` gained `key=`). FlowGraph: `FlowStep.merge_from` is the fan-in (UNION of the named steps' fields+roles via `Sample.merge`, slot order, last-write-wins), lowered to the `MergeFields` context op (`ops/context.py`, `sources`/`keys`/`drop`) and lifted back by `from_ops`; `bind:` gained the field form `step[key]` (the named item; bare `step` = the primary input item), lowered to `Apply(key=...)`. The derived-field branch idiom: produce → `SelectFields([new_field])` → `merge_from` (a FULL branch bag would last-wins-overwrite shared keys — deliberate). Pins: `tests/test_typed_flow.py`. `sampleflux.bag` imports without torchvision (adapters lazy-import). Entry point `sampleflux-bag-transform`. Usage: `docs/typed-model.md`; rationale: `docs/architecture.md` → "The typed-bag model"; pins: `tests/test_bag_*.py`, `examples/typed_pipeline.py`. Follow-ups (root TASKS.md): torch-Tensor-subclass item base, confluid-native item discovery, generated `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, the `decode` path. +- **Metadata Lives PER ITEM, Never as One Flat Sample Dict:** There is no `Sample.metadata` dict — each typed item OWNS its own metadata (an `Image` knows its layout, a `Regions` its canvas, a `Signal` its samplerate, a `Label` its class names), carried as item attributes and serialized per field. Read or derive a value from the item that owns it — resolve the field via `primary(sample, role)` → `(key, item)`, then read the item's attrs / its `item_data(item)` payload — never from a string-keyed side dict. Batching is `typed_collate` (auto-dispatched for `Sample` batches): it returns a batched `Sample` with payloads stacked per field and each item's per-sample attrs collected into a list, roles preserved — the ONE batch convention (no separate `list[dict]` batched-metadata form and no `{"per_sample": [...]}` nest). Pins: `tests/test_typed_storage.py`, `tests/test_bag_*.py`. +- **Typed Bag — Full Traceability Rides on Items/Aux Fields:** All data flows through a `Sample` named bag of typed items; provenance is never dropped — everything that describes a value lives on the item that owns it, or as its own `aux`-role field, never bypassed. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. +- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its primary input item, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' fields into the incoming sample, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the sample's own fields — a linear run's sample is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. +- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' fields into this step, in slot order with last-write-wins), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. -- **The Transform Taxonomy Is a GRID — field scope × call style, with COMBINABLE per-param bindings (`sampleflux.kinds`, 2026-07-18; EXTENDS the introspection mandate below):** A transform declares WHICH slice of the `Sample(input, target, metadata)` triple it processes and HOW it wants to be called, from its `__call__` signature alone. `SampleKind` is now the closed Literal `sample`/`pair`/`input`/`target`/`input_meta`/`target_meta`/`value`/`any`; `CallStyle` is `packed`/`unpacked`; `OpContract` carries both. **Named views are real NamedTuples in `sampleflux.sample`** — `Pair(input, target)`, `InputMeta(input, metadata)`, `TargetMeta(target, metadata)` (metadata typed the same dict-or-list `Metadata` duality as Sample) — recognised by `Sample.from_any` AND `classify_carrier` BEFORE the generic 2-tuple rule (a view IS a tuple; positional coercion would misread `(input, metadata)` as `(input, target)` — pinned). Bare-field intent uses the PEP-593 marks `INPUT`/`TARGET` (aliases `Input = Annotated[Any, INPUT]`, `Target = …`; mark your own type via `Annotated[np.ndarray, INPUT]`). **Detection (`op_contract`) — per-parameter BINDINGS:** arity counts REQUIRED positional params only (optional extras keep single-arg semantics — the back-compat guard). Arity 2–3 → unpacked with `OpContract.bindings`: each param resolves ANNOTATION-first (`InputMeta`/`TargetMeta`/`Input`/`Target` marks, `dict` → `metadata`), then NAME (`input`/`target`/`metadata`/`meta`), then the POSITIONAL default `(input, target, metadata)` — so `f(input, target)`, `f(input, metadata)`, `f(target, metadata)`, `f(input, target, metadata)` reproduce the classic rules AND every combination works: `f(im: InputMeta, tm: TargetMeta)` (input AND target each WITH metadata), `f(x: Input, tm: TargetMeta)`, name-reordered `f(target, input)`, … `accepts` stays the covered-fields grid SUMMARY (`_bindings_summary`: i+t+m→sample, i+m→input_meta, …). Arity 1 → packed, scope from the annotation (`dict` → the new `metadata`-only scope; the `MetaDict` alias/`METADATA` mark exist for explicitness); untyped single stays `any` (NEVER name-sniff a single param — existing ops are untouched). `CALL_STYLE` joins the class-attr escape hatches. **Binding + merge-back:** unpacked ops route through `core._apply_bindings` — each argument bound per its binding (`_BIND_GET`), the result MUST be a same-arity tuple / a full `Sample` / `None` (a wrong arity OR a single named VIEW from a multi-binding op is a LOUD TypeError — a view IS a 2-tuple and would silently misread as two elements, guarded + pinned); each returned element merges per its binding (a view/2-tuple element on a `*_meta` binding replaces value+metadata, a bare element replaces only the value; metadata-bearing elements merge left-to-right, LAST write wins). Packed scopes in `core._apply_view`: `None` drops; a returned `Sample` takes over; `input`/`target` → bare value replaces the field; `metadata` → the dict in, a dict out (else loud error); `pair` → a `Pair` in, a 2-tuple out replaces input+target (metadata KEPT); the meta views receive the ACTUAL metadata dict (in-place mutation propagates). Packed `pair` binds a `Pair` (it IS a tuple, so plain-tuple-annotated ops index it identically while Pair-annotated ops get named fields). **Native fast lanes** (`_apply_op_native`): pair-op on a pair carrier and input-op on a bare value stay metadata-free native; every other non-any combination PROMOTES via the view-correct `from_any` (sticky). Default collates for `input_meta`/`target_meta` mirror the pair form (stacked value + list-of-dicts metadata). These NAMES are the vocabulary FluxStudio will surface as socket types (a later stage — TASKS.md). Pins: `tests/test_kinds.py::TestGridContracts`/`TestGridEngineBinding`. +- **Transforms Dispatch on Item TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which item TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per sample, then applies the matching kernel to EVERY field whose item type it handles, passing untouched fields through. Because the parameters are sampled once and shared, multi-field consistency is automatic — one flip moves image + mask + boxes together (the thing a flat-metadata triple could not express). Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them. A transform MAY change an item's type under the same field key (e.g. `Fourier` turns a `Signal` field into a `Spectrogram` in place). Bare library transforms (torchvision `transforms.v2`, albumentations) drop into a `Pipeline` via registered adapters (`register_adapter` / `coerce_transform`); a plain function becomes a transform via `as_transform(fn, handles=(ItemType,), only=[field])`, and a type-changing shape (read one field, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. Optional `only=[keys]` narrows a transform to specific field keys. Pins: `tests/test_bag_*.py`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19):** Every op that wraps/applies OTHER ops — `TransformChain`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `sampleflux.core._apply_op(sample, op)`, NEVER `op(sample)` directly. `_apply_op` is the engine's single contract-aware chokepoint: it introspects the inner op's transform-taxonomy contract (`op_contract`) and binds the declared view (pair / input / target / `*_meta`, packed or unpacked), so a field-scoped op (e.g. a pair-scoped augmentation adapter) nests inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(sample)` call crashes on any non-sample-scoped op with a misleading "missing positional argument" `TypeError`. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Sample]`. Pins: `tests/test_augment_ops.py` (pair op inside `RandomApply`/`TransformChain`/`Enable`). - **Augmentation = Adapter Ops + GENERATED Per-Transform Families, Never Reimplement (`sampleflux.ops.albumentations` / `.torchvision` / `.albumentations_transforms` / `.torchvision_transforms` / `._augment_bridge`, 2026-07-19):** Library augmentation enters the pipeline through two SAMPLE-scoped adapter ops — `AlbumentationsOp` (albumentations, numpy HWC in/out, core dep) and `TorchvisionTransformOp` (torchvision `transforms.v2`, CHW tensors out, `[vision]` extra, ALL torchvision imports lazy in the ADAPTER module so it imports without the library — pinned by `test_module_imports_without_torchvision`) — plus the AUTO-GENERATED per-transform op families: `sampleflux.ops._augment_bridge.generate_transform_ops` (the waivefront-helios auto-bridge pattern) walks each library's public transform classes at import time and emits one adapter SUBCLASS per transform (`Alb` ~115 ops, group `augment/albumentations`; `Tv` ~55 ops, group `augment/torchvision`) with a synthesized `__signature__`/`__annotations__`/spliced `Args:` docstring (transform params + `target`/`seed` appended LAST), the base `__call__` RE-STATED in the class dict (canvas op-classification reads `vars(cls)` — inherited-only methods are invisible), and a `raw_transform` property an adapter's `transforms` list UNWRAPS (so canvas transform nodes dock into a Compose-style adapter node). The `Alb`/`Tv` prefixes are MANDATORY (confluid's registry is flat + name-keyed; the libraries share bare names like `ColorJitter`/`Normalize`/`Resize` — the helios `Helios*` precedent); composition/container transforms are NOT generated (chaining is native); per-class generation failures skip with a DEBUG note, never break import; `torchvision_transforms` imports safely without torchvision (zero ops). Adapters: `@configurable(category="op", group="augment", random=True)`, `__call__(self, sample: Sample)` — SAMPLE-scoped deliberately, because the visual-canvas op classifier only recognises `__call__(sample: Sample)` and invokes `op(sample)` directly (pair-scoped ops are engine-legal but canvas-invisible until the kinds-grid socket stage lands); ONE library draw still moves input AND target jointly per the closed `TargetMode = Literal["none","mask","boxes"]` knob (`"boxes"` consumes the torchvision detection dict from `CocoToTorchVisionDetectionOp`/`MasksToDetectionBoxesOp`; albumentations `bbox_params` are AUTO-ADDED when the op composes — only a prebuilt `A.Compose` must carry its own, validated loudly). Config surface: `transform` (ONE transform / prebuilt Compose) XOR `transforms` (list, composed lazily; entries may be live objects, Confluid markers — flowed lazily — or generated ops); `seed` on the albumentations side maps onto `A.Compose(seed=...)` (rejected with a prebuilt Compose); NO probability knobs (gating is `RandomApply`). **YAML is Confluid-NATIVE ONLY** — nested `!class:albumentations.HorizontalFlip` / `!class:torchvision.transforms.v2.X` nodes or registered short names (`!class:AlbHorizontalFlip`), dump→load round-trips both (the engine captures foreign-class ctor kwargs); the earlier `A.to_dict()` dict-spec surface was REMOVED (user-rejected — never resurrect a library-specific serialization format as config). Don't add a third adapter without real demand, and never bake a specific augmentation as a bespoke hand-written sampleflux op — wrap the library or use the generated family. Docs: `docs/augmentation.md`; examples: `examples/augmentation_ops.py` / `examples/augmentation_training.py`; pins: `tests/test_augment_ops.py`, `tests/test_categories.py`. -- **Op Kind Is INTROSPECTED, Never Declared in the Engine (`sampleflux.kinds`, 2026-07-17):** The native multi-type engine (`Flux(native=True)`, OPT-IN — the `native=False` default coerces to `Sample` exactly as before, so all consumers are untouched) carries `Sample` triplets, metadata-free **pairs** (2-tuples), and bare **values** through one pipeline, adapting each op via `op_contract(op)` → `OpContract(accepts, produces, expands)` cached per type: `__call__`'s first-param annotation (`Sample`→`sample`, `tuple[...]`→`pair`, missing/`Any`→`any`) and return annotation (`Iterator[...]`/`Iterable[...]`/`List[...]` → `expands=True` — a `Tuple` return is a PAIR, never an expansion). ANY introspection failure (lazy imports, unresolvable forward refs) degrades to `any` so an untyped/exotic op behaves exactly as today; the class attrs `SAMPLE_KIND_IN`/`SAMPLE_KIND_OUT`/`EXPANDS` are the explicit escape hatch and ALWAYS win. Adaptation rules (`core._apply_op_native`): a pair-op on a Sample gets `sample.to_pair()` and its returned pair merges back via `_replace` (METADATA PRESERVED); a sample-op on a pair/value gets a PROMOTED `Sample.from_any` view — promotion is one-way and STICKY (op-written metadata is never dropped); an any-op gets the carrier verbatim. `SampleKind` is a closed Literal (`sample`/`pair`/`value`/`any`; runtime tuple `SAMPLE_KINDS = get_args(...)` — one source of truth); `classify_carrier` is the runtime classifier (ONLY a 2-tuple is a pair). **Collation is the pluggable registry `sampleflux.collate`** (`register_collate(key)` / `get_collate` / `collate(items, key=None)` — key defaults to the detected kind): sampleflux registers `"sample"` (list-form batched metadata — the `is_batched` convention) / `"pair"` / `"value"` defaults; consumers register task aliases ADDITIVELY and their divergent conventions (deltaid/raidar `{"per_sample": …}`) are deliberately NOT unified (TASKS.md follow-up). Pins: `tests/test_kinds.py`. -- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op whose return annotation is `Iterator[...]`/`Iterable[...]`/`List[...]` (or that carries `EXPANDS = True`) is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op's contract expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `HDF5WindowSource`/`RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. `_refresh_type` is applied per CHILD (`core._expand`). Pins: `tests/test_expanding_ops.py`. +- **Collation Is a Pluggable Registry (`sampleflux.collate`):** Batching a list of `Sample` bags into ONE batched `Sample` goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"typed"`** = `typed_collate`: it stacks each field's array payload, gathers per-item attrs as per-sample lists, and preserves roles (see the per-item-metadata mandate above). Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` fields, `raidar.detection_collate_fn`); their divergent conventions are deliberately NOT unified. `typed_collate` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op whose return annotation is `Iterator[...]`/`Iterable[...]`/`List[...]` (or that carries `EXPANDS = True`) is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op's contract expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `HDF5WindowSource`/`RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. Pins: `tests/test_expanding_ops.py`. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `sampleflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into `Sample` triplets — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores no target/metadata). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `SAMPLEFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into typed-bag `Sample`s — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores only the primary input payload). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `SAMPLEFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`sampleflux.storage.query`, 2026-07-17):** `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; implemented on `HDF5Source` (attrs + array-metadata shape/dtype STUBS) and `ZarrGroupSource` (`.zattrs`) — existing files queryable with NO rewrite; the protocol is STRUCTURAL, so external storage sources (e.g. waivefront's `SigMFSource`) implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration (the projection-module pattern). Entry point `sampleflux-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair (`SigMFSink`/`SigMFSource`) MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; sampleflux keeps ZERO knowledge of it. Pins: `tests/test_query.py`, `waivefront/tests/test_sigmf.py`. - **Field Projection (`sampleflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `sampleflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). - **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`sampleflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing sampleflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a sampleflux dependency for this. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The sampleflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, `Copy*Op`, `Stash*`/`Swap*`, `Parallel`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over `sample.input` — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (incl. the full stash family `StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp` — input AND target snapshots on the METADATA bus, kept ONLY for crossing a `Parallel` boundary or deliberately persisting a snapshot into a sink (graph wiring uses the context ops); the `Unstash*Op`s default to `remove=True`, DELETING the key after restoring it so a snapshot never lingers on the bus into a sink — set `remove=False` only when the SAME key is unstashed again later, e.g. a fan-out's non-final restores, which the compiler emits — and `DropMetadataOp` = `sampleflux.ops.metadata`, a pass-through op that strips metadata keys matching `fnmatch` GLOB patterns (`*`/`?`/`[seq]`; a wildcard-free pattern = exact key, case-sensitive). A key drops iff it matches an `exclude` pattern AND NOT any `include` pattern — `include` PROTECTS keys and takes PRIORITY (rsync/gitignore include-wins model), e.g. `exclude=["spec_*"]` + `include=["spec_keep"]` clears every bulky snapshot key EXCEPT the protected one, before a sink serialises the bus (with no `exclude`, nothing drops) / `compose` (`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the structure ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, `Parallel`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over the sample's primary input item (`primary(sample, "input")`) — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (the typed field-plumbing ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — re-tag a field's role, rename or copy a field key, drop a field, or narrow the bag to a chosen set of fields; these are how a derived-field branch is assembled and how a snapshot is carried across a `Parallel` boundary or persisted into a sink as its own `aux`-role field) / `compose` (`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). -- **Type Specs Live in `sampleflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). The small fixed string sets are **closed `Literal`s, not bare `str`** (workspace "prefer closed `Literal`s over bare strings" mandate), all exported and enumerable via `typing.get_args(...)` for UIs / the FluxStudio connection-validator: `Framework = Literal["numpy", "torch", "tensorflow"]` (the `ArrayType.frameworks` element type + `image()`/`parse()` `framework=`), `ImageLayout = Literal["CHW", "HWC"]` (`ArrayType.image(layout=)`), and the dtype trio `Dtype` (concrete names — exactly the union of the `_DTYPE_FAMILIES` members) + `DtypeFamily` (the family names — exactly the `_DTYPE_FAMILIES` keys) + `DtypeSpec = Union[Dtype, DtypeFamily]` (the `ArrayType.dtype` field type). Extend a Literal — don't widen to `str` — when adding real support (a new framework, a new dtype). The `_DTYPE_FAMILIES` map (family→members) stays the runtime source of truth and is looked up by arbitrary canonical dtype string (so it stays `str`-keyed, NOT keyed by the Literal); `tests/test_typespec.py` pins `Dtype`/`DtypeFamily` equal to it so they can't drift. **`canonical_dtype` is the single boundary** where arbitrary input (aliases like `"double"`, casing like `"FLOAT32"`, framework dtype objects, and genuinely exotic platform dtypes like `float128`) is normalized into the typed `DtypeSpec` domain — hence its closing `cast`; an unmodeled dtype keeps its own name and matches no family. So authored `ACCEPTS`/`PRODUCES` dtypes must be canonical Literal members (a typo/alias is a type error at the call site), while runtime/inferred/deserialized values stay tolerant. The serialization `kind` discriminator stays `str` (read from untrusted JSON; `to_dict` returns `Dict[str, Any]`; round-trip tests guard it). -- **Stored Type Is Derived, Never a 4th Field:** (Classic engine; in `sampleflux.bag` type IS the item's Python class, carried per field — see the "Typed-Bag Redesign" mandate.) A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. +- **Type IS the Item's Python Class, Never a Separate Field:** A field's type is its item's Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key. A consumer reads a field's type by `type(item)` and its shape/dtype/framework off the item's own payload and attrs. A transform that changes a value's type replaces the item under the same field key (e.g. `Signal` → `Spectrogram`, `array` → `Mask` → `Regions`). Never carry a parallel type descriptor beside the bag. ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. diff --git a/README.md b/README.md index a567e73..1909b6c 100644 --- a/README.md +++ b/README.md @@ -7,34 +7,35 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `SampleF ## 🚀 Key Features - **Functional Purity:** Transforms are simple Python callables. No complex base classes required. -- **Standardized Sample Triplet:** Standardizes on `(input, target, metadata)` for full traceability — while the [transform taxonomy](docs/kinds.md) lets an op process just the slice it cares about (`input`, `pair`, `input_meta`, …) in whichever calling style its signature declares. +- **Typed Bag of Items:** A `Sample` is a named bag of typed items (`Image`, `Mask`, `Regions`, `Label`, …), each owning its own metadata — and [transforms dispatch on item type](docs/typed-model.md), so one sampled decision moves image + mask + boxes together. - **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-sample `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Flux` engine. - **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. - **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-samplefluxstoragequery) — filter stored datasets without loading a single array. -- **Passive Introspection:** ops declare [type contracts](docs/typespec.md) and are discoverable by category for visual editors and schema generators. +- **Passive Introspection:** transforms declare their [item-type contracts](docs/typed-model.md) (the item types they handle / consume / produce) and are discoverable by category for visual editors and schema generators. - **100% Reproducibility:** Entire pipelines are serializable via **Confluid** manifests. ## 🛠 Quick Start ```python import numpy as np -from sampleflux.core import Flux +from sampleflux import Sample, Image, Flux, as_transform, primary -# 1. Define a simple transformation -def normalize(data: np.ndarray, mean: float = 0.0): - return data - mean +# 1. A plain function becomes a transform, dispatched on item type +recenter = as_transform(lambda d: d - 0.5, handles=(Image,)) -# 2. Build a pipeline -raw_data = [np.random.randn(10) for _ in range(100)] +# 2. Build a pipeline over a source of typed samples +raw_data = [Sample({"input": Image(np.random.randn(10))}) for _ in range(100)] -flux = Flux(raw_data) \ - .map(normalize, mean=0.5) \ - .filter(lambda s: s.input.mean() > 0) \ +flux = ( + Flux(source=raw_data, ops=[recenter]) + .filter(lambda s: primary(s, "input")[1].mean() > 0) .parallel(workers=4) +) # 3. Collect or stream for sample in flux: - print(sample.input.shape) + _, item = primary(sample, "input") # (key, item) + print(item.shape) ``` ## 📚 Documentation @@ -45,12 +46,11 @@ for sample in flux: | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Flux.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources, array-valued metadata, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | -| [docs/typespec.md](docs/typespec.md) | The flexible array/type system, `ACCEPTS` / `PRODUCES` op contracts, closed `Literal` vocabularies | | [docs/projection.md](docs/projection.md) | Field projection (`SupportsProjection`), lazy target walks, `num_classes`, the fittable `LabelMap` | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImageOp`, `NormalizeToUint8Op`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via albumentations / torchvision `transforms.v2` — joint input+target (mask/boxes) adapters, the generated `Alb*`/`Tv*` per-transform ops, seeding, Confluid-native YAML | -| [docs/typed-model.md](docs/typed-model.md) | **Experimental** — the typed-bag model (`sampleflux.bag`): a named bag of typed items (each owning its metadata), type-dispatched transforms, torchvision/albumentations adapters, custom item types | +| [docs/typed-model.md](docs/typed-model.md) | The typed-bag data model: a `Sample` is a named bag of typed items (each owning its metadata), type-dispatched transforms, torchvision/albumentations adapters, custom item types | | [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | ## 🧭 Scope: a modality-neutral engine @@ -64,7 +64,7 @@ SampleFlux deliberately contains **no domain-specific code** — every op, sourc SampleFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: -- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into `Sample` triplets with full metadata traceability (see [docs/sources.md](docs/sources.md)). +- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into typed `Sample` bags with full metadata traceability (see [docs/sources.md](docs/sources.md)). - **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node, every run reproducible. - **PyTorch**: `Flux` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#multi-type-carriers--the-collate-registry-samplefluxcollate). - **Augmentation libraries**: `AlbumentationsOp` / `TorchvisionTransformOp` wrap [albumentations](https://albumentations.ai) and torchvision `transforms.v2` as ops that augment input AND target (mask / detection boxes) jointly — plus an auto-generated op per individual library transform (`AlbHorizontalFlip`, `TvColorJitter`, …), each a graph node and a Confluid `!class:` one-liner (see [docs/augmentation.md](docs/augmentation.md); torchvision via `pip install "sampleflux[vision]"`). diff --git a/docs/architecture.md b/docs/architecture.md index 2a73768..fb8056b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,9 +36,10 @@ and **two divergent batched-metadata conventions** emerged — the list-form `sampleflux/collate.py` is a **pluggable registry of collate functions keyed by representation**: `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`, where an omitted key -dispatches on the *detected* kind of the first item (`sampleflux.kinds.classify_carrier`). -sampleflux registers the five kind defaults (`"sample"`, `"pair"`, `"value"`, `"input_meta"`, -`"target_meta"`); consuming projects may register task aliases (`"yolo"`, `"segmentation"`, …) +uses the default `"typed"` collate (`typed_collate`) — batching a list of typed-bag `Sample`s into +one batched `Sample` (per-item type dispatch itself is the sibling kernel registry +`sampleflux.bag.dispatch`, which walks each item's MRO). sampleflux registers the `"typed"` +default; consuming projects may register task aliases (`"yolo"`, `"segmentation"`, …) **additively**. Re-registering a key deliberately overwrites (logged at debug) so a consumer can replace a default. @@ -227,7 +228,7 @@ module to introspect every callable *defined in it* — registered or not. `resolve_callable(path)` back to the live object (module import, `.py`-file load, or an already-callable passthrough). - **Introspection** — `introspect_callable(fn)` → a JSON-serializable schema (path, name, doc, - per-parameter type/default/required, the declared `ACCEPTS`/`PRODUCES` typespec contract), and + per-parameter type/default/required), and `scan_module(module_or_py)` applying it to every callable *defined in* a module (`__module__`-filtered, so imports don't leak in). @@ -335,13 +336,13 @@ both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flux_a --- -## The typed-bag model: a named bag of typed items (`sampleflux.bag`, PoC, 2026-07-21) +## The typed-bag model: a named bag of typed items (`sampleflux.bag`, 2026-07-21) ### Context -The classic carrier is `Sample(input, target, metadata)` — a fixed 3-tuple where `metadata` is one -flat `dict` shared by the whole sample. Everything that is not literally the model input or target -rides that dict by string key: segmentation masks, `[f0,f1,t0,t1]` region lists, window locators, +Before the typed model, the carrier was a fixed `(input, target, metadata)` 3-tuple where `metadata` +was one flat `dict` shared by the whole sample. Everything that is not literally the model input or +target rode that dict by string key: segmentation masks, `[f0,f1,t0,t1]` region lists, window locators, `spectrogram_params`, power stats, `snr_db`, a signal's samplerate, an image's canvas size, a label's class names. Two structural costs follow. First, **metadata has no owner** — `samplerate` belongs to *the signal*, `canvas` to *the image*, but the flat dict severs that link. Second, **a @@ -364,10 +365,10 @@ dispatches transforms on item TYPE via a kernel registry: pair hides the difference from kernels. sampleflux ships only MODALITY-NEUTRAL items; signal-domain items (`Signal`, `Spectrogram`) live in the domain package and register into the same registry (see "Consequences"). -- **`TypedSample` is a named bag; `input`/`target` are role TAGS, not positions.** A field carries a +- **`Sample` is a named bag; `input`/`target` are role TAGS, not positions.** A field carries a role (`input`/`target`/`aux`/`pred`); `inputs()`/`targets()`/`aux()` read them at the - train/collate/sink boundary. A field changes role without moving keys. The sample is immutable - (copy-on-write), mirroring `Sample._replace`. + train/collate/sink boundary. A field changes role without moving keys. The sample is immutable — + every mutator returns a new sample (copy-on-write). - **Transforms sample params ONCE, then dispatch a kernel per item type** (the torchvision-v2 `_KERNEL_REGISTRY` pattern, structurally the same registry idea as `sampleflux.collate`). Kernels are registered per `(transform, item type)` and resolved by MRO. Targeting is by type, with an @@ -381,23 +382,23 @@ dispatches transforms on item TYPE via a kernel registry: transform with one `@Transform.kernel(NewType)` registration. This keeps consumer-dialect knowledge (how to recognise/adapt a library) OUT of the core and open for any user library. -This is a **coexisting proof of concept**, not a replacement: it lives beside the classic engine and -changes none of it. The existing "Functional Purity", "Sample Triplet", and "Stored Type Is Derived" -mandates are scoped to the classic engine (see `AGENTS.md`), because the typed model deliberately -introduces a `Transform` base and typed item classes. +This is **THE sampleflux data model** — the one carrier every source, op, engine and sink handles. +It deliberately introduces a `Transform` base and typed item classes; the "Functional Purity" mandate +(see `AGENTS.md`) holds because that base is a thin type-dispatch shell and the per-type kernels stay +plain callables. ### Consequences - **Cross-field consistency is free** — one sampled decision flips image + mask + boxes together, - the thing the flat-metadata model could not do. -- **Names and types coexist**, so the "torchvision uses types / albumentations uses names" split is - resolved by one container: the key is the name, the item is the type. -- **The subpackage is `bag`, not `typed`** — `sampleflux.typespec.typed` (the `@typed(...)` contract - decorator) is re-exported at the package root as `sampleflux.typed`, so a `sampleflux/typed/` - submodule would shadow it. `bag` is collision-free; `TypedSample` / dispatch / docstrings carry the - "typed" concept. -- **Batching stays in `sampleflux.collate`** (transforms are per-sample); the classic model's - `list[dict]` batch-in-metadata form is not carried into the bag model. + the thing a flat-metadata triple could not do. +- **Names and types work together**, so the "torchvision uses types / albumentations uses names" + split is resolved by one container: the key is the name, the item is the type. +- **The subpackage is `bag`, an internal module home** — the whole typed surface is imported from + the package top level (`from sampleflux import Sample, ...`), so the module layout is never in a + consumer's import path and can move without touching consumers. +- **Batching is `typed_collate`** — it returns a batched `Sample` (payloads stacked per field, + per-item attrs collected as lists, roles preserved); there is no `list[dict]` batch-in-metadata + form. - **Deliberately deferred** (see root `TASKS.md`): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item-type discovery, the generated `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, and the `decode` path. @@ -405,11 +406,11 @@ introduces a `Transform` base and typed item classes. ### Example ```python -from sampleflux import TypedSample, Image, Mask, Regions, Label, Pipeline +from sampleflux import Sample, Image, Mask, Regions, Label, Pipeline from torchvision.transforms import v2 import albumentations as A -sample = TypedSample( +sample = Sample( {"image": Image(rgb), "mask": Mask(seg), "regions": Regions(boxes, canvas=(H, W)), "class": Label("drone_x")}, roles={"mask": "target", "regions": "target", "class": "target"}, ) @@ -430,10 +431,9 @@ out = Pipeline([ it is array-backed, subclass `NDArrayItem` and declare `_item_attrs`. - **A new per-type behaviour for an existing transform** — register a kernel (`@Transform.kernel(ItemType)`), no core edit. -- **Do not name the `bag` subpackage `typed`** — it shadows `sampleflux.typed` (the `@typed` - decorator). The rename rationale is pinned here and in the module docstrings. -- **Promoting this from PoC to the default model** is a workspace-wide decision that would re-scope - the classic-engine mandates and port every consumer — out of scope for the proof of concept. +- **The typed surface is imported from the package top level** — `bag/*` is the internal module + home; never teach a `sampleflux.bag.*` import path, so the module layout can change without + touching consumers. --- @@ -452,10 +452,10 @@ But a running detection/segmentation front-end needs a different shape: **read o field of a DIFFERENT type**. Turning a numeric array into a displayable image, thresholding an array into a boolean mask, and labelling that mask into a set of bin boxes are each a *type change* (`array → Image`, `array → Mask`, `Mask → Regions`), not an in-place per-type edit. No -library provides them, and the legacy classic-engine ops that do (`ConvertToImageOp`, -`ThresholdOp`, `ConnectedComponentsOp`) operate on the `Sample(input, target, metadata)` triple, -which the typed world does not carry. Without typed equivalents a `TypedSample` pipeline could not -reach `Regions` from a raw array — the critical path for typed detection was blocked. +library provides them, and the earlier ops that did (`ConvertToImageOp`, `ThresholdOp`, +`ConnectedComponentsOp`) operated on a flat `(input, target, metadata)` triple, which the typed +model does not carry. Without typed equivalents a `Sample` pipeline could not reach `Regions` from a +raw array — the critical path for typed detection was blocked. ### Decision @@ -488,7 +488,7 @@ coordinate frame, so the tuple order is load-bearing, not incidental. ### Consequences -- A `TypedSample` carrying a raw 2-D array runs `ConvertToImage → Threshold → ConnectedComponents` +- A `Sample` carrying a raw 2-D array runs `ConvertToImage → Threshold → ConnectedComponents` end-to-end and arrives at a `Regions` field with no legacy `Sample` anywhere — the typed detection/segmentation front-end is unblocked. - Parity is free and provable: because each twin reuses the legacy math, a twin's output is @@ -519,11 +519,11 @@ coordinate frame, so the tuple order is load-bearing, not incidental. ### Example ```python -from sampleflux import TypedSample, Mask +from sampleflux import Sample, Mask from sampleflux.ops.image import ConvertToImage from sampleflux.ops.numpy import Threshold, ConnectedComponents -sample = TypedSample({"spec": Mask(db_spectrogram)}) # a raw 2-D array item +sample = Sample({"spec": Mask(db_spectrogram)}) # a raw 2-D array item sample = ConvertToImage()(sample) # + Image field (role "input") sample = Threshold(field="spec", low_level=-30.0)(sample) # + Mask field (role "aux") sample = ConnectedComponents(field="mask")(sample) # + Regions field (role "aux") @@ -549,8 +549,8 @@ sample["boxes"].boxes # [(row_min, row_max, col_min, col_max), ...] — the pin The typed detection twins above reach `Regions`; a typed CLASSIFICATION front-end needs the other two shapes: turn the working image into the model's **input tensor**, and turn the class-name label -into the encoded **target id**. The legacy ops that do this (`ToTensorOp`, `MetadataToTargetOp`, -`EncodeTargetOp` / `DecodeTargetOp`) operate on the `Sample(input, target, metadata)` triple. Two +into the encoded **target id**. The earlier ops that did this (`ToTensorOp`, `MetadataToTargetOp`, +`EncodeTargetOp` / `DecodeTargetOp`) operated on a flat `(input, target, metadata)` triple. Two facts of the typed model shape the twins: (1) there is NO shared metadata dict — the label already rides a `Label` field that owns its metadata; (2) an array item is an `np.ndarray` SUBCLASS whose `__new__` runs `np.asarray(data)`, so **a field payload is coerced to numpy** — an `Image` cannot @@ -571,7 +571,7 @@ detection twins), each reusing its legacy op VERBATIM on a shim `Sample` for byt new field tagged `input` instead). `typed_collate` stacks these payloads with `np.stack`; the numpy→tensor conversion is the collate / model boundary's job, exactly as for any numpy dataset. A Tensor-subclass item that would let a field carry a live tensor is the documented follow-up - (`bag/items.py` PoC note + root TASKS.md). + (`bag/items.py` note + root TASKS.md). - **`EncodeTarget` / `DecodeTarget`** (`ops/target.py`, `group="structure"`) resolve a `Label` field, map its `.value` through the config-pinned `mapping` by delegating to `EncodeTargetOp` / `DecodeTargetOp` (so the non-empty-mapping validation AND the shared `_lookup` are byte-identical), @@ -585,11 +585,11 @@ detection twins), each reusing its legacy op VERBATIM on a shim `Sample` for byt ### Consequences -- A `TypedSample` carrying an HWC `Image` (role input) + a name `Label` (role target) runs +- A `Sample` carrying an HWC `Image` (role input) + a name `Label` (role target) runs `ToTensor → EncodeTarget` into a CHW-float input field + an int-id target field, with no legacy `Sample` anywhere — the typed classification front-end is unblocked. - The model-input payload is CHW-float **numpy**, not a live `torch.Tensor`; a consumer / trainer - tensorizes at the collate or forward boundary. This is a deliberate PoC limitation, not a bug — + tensorizes at the collate or forward boundary. This is a deliberate current limitation, not a bug — it disappears when the Tensor-subclass item lands. - The twins carry `category="op"` + the legacy `group`, so they are discoverable like the legacy ops (their modules — `sampleflux-ops-torch` / `sampleflux-ops-target` — are already entry-pointed; a @@ -598,11 +598,11 @@ detection twins), each reusing its legacy op VERBATIM on a shim `Sample` for byt ### Example ```python -from sampleflux import TypedSample, Image, Label +from sampleflux import Sample, Image, Label from sampleflux.ops.torch import ToTensor from sampleflux.ops.target import EncodeTarget -sample = TypedSample( +sample = Sample( {"image": Image(hwc_uint8), "class": Label("cat")}, roles={"image": "input", "class": "target"}, ) @@ -613,7 +613,7 @@ sample = EncodeTarget(mapping={"cat": 0, "dog": 1}, field="class")(sample) # cl ### What you may change (and where it's documented) - **The Tensor-subclass item follow-up** — once a field can carry a live tensor, `ToTensor` should - store it directly; update this record and the `bag/items.py` PoC note together. + store it directly; update this record and the `bag/items.py` note together. - **`ToTensor`'s replace-in-place default vs a new output field** — keep role preservation (in place) as the default; a new `output` field is tagged `input`. - **Do not modify the legacy ops or reimplement their math in a twin** — the twins delegate to the diff --git a/docs/augmentation.md b/docs/augmentation.md index 7d0addc..fcd7dcd 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -14,8 +14,8 @@ its own first-class op: All are ordinary sample-scoped ops (`__call__(sample)`): they chain in a `Flux` ops list, inside `TransformChain` / `RandomApply` / `Enable`, in Confluid YAML, and as individual nodes on a visual canvas (palette groups `augment`, `augment/albumentations`, -`augment/torchvision`). One library draw applies jointly to `sample.input` and — per the -`target` mode — its mask / boxes; metadata passes through untouched. +`augment/torchvision`). One library draw applies jointly to the input-role field and — per the +`target` mode — its mask / boxes; other fields pass through untouched. Torchvision requires the `vision` extra: `pip install "sampleflux[vision]"` (albumentations is a core dependency; without torchvision the `Tv*` family is simply @@ -25,11 +25,11 @@ empty and everything else works). The `target` knob is a closed `Literal["none", "mask", "boxes"]` on every op above: -- `"none"` (default) — input-only augmentation (color jitter, noise, blur); the sample's - target passes through untouched. -- `"mask"` — `sample.target` is a segmentation mask (2-D array or PIL `L` image); image +- `"none"` (default) — input-only augmentation (color jitter, noise, blur); the target-role + field passes through untouched. +- `"mask"` — the target-role field is a segmentation mask (2-D array or PIL `L` image); image and mask receive the SAME spatial transform. -- `"boxes"` — `sample.target` is the torchvision detection dict +- `"boxes"` — the target-role field is the torchvision detection target `{"boxes": [N,4] xyxy-pixel, "labels": [N]}` — exactly what `CocoToTorchVisionDetectionOp` and `MasksToDetectionBoxesOp` emit — and boxes move with the image. The required albumentations `bbox_params` are added automatically when the op builds the Compose; diff --git a/docs/configure.md b/docs/configure.md index 2864db9..1f5c427 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -2,7 +2,7 @@ Some op parameters are only known *per sample*. Two mechanisms cover this: -- **`ConfigureOp(ops, target, param, key)`** — runs `ops` on the sample as a side-branch; the chain's final `sample.input` is written to `metadata[key]` and injected as `target.`, then `target` is applied. Use it when the value is *derived from the sample itself* (e.g. a threshold from the sample's own max) — the whole derivation reads as one node/YAML block. +- **`ConfigureOp(ops, target, param, key)`** — runs `ops` on the sample as a side-branch; the chain's final primary input value (`primary(sample, "input")`) is injected as `target.` and also recorded as an `aux`-role field named `key`, then `target` is applied. Use it when the value is *derived from the sample itself* (e.g. a threshold from the sample's own max) — the whole derivation reads as one node/YAML block. - **`Capture` + `Apply`** (`sampleflux.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. ```yaml @@ -32,4 +32,4 @@ ops: key: derived_threshold ``` -`ConfigureOp` also stamps the derived value into `metadata[key]` (traceability — it persists into a sink); `Capture`/`Apply` move values through the per-sample Context, which never touches `sample.metadata`. +`ConfigureOp` also records the derived value as an `aux`-role field named `key` (traceability — it persists into a sink); `Capture`/`Apply` move values through the per-sample Context, which never alters the sample's fields. diff --git a/docs/graph.md b/docs/graph.md index ce6a0ba..02bbbcf 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -41,7 +41,7 @@ flow2 = from_ops(ops) # flat ops -> flow (lif ## Graph pipelines on a flat op list (Context ops) -A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-sample **`Context`** (a named-cell store, `sampleflux.context`) around each sample's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never touches `sample.metadata` — the metadata bus stays byte-identical to a linear run. +A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-sample **`Context`** (a named-cell store, `sampleflux.context`) around each sample's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never mutates the sample's fields — a linear run's fields stay byte-identical whether or not context threading exists. | Op | Semantics | |---|---| @@ -74,9 +74,9 @@ with activate(Context()): sample = op(sample) ``` -Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's `input`, `Mix` reads each cell's corresponding field. Copy discipline mirrors the stash family: stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-sample store instead of `sample.metadata` (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#the-per-sample-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). +Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's primary input item, `Mix` reads each cell's corresponding field. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-sample store instead of extra fields on the sample (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#the-per-sample-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). -> **What about the stash family?** `sampleflux.ops.stash` (`StashInputOp`/`UnstashInputOp`/`StashTargetOp`/`UnstashTargetOp`) snapshots a field into `sample.metadata` instead of a cell. It is NOT a wiring mechanism — the context ops are — and remains only for the two jobs cells cannot do: carrying a snapshot **across a `Parallel` boundary** (metadata rides the sample through the stream split; cells deliberately raise there) and deliberately **persisting a snapshot into a sink**. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. +> **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the field into its own `aux`-role field with the structure ops (`CopyField` + `SetRole`, `sampleflux.ops.structure`); the snapshot then rides the sample as a real field. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. ## Reattach an ops-only YAML (`Flux.from_ops_yaml`) diff --git a/docs/image.md b/docs/image.md index 41ab683..7cc1ba3 100644 --- a/docs/image.md +++ b/docs/image.md @@ -5,13 +5,13 @@ The single, modality-agnostic "any value → image" layer — generic so every c ```python from sampleflux.ops.image import ConvertToImageOp, value_to_image -# Op: sample.input (2-D map / CHW tensor / PIL / bool mask) -> PIL image. +# Op: the sample's primary input item (2-D map / CHW tensor / PIL / bool mask) -> an Image field. op = ConvertToImageOp( colormap="viridis", # closed `Colormap` Literal -> enumerable in GUIs / schemas width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top ) -sample = op(sample) # also publishes image_width_px / image_height_px to metadata +sample = op(sample) # writes an Image field; the pixel dimensions live in its array shape # Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): rgb = value_to_image(some_value, colormap="magma", max_size=512) diff --git a/docs/kinds.md b/docs/kinds.md index 14d2525..e80e4d9 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -1,88 +1,68 @@ -# The transform taxonomy, multi-type carriers & expanding ops (`sampleflux.kinds`) +# Transforms, batching & expanding ops (`sampleflux.bag` / `sampleflux.collate`) -## What an op processes, how it's called +## What a transform processes — dispatch on item type -A **sample** is the triple `(input, target, metadata)`; the classic AI tuple is the **pair** `(input, target)`. A transform declares — via its `__call__` signature alone — exactly which *slice* of the triple it processes, and the engine binds that view and merges the result back (untouched fields preserved): - -| scope | without metadata | with metadata | -|---|---|---| -| input only | `input` — the bare value | `input_meta` — `InputMeta(input, metadata)` | -| target only | `target` — the bare value | `target_meta` — `TargetMeta(target, metadata)` | -| both | `pair` — `(input, target)` / `Pair` | `sample` — the full `Sample` | -| metadata only | — | `metadata` — the bare dict (`m: dict` / `MetaDict`) | - -Each scope works in **two calling styles** — packed (one argument) or unpacked (the fields as separate arguments) — and unpacked arguments COMBINE freely: each parameter binds its own view (annotation first, then the name, then the classic `f(input, target, metadata)` positional defaults): +A **sample** is a named bag of typed items (`Image`, `Mask`, `Regions`, `Label`, … — see [typed-model.md](typed-model.md)). A transform declares which item TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per sample, then applies the matching kernel to every field whose item type it handles, passing untouched fields through: ```python -class A: # bare input value — any array/tensor/dict; target+metadata pass through - def __call__(self, x: Input): return x / 255.0 # Annotated[T, INPUT] keeps a real T - -class B: # the classic AI signature, unpacked - def __call__(self, input, target): return aug(input), target - -class C: # input with its metadata, unpacked (2nd arg named `metadata`/`meta`) - def __call__(self, input, metadata): return crop(input, metadata["roi"]), metadata - -class D: # packed named view - def __call__(self, v: TargetMeta) -> TargetMeta: return TargetMeta(encode(v.target), v.metadata) - -class E: # the full triple, unpacked - def __call__(self, input, target, metadata): return input, target, {**metadata, "seen": True} +from sampleflux import Transform, Image -class F: # today's classic — completely unchanged - def __call__(self, sample: Sample) -> Sample: ... +class Recenter(Transform): + handles = (Image,) # which item types this transform touches -class G: # COMBINED views: input WITH its metadata + target WITH its metadata - def __call__(self, im: InputMeta, tm: TargetMeta): - return InputMeta(aug(im.input), im.metadata), TargetMeta(remap(tm.target), tm.metadata) + def params(self): # sampled ONCE per sample, shared across fields + return {"mean": 0.5} -class H: # metadata-only transform - def __call__(self, m: dict) -> dict: return {**m, "canonical": True} +@Recenter.kernel(Image) # per-type behaviour +def _(item, params): + return item - params["mean"] ``` -Detection rules: arity counts **required** parameters (optional extras don't change anything); 3 args → unpacked `sample`; 2 args → `input_meta`/`target_meta` when the 2nd is named `metadata`/`meta` (or annotated `dict`), first-arg name `target` selects the target side, else the `pair`; 1 arg → the annotation (`Sample`, `tuple`/`Pair`, `InputMeta`/`TargetMeta`, `Input`/`Target` marks; untyped = **any** — exactly today's behavior). `op_contract(op)` exposes the result — `OpContract(accepts, produces, expands, style, bindings)`, where `bindings` lists each unpacked parameter's scope in order (e.g. `("input_meta", "target_meta")`) and `accepts` is the grid summary of the covered fields — the vocabulary a visual editor can surface as socket types. Escape hatches: `SAMPLE_KIND_IN`/`SAMPLE_KIND_OUT`/`CALL_STYLE`/`EXPANDS` class attrs. +Because the parameters are sampled once and shared, a transform that handles several types moves those fields **consistently** — one flip decision applies to `Image`, `Mask` and `Regions` together, the thing a flat `(input, target, metadata)` triple could not express. Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them. -Merge-back: `None` drops the sample; a returned `Sample` takes over; otherwise only the declared fields update (a `pair` op keeps metadata; an `input` op keeps target+metadata; the meta variants receive the *actual* metadata dict, so in-place mutation propagates). The views are real NamedTuples (`Pair`/`InputMeta`/`TargetMeta`), recognized by `Sample.from_any`/`classify_carrier` *before* the generic tuple rule, flow natively under `Flux(native=True)`, and have default collates. +Two smaller shapes round it out: -## Multi-type carriers & the collate registry (`sampleflux.collate`) +- **A plain function** becomes a transform via `as_transform(fn, handles=(Image,), only=["image"])` — `only=` narrows a transform to specific field keys. +- **A type-changing transform** — read one field, write a differently-typed item (`array → Image`, `Signal → Spectrogram`, `Mask → Regions`) — subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. -Pipelines can carry more than `Sample` triplets: **`Flux(native=True)`** (opt-in) keeps each carrier's own kind — a metadata-free **pair** (`(image, label)`, `(tensor, mask)`, `(tensor, coco_dict)`) or a bare **value** — and adapts every op via its introspected contract: +Bare library transforms (torchvision `transforms.v2` dispatching by type, albumentations by keyword name) drop straight into a `Pipeline` through registered adapters — each one hits only the field(s) it handles. See [typed-model.md](typed-model.md#mixing-libraries--one-pipeline-many-worlds). ```python -from confluid import configurable -from sampleflux import Flux, Sample, op_contract +from sampleflux import Sample, Image, Mask, Regions, Label, Pipeline +from torchvision.transforms import v2 +import albumentations as A + +out = Pipeline([ + v2.RandomHorizontalFlip(p=1.0), # Image + Mask + Regions together (one library draw) + v2.Normalize(mean, std), # Image only — wrapped by a registered adapter + A.GaussNoise(p=1.0), # Image only — wrapped by a registered adapter +])(sample) +# a Label field is untouched (no kernel handles it); roles are preserved. +``` -@configurable -class NormalizePair: # a pair-native op — no metadata anywhere - def __call__(self, pair: tuple) -> tuple: - img, label = pair - return img / 255.0, label +## Batching — `typed_collate` & the collate registry (`sampleflux.collate`) -@configurable -class StampOp: # a classic Sample op — unchanged - def __call__(self, sample: Sample) -> Sample: ... +Transforms are per-sample; batching is a separate stage. **`typed_collate`** (auto-dispatched for `Sample` batches) stacks each field's payload and collects each item's per-sample attributes into a list, preserving roles — the ONE batch convention: -flux = Flux(source=[(img_a, 3), (img_b, 7)], ops=[NormalizePair(), StampOp()], native=True) -# NormalizePair receives the raw pair; StampOp receives a PROMOTED Sample view -# (promotion is one-way and sticky, so op-written metadata is never dropped). +```python +from sampleflux import typed_collate +from torch.utils.data import DataLoader -op_contract(NormalizePair()) # OpContract(accepts='pair', produces='pair', expands=False) +batch = typed_collate(list(flux)) # a batched Sample: payloads stacked per field +loader = DataLoader(flux, collate_fn=typed_collate) ``` -Detection reads the `__call__` annotations (`Sample` → sample-op, `tuple[...]` → pair-op, untyped → works-on-anything — **untyped ops behave exactly as today**); the class attrs `SAMPLE_KIND_IN` / `SAMPLE_KIND_OUT` / `EXPANDS` override detection where introspection can't see. `native=False` (the default) coerces everything to `Sample` exactly as before — no consumer changes. - -**Collation** is a pluggable registry keyed by representation: +Collation is a pluggable registry keyed by name, so a task can register its own convention additively: ```python -from sampleflux import collate, get_collate, register_collate +from sampleflux import register_collate, get_collate -batch = collate(list(flux)) # dispatches on the detected kind -@register_collate("yolo") # task aliases are additive +@register_collate("yolo") # task aliases are additive def yolo_collate(items): ... loader = DataLoader(flux, collate_fn=get_collate("yolo")) ``` -Defaults: `"sample"` (stacked input/target + list-form batched metadata — the `is_batched` convention), `"pair"` (`(stacked_inputs, stacked_targets)`), `"value"`, and the view forms `"input_meta"`/`"target_meta"`. Consumer collates (classification/segmentation/detection) register additively and keep their own conventions. The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path; the full rationale is recorded in [architecture.md](architecture.md#batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17). +The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17). ## 1→N expanding ops (iterable-only pipelines) @@ -90,12 +70,14 @@ An op may return **several** carriers — a windowing op splitting one capture i ```python from typing import Iterator +from sampleflux import Sample, Transform, primary, with_data @configurable -class SlidingWindowOp: +class SlidingWindowOp(Transform): def __call__(self, sample: Sample) -> Iterator[Sample]: - for w in sliding_windows(sample.input, self.size, self.stride): - yield sample._replace(input=w) + key, item = primary(sample, "input") + for w in sliding_windows(item, self.size, self.stride): + yield sample.replace_field(key, with_data(item, w)) ``` Expansion is detected from the return annotation (`Iterator[...]` / `Iterable[...]` / `List[...]`; or the explicit `EXPANDS = True` marker) and flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. diff --git a/docs/projection.md b/docs/projection.md index d9f9001..b2bc582 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -12,7 +12,7 @@ from sampleflux import ProjectionField # Literal["input", "target", "metadata"] # unrequested fields — e.g. an image dataset reads only the label column for a # target-only walk, never decoding an image. for sample in project(my_source, ("target",)): - ... # sample.input is None; sample.target populated + ... # only target-role fields are built; input-role fields are skipped labels = list(iter_targets(my_source)) # lazy n = num_classes(my_source) # max(class_id) + 1 — always walks diff --git a/docs/storage.md b/docs/storage.md index a03677c..8328e2f 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -15,35 +15,37 @@ Flux.from_source(HDF5Source("input.h5")) \ ## Sinks and their matching sources -Every sink has a source that reads its layout back into `Sample` triplets: +Every sink has a source that reads its layout back into typed `Sample` bags: | Backend | Sink | Source | Round-trips | |---|---|---|---| -| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | input + target + metadata | -| Zarr group (one group / sample) | `ZarrGroupSink` | `ZarrGroupSource` | input + target + metadata | -| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | input only (uniform shape) | -| Directory (one dir / sample) | `DirectorySink` | — | — | +| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | all fields + roles | +| Zarr group (one group / sample) | `ZarrGroupSink` | `ZarrGroupSource` | all fields + roles | +| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | primary input field only (uniform shape) | +| Directory (one dir / sample) | `DirectorySink` | `DirectorySource` | all fields + roles | ```python from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource Flux(samples).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) -for sample in ZarrGroupSource("ds.zarr"): # input/target as before, metadata from .zattrs +for sample in ZarrGroupSource("ds.zarr"): # exact fields, roles and item attrs reconstructed ... ``` > Domain-specific storage formats implement the same `DataSink`/`DataSource` protocols in their own package — e.g. the SigMF waveform-recording pair (`SigMFSink`/`SigMFSource`) lives in `waivefront.sigmf`, not here. The engine never couples to a specific format. -## Array-valued metadata (e.g. segmentation masks) +## Array-valued item attributes -`HDF5Sink` stores scalar/string metadata as HDF5 **attributes**, but HDF5 caps attribute size — a large array (a segmentation mask, a per-sample weight map) put in `Sample.metadata` would overflow that limit. So **array-valued metadata (`np.ndarray` / `torch.Tensor`) is written as its own dataset** under a per-sample group `{prefix}_meta/`, and `HDF5Source` merges it back into `Sample.metadata` on read. This is fully backward-compatible: files written before this layout (no `{prefix}_meta` group) read exactly as before. +Each field is stored as its own group: the item's payload as a `data` dataset and its scalar attributes as HDF5 **attributes**. HDF5 caps attribute size, so any **array-valued attribute** (`np.ndarray` / `torch.Tensor`, e.g. a per-sample weight map) is written as its own sub-dataset under `attrs/` instead — a large array never overflows the attribute limit, and `HDF5Source` restores every attribute on read. A segmentation mask is not an attribute at all: it is a first-class `Mask` field with its own payload. ```python -sample = Sample(input=data, target=label, metadata={"mask": mask_2d, "snr": 12.0}) +from sampleflux import Sample, Image, Mask, item_data + +sample = Sample({"image": Image(data), "mask": Mask(mask_2d)}, roles={"mask": "target"}) Flux([sample]).to_sink(HDF5Sink("ds.h5", overwrite=True)) loaded = next(iter(HDF5Source("ds.h5"))) -loaded.metadata["mask"] # the full array, byte-exact (not a truncated repr) -loaded.metadata["snr"] # scalar, via attributes as before +item_data(loaded["mask"]) # the full mask array, byte-exact (not a truncated repr) +loaded.role_of("mask") # "target" — roles round-trip too ``` ## Queryable metadata (`sampleflux.storage.query`) diff --git a/docs/typed-model.md b/docs/typed-model.md index a33375a..7173af2 100644 --- a/docs/typed-model.md +++ b/docs/typed-model.md @@ -1,18 +1,15 @@ # The typed-bag model — THE sampleflux data model -> **Status: the data model (migration in progress).** The typed bag replaces the classic -> `Sample(input, target, metadata)` triple; the legacy engine survives only until every consumer -> has migrated (staged in the root `TASKS.md`), after which it is deleted and `TypedSample` is -> renamed `Sample`. Import the typed surface from the PACKAGE TOP LEVEL -> (`from sampleflux import TypedSample, Image, Transform, ...`) — internal module paths are -> transitional. The design rationale is recorded in -> [architecture.md](architecture.md#the-typed-bag-model-a-named-bag-of-typed-items-sampleflux-bag-poc-2026-07-21). +Import the typed surface from the PACKAGE TOP LEVEL +(`from sampleflux import Sample, Image, Mask, Regions, Label, Transform, Pipeline, primary, item_data, typed_collate, register_item, register_kernel, register_adapter, register_io, ...`). +The design rationale is recorded in +[architecture.md](architecture.md#the-typed-bag-model-a-named-bag-of-typed-items-sampleflux-bag-2026-07-21). ## Why -In the classic model everything that is not literally the model input or target — a segmentation -mask, `[f0,f1,t0,t1]` regions, a signal's samplerate, an image's canvas size, a label's class -names — is jammed into one flat `metadata` dict keyed by string, disconnected from the value it +If everything that is not literally the model input or target — a segmentation mask, +`[f0,f1,t0,t1]` regions, a signal's samplerate, an image's canvas size, a label's class names — is +jammed into one flat `metadata` dict keyed by string, it is disconnected from the value it describes. That makes two things hard: metadata has no natural home, and a transform cannot move several fields together consistently (flip an image → flip its mask → flip its boxes). @@ -46,12 +43,12 @@ item_data(Image(arr)) # -> the plain ndarray with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved ``` -### `TypedSample` — a named bag with role tags +### `Sample` — a named bag with role tags ```python -from sampleflux import TypedSample +from sampleflux import Sample -sample = TypedSample( +sample = Sample( {"image": Image(rgb), "regions": Regions(boxes), "class": Label("drone_x")}, roles={"regions": "target", "class": "target"}, # default role is "input" ) @@ -61,7 +58,7 @@ sample.set_role("regions", "aux") # copy-on-write; a field's role changes with ``` `input` / `target` / `aux` / `pred` are **tags read at the train/collate/sink boundary**, not tuple -positions. `TypedSample` is immutable — every mutator returns a new sample. +positions. `Sample` is immutable — every mutator returns a new sample. ### Transforms — type dispatch with once-per-sample parameters @@ -92,7 +89,7 @@ Pipeline([ ])(sample) ``` -The coercion is a small **registry** (`sampleflux.bag.register_adapter` / `coerce_transform`): the +The coercion is a small **registry** (`register_adapter` / `coerce_transform`, both top-level): the built-in torchvision-v2 and albumentations adapters register a matcher (by MRO module name, no eager import) at package load. Teach a `Pipeline` about your own library's transforms with one call: @@ -143,7 +140,7 @@ drops into a `sampleflux.bag.Pipeline` alongside the generic transforms with no ## Engines — Flux and FlowGraph carry the typed bag -A `TypedSample` is **never coerced**: on every `Flux` route (sequential / parallel / streamed / +A `Sample` is **never coerced**: on every `Flux` route (sequential / parallel / streamed / `__getitem__`) and in `FlowGraph`, a typed source item passes through verbatim and each op receives the whole bag (`Pipeline` transforms, structure ops, and the compose plane — `TransformChain`, `RandomApply`, `Enable`, `Apply`, `Capture` — all route typed carriers correctly). @@ -154,10 +151,9 @@ Flux(source=typed_source, ops=[v2.RandomHorizontalFlip(p=0.5), Fourier()]).to_si ### Typed fan-in (`merge_from`) and field binds (`step[key]`) -In a `flow:` document, the typed fan-in is **`merge_from`** — the UNION of the named steps' fields -and roles, in slot order, last-write-wins on a key collision (the typed replacement for the legacy -metadata dict-merge). The idiom for a derived-field branch: produce, `SelectFields` the new -field(s), merge: +In a `flow:` document, the fan-in is **`merge_from`** — the UNION of the named steps' fields +and roles, in slot order, last-write-wins on a key collision. The idiom for a derived-field branch: +produce, `SelectFields` the new field(s), merge: ```yaml flow: @@ -171,29 +167,26 @@ flow: `bind:` references gain a field form: `step[key]` binds the named ITEM of that step's bag as an op parameter; a bare `step` reference binds the step's PRIMARY input-role item (`sampleflux.primary`). Lowering (`to_ops`) compiles `merge_from` to the `MergeFields` context op -and key-binds to `Apply(key=...)`; lifting (`from_ops`) round-trips both. `target_from` / -`metadata_from` stay legacy-`Sample`-only (a typed step using them raises; `merge_from` on a legacy -carrier likewise). +and key-binds to `Apply(key=...)`; lifting (`from_ops`) round-trips both. ## Storage — the typed field-group layout All three backends (`HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, -`DirectorySink`↔`DirectorySource`) write a `TypedSample` in ONE logical schema: per sample, one +`DirectorySink`↔`DirectorySource`) write a `Sample` in ONE logical schema: per sample, one group per FIELD carrying the item's registered type name, its role, the payload as a dataset, and its attrs (scalars natively — queryable; arrays as sub-datasets; structured values JSON-tagged so -tuples survive). The store is stamped `sampleflux_format = "typedsample-v1"`; a store holds ONE -carrier — appending a legacy `Sample` to a typed store (or vice versa) raises. Backends never -inspect item internals — everything serializes through the `sampleflux.bag.io` codec -(`encode_item`/`decode_item`), so an externally-registered item type round-trips with zero storage +tuples survive). The store is stamped `sampleflux_format = "typedsample-v1"`. Backends never +inspect item internals — everything serializes through the item codec (`encode_item` / `decode_item`, +top-level), so an externally-registered item type round-trips with zero storage edits; `register_io(MyItem, encode=..., decode=...)` overrides the default structural codec when needed. ```python sink = HDF5Sink(path="out.h5", overwrite=True) with sink: - for sample in flux: # TypedSamples + for sample in flux: # Samples sink.write(sample) -back = list(HDF5Source(path="out.h5")) # exact TypedSamples: fields, roles, order, tuple attrs +back = list(HDF5Source(path="out.h5")) # exact Samples: fields, roles, order, tuple attrs ``` `ZarrBatchSink` (the uniform single-array sink) appends the PRIMARY input field's payload per row @@ -212,18 +205,19 @@ Array-valued attrs appear as shape/dtype stubs (presence/shape testable, never l named like a Python keyword (e.g. `class`) can't be addressed in an expression — use the programmatic `predicate` or a non-keyword field name. -## Interop with the classic `Sample` +## Interop with plain `(input, target, metadata)` tuples -Run a typed pipeline against the existing sources/sinks by bridging both ways. The lowering is -lossless — the whole bag is encoded in the legacy metadata while `input` / `target` still expose the -primary payloads for a legacy consumer: +Bridge to and from a plain 3-tuple — for an external consumer that expects one, or when adopting a +non-typed dataset. The bridge is lossless: `to_legacy` flattens the bag into an +`(input, target, metadata)` tuple (the whole bag encoded in the metadata, while `input` / `target` +still expose the primary payloads), and `to_typed` reconstructs the exact bag: ```python -from sampleflux.bag.interop import to_legacy, to_typed -legacy = to_legacy(sample) # a classic Sample; to_typed(legacy) == sample -typed = to_typed(legacy) # exact reconstruction -# adopting an arbitrary legacy dataset needs a per-dataset builder: -to_typed(raw_sample, builder=lambda s: TypedSample({"image": Image(s.input), "class": Label(s.target)})) +from sampleflux import to_legacy, to_typed +plain = to_legacy(sample) # (input, target, metadata); to_typed(plain) == sample +typed = to_typed(plain) # exact reconstruction +# adopting an arbitrary external dataset needs a per-dataset builder: +to_typed(record, builder=lambda r: Sample({"image": Image(r.image), "class": Label(r.label)})) ``` ## What is NOT here yet (follow-ups) diff --git a/docs/typespec.md b/docs/typespec.md deleted file mode 100644 index 58cbbf2..0000000 --- a/docs/typespec.md +++ /dev/null @@ -1,37 +0,0 @@ -# Type specs — `ACCEPTS` / `PRODUCES` (`sampleflux.typespec`) - -`sampleflux.typespec` describes *what flows through a `Sample`* and lets ops declare what they accept/produce, so tools (connection validators, visual config editors, schema generators) can filter which ops may connect. It is flexible by design — N-dimensional arrays across numpy/torch/tensorflow, **per-axis bounded ranges**, dtype families, images, and arbitrary Python types — and anything left unspecified defaults to `Any`. - -```python -from sampleflux.typespec import SampleType, ArrayType, Dim, PythonType, UnionType - -# "a 2-D float array whose first axis is 1–10, second axis any size" -ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="floating") -ArrayType.parse("3 h w", dtype="float32", framework="torch") # jaxtyping-style shorthand -ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") # an image convenience -``` - -`dtype`, `framework`/`frameworks`, and the image `layout` are **closed `Literal`s**, not bare strings — a typo is a type error and a UI / connection-validator enumerates the choices via `typing.get_args(...)`: - -- `Dtype` — concrete names (`"float32"`, `"int64"`, …); `DtypeFamily` — relaxed families (`"floating"`, `"numeric"`, …); `DtypeSpec = Dtype | DtypeFamily` is the `dtype` field type. -- `Framework = Literal["numpy", "torch", "tensorflow"]`, `ImageLayout = Literal["CHW", "HWC"]`. - -Authored dtypes must be canonical names; aliases / casing (`"double"`, `"FLOAT32"`) and exotic platform dtypes (`float128`) are runtime-only conveniences normalized by `canonical_dtype` — the single boundary where arbitrary input crosses into the typed domain. - -## Declaring an op's contract - -Use the class attributes `ACCEPTS` / `PRODUCES` (each a `SampleType`; both default to `Any`, so annotating is optional and backward-compatible). No base class — transforms stay plain callables: - -```python -@configurable -class StandardizeOp: - ACCEPTS = SampleType(input=UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image")))) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - def __call__(self, sample): ... -``` - -Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime against a concrete inferred type); `compatible(consumer, producer)` is permissive (used at edit time — `Any`/unknown on either side passes). - -## A Sample's own type - -A `Sample`'s type comes from `sample.describe()` — it returns a type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict) + `__spec__` (sidecar refinements), or infers one from the live data; attach a stored type with `sample.with_type(SampleType(...))`. Default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. diff --git a/sampleflux/context.py b/sampleflux/context.py index ee77def..d519642 100644 --- a/sampleflux/context.py +++ b/sampleflux/context.py @@ -7,11 +7,11 @@ ``Mix``) move data between the linear sample stream and these cells, which is what lets a plain sequential op list execute a fan-out/fan-in graph. -Deliberately NOT ``sample.metadata``: metadata rides *inside* each sample and is the -shared accumulating bus ops hand values to each other on. The Context is the *wiring* -plane — engine-created, per sample, empty again by the end of a well-formed graph (every -cell freed after its last read). Nothing here is ``@configurable``; a Context never -appears in YAML. +The context ops route graph data through these per-sample Context CELLS and never touch +the sample's own fields — each typed item still owns its own metadata inside the sample. +The Context is the *wiring* plane — engine-created, per sample, empty again by the end of +a well-formed graph (every cell freed after its last read). Nothing here is +``@configurable``; a Context never appears in YAML. The engine (``Flux`` — and ``FlowGraph``, which manages its env directly) creates one Context per source item and activates it around the op loop via a @@ -35,8 +35,7 @@ class Context: """Named-cell store for one sample's trip through a graph-shaped pipeline. Cells are stored and returned **by reference** — copy semantics are the reading - op's decision (``Use`` deep-copies unless it drops the cell), mirroring the stash - family's copy-on-restore convention. + op's decision (``Use`` deep-copies unless it drops the cell). """ __slots__ = ("_cells",) diff --git a/sampleflux/discovery.py b/sampleflux/discovery.py index c1f31d3..9634e13 100644 --- a/sampleflux/discovery.py +++ b/sampleflux/discovery.py @@ -10,10 +10,10 @@ :func:`resolve_callable` imports it back. This is how a pipeline step is referenced in a Confluid manifest and resurrected later for reproducibility. * **Discovery** (callable -> JSON schema): :func:`introspect_callable` reflects - a single callable into a schema (signature + docstring + the ``ACCEPTS`` / - ``PRODUCES`` typespec contract), and :func:`scan_module` does the same for - every callable *defined in* a module — a visual editor's node bridge reads - these to auto-generate canvas nodes and their property panels. + a single callable into a schema (signature + docstring), and + :func:`scan_module` does the same for every callable *defined in* a module — a + visual editor's node bridge reads these to auto-generate canvas nodes and + their property panels. The serialization half doubles as the workspace's generic string-callable hook pattern (:class:`~sampleflux.core.WrappedOp` stores its ``f`` this way; consuming @@ -30,7 +30,7 @@ import os import sys from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union, cast +from typing import Any, Callable, Dict, List, Union, cast def get_callable_path(func: Callable) -> str: @@ -113,8 +113,7 @@ def introspect_callable(func: Callable) -> Dict[str, Any]: Build a JSON-serializable schema for a callable by reflecting over its signature: ``path``, ``name``, ``doc``, per-parameter info (``name`` / ``type`` / ``default`` / ``required``, skipping ``self`` / ``cls`` / - ``*args`` / ``**kwargs``), plus the declared ``ACCEPTS`` / ``PRODUCES`` - typespec contract when present. + ``*args`` / ``**kwargs``). Use: a visual editor reads this to render a node and its property-panel widgets. """ @@ -141,18 +140,9 @@ def introspect_callable(func: Callable) -> Dict[str, Any]: "name": getattr(func, "__name__", str(func)), "doc": func.__doc__.strip() if func.__doc__ else "", "parameters": params, - "accepts": _spec_dict(getattr(func, "ACCEPTS", None)), - "produces": _spec_dict(getattr(func, "PRODUCES", None)), } -def _spec_dict(spec: Any) -> Optional[Dict[str, Any]]: - """JSON-serialize a declared ``ACCEPTS`` / ``PRODUCES`` (a :class:`~sampleflux.typespec.SampleType`), - or ``None`` when undeclared. Duck-typed so ``discovery`` needn't import ``typespec``.""" - to_dict = getattr(spec, "to_dict", None) - return cast(Dict[str, Any], to_dict()) if callable(to_dict) else None - - def scan_module(path_or_name: Union[str, Path]) -> List[Dict[str, Any]]: """ Scan a module (by import name) or a ``.py`` script (by path) and return an diff --git a/sampleflux/ops/_augment_bridge.py b/sampleflux/ops/_augment_bridge.py index eb7691c..6cc0a5e 100644 --- a/sampleflux/ops/_augment_bridge.py +++ b/sampleflux/ops/_augment_bridge.py @@ -117,7 +117,7 @@ def __init__(self: Any, **kwargs: Any) -> None: # to_pydantic's get_type_hints evals them against THIS module. Anything that still won't # resolve degrades to Any so introspection never chokes on a stray name. try: - _hints = get_type_hints(transform_cls.__init__) + _hints = get_type_hints(transform_cls.__init__) # type: ignore[misc] except Exception: _hints = {} diff --git a/sampleflux/ops/albumentations.py b/sampleflux/ops/albumentations.py index d219147..2971c5d 100644 --- a/sampleflux/ops/albumentations.py +++ b/sampleflux/ops/albumentations.py @@ -1,8 +1,8 @@ """``AlbumentationsOp`` — run `albumentations `_ transforms as a SampleFlux op. -One random draw is applied jointly to ``sample.input`` and (per the ``target`` mode) its -segmentation mask / detection boxes, so a geometric augmentation moves image AND target -consistently; metadata passes through untouched. +One random draw is applied jointly to the primary input item and (per the ``target`` mode) +its segmentation mask / detection boxes, so a geometric augmentation moves image AND target +consistently; the aux fields pass through untouched. Transforms are authored **Confluid-natively** — nested ``!class:`` nodes, never albumentations' own ``to_dict`` format:: @@ -63,7 +63,7 @@ def _resolve_transform(entry: Any) -> Any: @configurable(category="op", group="augment", random=True) class AlbumentationsOp: - """Apply albumentations transforms to ``sample.input`` (and optionally the target). + """Apply albumentations transforms to the primary input item (and optionally the target). Pass EITHER ``transform`` (one transform, or a prebuilt ``A.Compose``) OR ``transforms`` (a list composed into an ``A.Compose`` lazily) — never both. Entries @@ -75,9 +75,9 @@ class AlbumentationsOp: * ``"none"`` — input-only augmentation (color jitter, noise, blur); the sample's target passes through untouched. - * ``"mask"`` — ``sample.target`` is a segmentation mask (2-D array or PIL ``L`` + * ``"mask"`` — the target-role item is a segmentation mask (2-D array or PIL ``L`` image); image and mask receive the SAME spatial transform. - * ``"boxes"`` — ``sample.target`` is the torchvision detection dict + * ``"boxes"`` — the target-role item is the torchvision detection dict ``{"boxes": [N,4] xyxy-pixel, "labels": [N]}`` (what :class:`~sampleflux.ops.target.CocoToTorchVisionDetectionOp` / :class:`~sampleflux.ops.target.MasksToDetectionBoxesOp` emit). When the op builds diff --git a/sampleflux/ops/configure.py b/sampleflux/ops/configure.py index 0647ed3..af30942 100644 --- a/sampleflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -27,7 +27,7 @@ class ConfigureOp: The ``ops`` chain runs on the incoming sample as a SIDE branch — its input/target transformations are discarded (the original sample continues), while metadata writes - survive (the shared metadata-bus convention). The final ``sample.input`` of that chain + survive (the shared metadata-bus convention). The chain's final primary input item becomes the VALUE: it is written to ``metadata[key]`` and set as the ``param`` attribute of ``target``, then ``target`` is applied to the original sample. @@ -47,7 +47,7 @@ class ConfigureOp: param: low_level Args: - ops: Value-computing op-chain; the chain's final ``sample.input`` is injected. Empty = the incoming input. + ops: Value-computing op-chain; the chain's final primary input item is injected. Empty = the incoming input. target: The op to configure and apply; required at call time, validated lazily. param: Target attribute name to set with the computed value (e.g. ``low_level``). key: Metadata key the value is also written to. Blank (default) = ``param``. diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py index f834c2c..3c59d78 100644 --- a/sampleflux/ops/context.py +++ b/sampleflux/ops/context.py @@ -10,7 +10,8 @@ Unlike the stash family these NEVER touch ``sample.metadata``: graph wiring lives on the engine-created Context data plane, so the metadata bus stays byte-identical to a linear run. Cells are stored by reference (ops are copy-on-write by convention); ``Use`` copies -on read unless it drops the cell — mirroring ``UnstashInputOp(copy=True, remove=True)``. +on read unless it drops the cell — the same copy-on-read / move-on-drop idiom as +``Apply(source=cell)``. """ from copy import deepcopy @@ -87,7 +88,7 @@ class Use: """Replace the stream sample with a Context cell's value (a branch start). The incoming sample is discarded; the cell's value becomes the stream sample - (``Sample.from_any`` coerces a raw cell value). Reads a DEEP COPY so two branches + (a raw, non-``Sample`` cell value is used verbatim). Reads a DEEP COPY so two branches reading one fork stay independent — unless ``drop`` frees the cell, which skips the copy (move semantics, the right choice for a cell's LAST reader). diff --git a/sampleflux/ops/formula.py b/sampleflux/ops/formula.py index accbb7f..cd06573 100644 --- a/sampleflux/ops/formula.py +++ b/sampleflux/ops/formula.py @@ -1,7 +1,7 @@ -"""``FormulaOp`` — evaluate a math formula over ``sample.input``. +"""``FormulaOp`` — evaluate a math formula over the primary input item. The op-form of a visual canvas *Math* node: a restricted Python expression over one -named variable bound to the incoming ``sample.input`` (plus the stdlib ``math`` namespace +named variable bound to the incoming primary input item (plus the stdlib ``math`` namespace and the scalar helpers ``abs``/``min``/``max``/``round``/``pow`` — no builtins, so ``__import__``/``open``/``exec`` are unavailable). Its main consumer is the ops-export's value-chain compilation: an on-canvas ``… → Extract → Math → widget`` wire becomes @@ -25,11 +25,11 @@ @configurable(category="op", group="compose") class FormulaOp: - """Replace ``sample.input`` with ``formula`` evaluated over it. + """Replace the primary input item with ``formula`` evaluated over it. Args: formula: Expression over ``var`` (e.g. ``"a * 0.2"``); ``math.*`` + ``abs``/``min``/``max``/``round`` allowed. - var: Variable name the incoming ``sample.input`` binds to. Defaults to ``a``. + var: Variable name the incoming primary input item binds to. Defaults to ``a``. """ def __init__(self, formula: str = "a", var: str = "a") -> None: diff --git a/sampleflux/ops/image.py b/sampleflux/ops/image.py index 3cab695..b96ea45 100644 --- a/sampleflux/ops/image.py +++ b/sampleflux/ops/image.py @@ -13,7 +13,7 @@ op produces, and ``RenderSignalPlotOp`` builds IQ time/freq/constellation panels. Those need signal semantics; this op does not. -PIL is a hard dependency here (already used by ``sampleflux.typespec``). Matplotlib +PIL is a hard dependency here (used directly for the image rendering). Matplotlib is imported lazily inside :func:`_apply_colormap` — only non-``"gray"`` colormaps need it, so the pure-greyscale path stays matplotlib-free. """ @@ -599,7 +599,7 @@ class ConvertToImage(Transform): """Typed twin of :class:`ConvertToImageOp` — an array-bearing field → an ``Image`` item. The typed-bag counterpart of :class:`ConvertToImageOp`: instead of rendering - ``sample.input`` into a PIL image in place, it reads an array-bearing field from a + the primary input item into a PIL image in place, it reads an array-bearing field from a :class:`~sampleflux.Sample` and writes a fresh :class:`~sampleflux.Image` item (HWC ``uint8`` RGB) under ``output``, tagged with the ``input`` role (it is the pipeline's working image). Any other field passes through untouched. diff --git a/sampleflux/ops/sink.py b/sampleflux/ops/sink.py index cce5f4b..5ff1ff0 100644 --- a/sampleflux/ops/sink.py +++ b/sampleflux/ops/sink.py @@ -26,7 +26,7 @@ class SampleSinkOp: attached to a :class:`marainer.processing.DatasetProcessor` as the flux's terminal sink. This adapter lets the same sinks slot into any Sample-based op chain — notably the ``ops`` list of - :class:`waivefront.sinks.DetectionPredictionsSink`, where the model's + :class:`waivefront.sinks.SigMFPredictionsSink`, where the model's predictions arrive as a Sample whose metadata carries the new ``predicted_regions`` and need to be persisted to disk just like a segment-pipeline output. @@ -35,7 +35,7 @@ class SampleSinkOp: subsequent call forwards the Sample to ``sink.write(sample)`` and returns the Sample unchanged. ``close()`` flushes (when present) and closes the underlying sink — propagated by :class:`sampleflux.ops.enable.Enable` and - :class:`waivefront.sinks.DetectionPredictionsSink` at end-of-run. + :class:`waivefront.sinks.SigMFPredictionsSink` at end-of-run. YAML:: diff --git a/sampleflux/ops/torchvision.py b/sampleflux/ops/torchvision.py index a8e22fb..f88675d 100644 --- a/sampleflux/ops/torchvision.py +++ b/sampleflux/ops/torchvision.py @@ -1,9 +1,9 @@ """``TorchvisionTransformOp`` — run torchvision ``transforms.v2`` transforms as a SampleFlux op. v2 transforms draw their random parameters ONCE per call and apply them to every -``tv_tensors`` carrier passed in, so a geometric augmentation moves ``sample.input`` AND -(per the ``target`` mode) its segmentation mask / detection boxes consistently; metadata -passes through untouched. +``tv_tensors`` carrier passed in, so a geometric augmentation moves the primary input item +AND (per the ``target`` mode) its segmentation mask / detection boxes consistently; the +aux fields pass through untouched. Transforms are authored **Confluid-natively** as nested ``!class:`` nodes:: @@ -53,7 +53,7 @@ def _import_v2() -> Any: @configurable(category="op", group="augment", random=True) class TorchvisionTransformOp: - """Apply torchvision ``transforms.v2`` transforms to ``sample.input`` (and optionally the target). + """Apply torchvision ``transforms.v2`` transforms to the primary input item (and optionally the target). Pass EITHER ``transform`` (one v2 transform, or a prebuilt ``v2.Compose``) OR ``transforms`` (a list composed into a ``v2.Compose`` lazily) — never both. Entries @@ -64,10 +64,10 @@ class TorchvisionTransformOp: Target modes (the ``target`` knob): * ``"none"`` — input-only augmentation; the sample's target passes through untouched. - * ``"mask"`` — ``sample.target`` is a segmentation mask (2-D array / tensor or PIL + * ``"mask"`` — the target-role item is a segmentation mask (2-D array / tensor or PIL ``L`` image), wrapped as ``tv_tensors.Mask`` so image and mask receive the SAME spatial transform. - * ``"boxes"`` — ``sample.target`` is the torchvision detection dict + * ``"boxes"`` — the target-role item is the torchvision detection dict ``{"boxes": [N,4] xyxy-pixel, "labels": [N]}`` (what :class:`~sampleflux.ops.target.CocoToTorchVisionDetectionOp` / :class:`~sampleflux.ops.target.MasksToDetectionBoxesOp` emit); boxes are wrapped as diff --git a/sampleflux/storage/hdf5.py b/sampleflux/storage/hdf5.py index 85e4805..1fbd830 100644 --- a/sampleflux/storage/hdf5.py +++ b/sampleflux/storage/hdf5.py @@ -99,7 +99,7 @@ def iter_metadata(self) -> "Iterator[tuple[str, dict]]": # category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @configurable(category="sink") class HDF5Sink(Storage, DataSink): - """High-performance HDF5 data sink focused on Sample triplets.""" + """High-performance HDF5 data sink focused on typed-bag ``Sample``s.""" def __init__( self, diff --git a/sampleflux/storage/zarr.py b/sampleflux/storage/zarr.py index b596b13..e350667 100644 --- a/sampleflux/storage/zarr.py +++ b/sampleflux/storage/zarr.py @@ -112,8 +112,8 @@ class ZarrGroupSource(Storage, DataSource): Args: path: Path to the Zarr group written by ZarrGroupSink. - sample_key: Name of the per-sample array holding ``Sample.input``. - target_key: Name of the per-sample array holding ``Sample.target`` (absent when the sample had no target). + sample_key: Name of the per-sample array holding the primary input item's payload. + target_key: Name of the per-sample array holding the target-role item's payload (absent when no target). """ def __init__( From 5f093ecfc20a77eb420ffa5daacd6189e75896f3 Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 24 Jul 2026 10:11:56 +0200 Subject: [PATCH 037/102] =?UTF-8?q?docs:=20*Op=E2=86=92*=20op-rename=20swe?= =?UTF-8?q?ep=20(ops=20dropped=20the=20Op=20suffix=20in=20the=20typed=20mi?= =?UTF-8?q?gration;=20Fourier/Window/GainControl/=E2=80=A6=20kept=20theirs?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 +- README.md | 2 +- docs/architecture.md | 8 ++++---- docs/augmentation.md | 4 ++-- docs/image.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a05dcbd..3f903e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the structure ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, `Parallel`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over the sample's primary input item (`primary(sample, "input")`) — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (the typed field-plumbing ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — re-tag a field's role, rename or copy a field key, drop a field, or narrow the bag to a chosen set of fields; these are how a derived-field branch is assembled and how a snapshot is carried across a `Parallel` boundary or persisted into a sink as its own `aux`-role field) / `compose` (`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImageOp` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlaysOp` / `RenderSignalPlotOp`), NOT here. +- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type IS the Item's Python Class, Never a Separate Field:** A field's type is its item's Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key. A consumer reads a field's type by `type(item)` and its shape/dtype/framework off the item's own payload and attrs. A transform that changes a value's type replaces the item under the same field key (e.g. `Signal` → `Spectrogram`, `array` → `Mask` → `Regions`). Never carry a parallel type descriptor beside the bag. diff --git a/README.md b/README.md index 1909b6c..4543270 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ for sample in flux: | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources, array-valued metadata, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | | [docs/projection.md](docs/projection.md) | Field projection (`SupportsProjection`), lazy target walks, `num_classes`, the fittable `LabelMap` | -| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImageOp`, `NormalizeToUint8Op`), array introspection helpers | +| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `NormalizeToUint8Op`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via albumentations / torchvision `transforms.v2` — joint input+target (mask/boxes) adapters, the generated `Alb*`/`Tv*` per-transform ops, seeding, Confluid-native YAML | | [docs/typed-model.md](docs/typed-model.md) | The typed-bag data model: a `Sample` is a named bag of typed items (each owning its metadata), type-dispatched transforms, torchvision/albumentations adapters, custom item types | diff --git a/docs/architecture.md b/docs/architecture.md index fb8056b..bd680ba 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -452,7 +452,7 @@ But a running detection/segmentation front-end needs a different shape: **read o field of a DIFFERENT type**. Turning a numeric array into a displayable image, thresholding an array into a boolean mask, and labelling that mask into a set of bin boxes are each a *type change* (`array → Image`, `array → Mask`, `Mask → Regions`), not an in-place per-type edit. No -library provides them, and the earlier ops that did (`ConvertToImageOp`, `ThresholdOp`, +library provides them, and the earlier ops that did (`ConvertToImage`, `ThresholdOp`, `ConnectedComponentsOp`) operated on a flat `(input, target, metadata)` triple, which the typed model does not carry. Without typed equivalents a `Sample` pipeline could not reach `Regions` from a raw array — the critical path for typed detection was blocked. @@ -549,7 +549,7 @@ sample["boxes"].boxes # [(row_min, row_max, col_min, col_max), ...] — the pin The typed detection twins above reach `Regions`; a typed CLASSIFICATION front-end needs the other two shapes: turn the working image into the model's **input tensor**, and turn the class-name label -into the encoded **target id**. The earlier ops that did this (`ToTensorOp`, `MetadataToTargetOp`, +into the encoded **target id**. The earlier ops that did this (`ToTensor`, `MetadataToTargetOp`, `EncodeTargetOp` / `DecodeTargetOp`) operated on a flat `(input, target, metadata)` triple. Two facts of the typed model shape the twins: (1) there is NO shared metadata dict — the label already rides a `Label` field that owns its metadata; (2) an array item is an `np.ndarray` SUBCLASS whose @@ -564,9 +564,9 @@ Add native typed twins subclassing `Transform` and overriding `__call__` (the sa detection twins), each reusing its legacy op VERBATIM on a shim `Sample` for byte-parity: - **`ToTensor`** (`ops/torch.py`, `group="torch"`) resolves an array-bearing field (explicit `field` - or the first array/PIL item), runs `ToTensorOp` (HWC→CHW + `normalize`), and writes an `Image` + or the first array/PIL item), runs `ToTensor` (HWC→CHW + `normalize`), and writes an `Image` with `layout="CHW"`. Because `NDArrayItem` coerces the payload, the stored value is a CHW `float32` - **numpy** array whose values equal `ToTensorOp(...).input.numpy()` — NOT a live tensor. By default + **numpy** array whose values equal `ToTensor(...).input.numpy()` — NOT a live tensor. By default it REPLACES the source field in place so the field's `input` role is preserved (`output` writes a new field tagged `input` instead). `typed_collate` stacks these payloads with `np.stack`; the numpy→tensor conversion is the collate / model boundary's job, exactly as for any numpy dataset. A diff --git a/docs/augmentation.md b/docs/augmentation.md index fcd7dcd..49d57a0 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -102,9 +102,9 @@ The two libraries disagree about layout, and the ops keep each library's native convention instead of hiding it: - **albumentations** (`AlbumentationsOp`, `Alb*`) consumes numpy **HWC** (PIL converts on - entry) and emits numpy HWC — put it BEFORE `ToTensorOp` in the chain. + entry) and emits numpy HWC — put it BEFORE `ToTensor` in the chain. - **torchvision** (`TorchvisionTransformOp`, `Tv*`) emits **CHW torch tensors** (numpy - HWC converts on entry, PIL passes through as PIL) — no `ToTensorOp` needed after it. + HWC converts on entry, PIL passes through as PIL) — no `ToTensor` needed after it. Don't chain one library's output straight into the other without accounting for this. diff --git a/docs/image.md b/docs/image.md index 7cc1ba3..a8a0a48 100644 --- a/docs/image.md +++ b/docs/image.md @@ -3,10 +3,10 @@ The single, modality-agnostic "any value → image" layer — generic so every consuming project (spectrogram previews, dataset browsers, GUI viewers) reuses one implementation. Domain-specific rendering (overlays, signal plots) stays in the consuming package. ```python -from sampleflux.ops.image import ConvertToImageOp, value_to_image +from sampleflux.ops.image import ConvertToImage, value_to_image # Op: the sample's primary input item (2-D map / CHW tensor / PIL / bool mask) -> an Image field. -op = ConvertToImageOp( +op = ConvertToImage( colormap="viridis", # closed `Colormap` Literal -> enumerable in GUIs / schemas width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top From 08f0655f42309ea36aa7f022b57b29b5a15c08f1 Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 24 Jul 2026 12:15:57 +0200 Subject: [PATCH 038/102] docs: point op docstrings at sampleflux.processing / sampleflux run (post-marainer-refactor) --- sampleflux/ops/enable.py | 10 +++++----- sampleflux/ops/sink.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sampleflux/ops/enable.py b/sampleflux/ops/enable.py index b194fd5..8ea5eb3 100644 --- a/sampleflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -47,9 +47,9 @@ class Enable: .. code-block:: bash - marainer process pipeline.yaml --visualize true - marainer process pipeline.yaml --visualize+ # polarity shorthand → True - marainer process pipeline.yaml --visualize- # polarity shorthand → False + sampleflux run pipeline.yaml --visualize true + sampleflux run pipeline.yaml --visualize+ # polarity shorthand → True + sampleflux run pipeline.yaml --visualize- # polarity shorthand → False Inner ops stay deferred (not materialized) until the wrapper actually fires for the first time, so guarding expensive-to-construct ops with @@ -78,10 +78,10 @@ class Enable: .. code-block:: bash # Targeted — only the overlay chain fires. - marainer process pipeline.yaml --overlay.visualize true + sampleflux run pipeline.yaml --overlay.visualize true # Broadcast — every Fluid with a `visualize` kwarg flips. - marainer process pipeline.yaml --visualize true + sampleflux run pipeline.yaml --visualize true ``name`` is a plain string on the instance; Confluid's post-construction paradigm setattr's it automatically from YAML with no ctor change. diff --git a/sampleflux/ops/sink.py b/sampleflux/ops/sink.py index 5ff1ff0..408dab2 100644 --- a/sampleflux/ops/sink.py +++ b/sampleflux/ops/sink.py @@ -23,7 +23,7 @@ class SampleSinkOp: Sinks (``JsonPerWindowSink``, ``JsonSink``, ``HDF5Sink`` …) implement the ``open()`` / ``write(sample)`` / ``close()`` protocol and are normally - attached to a :class:`marainer.processing.DatasetProcessor` as the + attached to a :class:`sampleflux.processing.DatasetProcessor` as the flux's terminal sink. This adapter lets the same sinks slot into any Sample-based op chain — notably the ``ops`` list of :class:`waivefront.sinks.SigMFPredictionsSink`, where the model's From 159f9809e141be40bb2cc819aac34224ed22b007 Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 24 Jul 2026 13:30:54 +0200 Subject: [PATCH 039/102] =?UTF-8?q?docs:=20NormalizeToUint8Op=20class=20wa?= =?UTF-8?q?s=20purged=20=E2=80=94=20document=20the=20surviving=20normalize?= =?UTF-8?q?=5Fto=5Fuint8=20free=20function=20as=20the=20single=20quantizat?= =?UTF-8?q?ion=20entry=20point?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3f903e6..d684ea2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the structure ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, `Parallel`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over the sample's primary input item (`primary(sample, "input")`) — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (the typed field-plumbing ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — re-tag a field's role, rename or copy a field key, drop a field, or narrow the bag to a chosen set of fields; these are how a derived-field branch is assembled and how a snapshot is carried across a `Parallel` boundary or persisted into a sink as its own `aux`-role field) / `compose` (`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and `NormalizeToUint8Op` (`category="op"`, `group="image"`: the standalone min-max value→`uint8` quantization step, decoupled from colormap/PIL; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is `NormalizeToUint8Op.normalize_to_uint8` (a `@staticmethod`) — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (there is no separate `_to_uint8` free function — the op's static method is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. +- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples — the standalone `NormalizeToUint8Op` op class was DELETED in the typed purge; only the function remains), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type IS the Item's Python Class, Never a Separate Field:** A field's type is its item's Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key. A consumer reads a field's type by `type(item)` and its shape/dtype/framework off the item's own payload and attrs. A transform that changes a value's type replaces the item under the same field key (e.g. `Signal` → `Spectrogram`, `array` → `Mask` → `Regions`). Never carry a parallel type descriptor beside the bag. diff --git a/README.md b/README.md index 4543270..63b27c8 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ for sample in flux: | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources, array-valued metadata, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | | [docs/projection.md](docs/projection.md) | Field projection (`SupportsProjection`), lazy target walks, `num_classes`, the fittable `LabelMap` | -| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `NormalizeToUint8Op`), array introspection helpers | +| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via albumentations / torchvision `transforms.v2` — joint input+target (mask/boxes) adapters, the generated `Alb*`/`Tv*` per-transform ops, seeding, Confluid-native YAML | | [docs/typed-model.md](docs/typed-model.md) | The typed-bag data model: a `Sample` is a named bag of typed items (each owning its metadata), type-dispatched transforms, torchvision/albumentations adapters, custom item types | From 88ed71fb119a10689d7e1282c595d484d97186df Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 25 Jul 2026 17:00:02 +0200 Subject: [PATCH 040/102] =?UTF-8?q?feat!:=20the=20record=20data=20model=20?= =?UTF-8?q?=E2=80=94=20plain-dict=20records,=20type-dispatched=20ops,=20li?= =?UTF-8?q?braries=20as-is?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sample is now a PLAIN dict of typed values (items.Record); the Sample container, roles, primary(), and the whole bag/ package are DELETED. Ops are type-dispatched Transforms (get_params once per record, kernels per value type, field= pins a key). core._apply_op is the op-FAMILY dispatch: bare albumentations ops get their own kwarg vocabulary (one joint draw, NDArrayItem outputs re-wrapped), bare torchvision-v2 ops get the dict as-is — the adapter plane (AlbumentationsOp, TorchvisionTransformOp, generated Alb*/Tv* families, coerce_transform/register_adapter) and TransformChain are DELETED; Pipeline is THE composer. ToTensor now emits a live CHW float torch.Tensor. Storage: typedrecord-v1 (no back-compat, typedsample-v1 raises), no __role__, plain values ride the value attr. Renames: typed_collate→collate_records (registry key 'record'), iter_inputs/iter_targets→iter_key, key-based projection, num_classes(key='class'); SetRole deleted; WrappedOp/Flux.map take key=. Suite rewritten (408 tests), docs rewritten (record-model.md, augmentation.md, architecture decision record), examples updated. --- AGENTS.md | 44 +- README.md | 88 ++- docs/architecture.md | 559 ++++++++---------- docs/augmentation.md | 203 +++---- docs/configure.md | 20 +- docs/graph.md | 47 +- docs/image.md | 19 +- docs/kinds.md | 75 +-- docs/projection.md | 41 +- docs/record-model.md | 425 +++++++++++++ docs/sources.md | 12 +- docs/storage.md | 67 ++- docs/typed-model.md | 227 ------- examples/dataset_split.yaml | 5 +- examples/record_pipeline.py | 114 ++++ examples/typed_pipeline.py | 81 --- pyproject.toml | 30 +- sampleflux/__init__.py | 72 +-- sampleflux/bag/__init__.py | 96 --- sampleflux/bag/adapters/__init__.py | 13 - sampleflux/bag/adapters/albumentations.py | 102 ---- sampleflux/bag/adapters/torchvision.py | 143 ----- sampleflux/bag/sample.py | 211 ------- sampleflux/bag/transform.py | 198 ------- sampleflux/collate.py | 65 +- sampleflux/context.py | 2 +- sampleflux/core.py | 197 +++--- sampleflux/{bag => }/dispatch.py | 20 +- sampleflux/flow.py | 51 +- sampleflux/{bag => }/io.py | 57 +- sampleflux/{bag => }/items.py | 40 +- sampleflux/labels.py | 8 +- sampleflux/ops/__init__.py | 21 +- sampleflux/ops/_augment_bridge.py | 222 ------- sampleflux/ops/albumentations.py | 182 ------ sampleflux/ops/albumentations_transforms.py | 74 --- sampleflux/ops/configure.py | 76 +-- sampleflux/ops/context.py | 124 ++-- sampleflux/ops/debug.py | 46 +- sampleflux/ops/enable.py | 29 +- sampleflux/ops/formula.py | 39 +- sampleflux/ops/image.py | 66 +-- sampleflux/ops/numpy.py | 67 +-- sampleflux/ops/parallel.py | 22 +- sampleflux/ops/random_apply.py | 30 +- sampleflux/ops/sink.py | 41 +- sampleflux/ops/structure.py | 103 ++-- sampleflux/ops/target.py | 146 +++-- sampleflux/ops/torch.py | 50 +- sampleflux/ops/torchvision.py | 198 ------- sampleflux/ops/torchvision_transforms.py | 80 --- sampleflux/ops/transform_chain.py | 88 --- sampleflux/projection.py | 125 ++-- sampleflux/sources.py | 141 ++--- sampleflux/storage/base.py | 72 ++- sampleflux/storage/directory.py | 99 ++-- sampleflux/storage/hdf5.py | 143 +++-- sampleflux/storage/query.py | 170 +++--- sampleflux/storage/zarr.py | 176 +++--- sampleflux/transform.py | 173 ++++++ tests/{_bag_fixtures.py => _fixtures.py} | 28 +- tests/test_bag_pipeline.py | 186 ------ tests/test_bag_sample.py | 114 ---- tests/test_bag_transform.py | 111 ---- tests/test_categories.py | 46 +- ...{test_bag_dispatch.py => test_dispatch.py} | 10 +- tests/{test_bag_io.py => test_io.py} | 60 +- tests/{test_bag_items.py => test_items.py} | 11 +- tests/test_labels.py | 8 +- tests/test_node_docs.py | 37 +- tests/test_op_families.py | 271 +++++++++ tests/test_parallel.py | 17 +- tests/test_pipeline.py | 124 ++++ tests/test_structure_ops.py | 133 ++--- tests/test_transform.py | 121 ++++ tests/test_typed_collate.py | 83 +-- tests/test_typed_detection_target_ops.py | 111 ++-- tests/test_typed_flow.py | 117 ++-- tests/test_typed_generic_ops.py | 135 +++-- tests/test_typed_storage.py | 188 ++++-- tests/test_typed_target_ops.py | 199 +++---- 81 files changed, 3602 insertions(+), 4643 deletions(-) create mode 100644 docs/record-model.md delete mode 100644 docs/typed-model.md create mode 100644 examples/record_pipeline.py delete mode 100644 examples/typed_pipeline.py delete mode 100644 sampleflux/bag/__init__.py delete mode 100644 sampleflux/bag/adapters/__init__.py delete mode 100644 sampleflux/bag/adapters/albumentations.py delete mode 100644 sampleflux/bag/adapters/torchvision.py delete mode 100644 sampleflux/bag/sample.py delete mode 100644 sampleflux/bag/transform.py rename sampleflux/{bag => }/dispatch.py (79%) rename sampleflux/{bag => }/io.py (67%) rename sampleflux/{bag => }/items.py (83%) delete mode 100644 sampleflux/ops/_augment_bridge.py delete mode 100644 sampleflux/ops/albumentations.py delete mode 100644 sampleflux/ops/albumentations_transforms.py delete mode 100644 sampleflux/ops/torchvision.py delete mode 100644 sampleflux/ops/torchvision_transforms.py delete mode 100644 sampleflux/ops/transform_chain.py create mode 100644 sampleflux/transform.py rename tests/{_bag_fixtures.py => _fixtures.py} (69%) delete mode 100644 tests/test_bag_pipeline.py delete mode 100644 tests/test_bag_sample.py delete mode 100644 tests/test_bag_transform.py rename tests/{test_bag_dispatch.py => test_dispatch.py} (82%) rename tests/{test_bag_io.py => test_io.py} (58%) rename tests/{test_bag_items.py => test_items.py} (92%) create mode 100644 tests/test_op_families.py create mode 100644 tests/test_pipeline.py create mode 100644 tests/test_transform.py diff --git a/AGENTS.md b/AGENTS.md index d684ea2..1f6e614 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,33 +2,33 @@ - **The Runnable Protocol Lives Here (`sampleflux.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** sampleflux owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `sampleflux.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `sampleflux.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `sampleflux.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `flux` validated in `run()`). `sampleflux.cli`: the `sampleflux run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `sampleflux-processing`/`sampleflux-workflow` + the `sampleflux` console script + `liquifai.apps`. - **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). -- **Op Consolidation (2026-07-18) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases). `Tee` threaded the sample through its branches sequentially, making it executionally identical to `TransformChain(ops=[...])` — use `TransformChain` for grouping and the context ops (`Save`/`Use`/`Mix`) for real, isolated fan-out. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom `ConfigureOp(ops=[UnstashInputOp(key)])` is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-sample side-branch (`ops` chain → `metadata[key]` + setattr) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters (fluxstudio export.py AND graphio.py) emit ONLY context ops for wiring; graphio's legacy `__taidal_stash_*` import replay was removed (pre-2026-07 stash-format ops-docs no longer import — re-export from the canvas). Do NOT reintroduce a metadata-bus twin of a context op. -- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on item TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `sampleflux.ops` stay plain callables. The `Transform` base is a thin type-dispatch shell (it samples params once per sample, then applies the per-type kernel to each handled field), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The Typed-Bag Model Is THE Data Model (`Sample`):** A `Sample` is a NAMED BAG of TYPED ITEMS, each item owning its own metadata. Import the whole typed surface from the PACKAGE TOP LEVEL (`from sampleflux import Sample, Image, Mask, Regions, Label, Transform, Pipeline, primary, item_data, typed_collate, register_item, register_kernel, register_adapter, register_io, ...`) — the `sampleflux.bag.*` module path is an internal/transitional home, never the taught import path. Transforms dispatch on item TYPE via a kernel registry (`@Transform.kernel(ItemType)` / `register_kernel`, the torchvision-v2 pattern) sampling params ONCE per sample (so a flip moves image+mask+boxes together), and external libraries (torchvision `transforms.v2`, albumentations) + user types plug in via registered adapters + one-line kernel registrations. BARE library transforms drop straight into a `Pipeline` — `coerce_transform` wraps any element via a registered matcher/factory (`register_adapter`); the torchvision/albumentations adapters self-register a matcher (by MRO module name, no eager library import) at package load, so `Pipeline([Fourier(), v2.Normalize(...), A.GaussNoise(...)])` works with no explicit adapter wrapper (use the explicit adapter with `only=` for per-key targeting). Items are HYBRID (array items subclass `np.ndarray` w/ attr-preserving `__array_finalize__`; structured items are dataclass wrappers). `input`/`target`/`aux` are ROLE TAGS on named fields, not tuple positions. **Modality-neutral (mandate above):** sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and ships **NO native augmentation transforms** — geometric/photometric augmentation comes from the libraries via coercion, and native `Transform`s exist only where no library covers them; the signal-domain items (`Signal`/`Spectrogram`) and the `Fourier` transform live in `waivefront` and register into the SAME sampleflux registries on import — do NOT add signal-domain items/transforms here. **Typed engine primitives:** `primary(sample, role)` (first field of a role — THE "the input" accessor for bind/Apply/engines) + `Sample.merge(*samples)` (ordered field/role union, last-listed wins on collision — the typed fan-in) + `Sample.rename`; the item CODEC registry (`EncodedItem`/`encode_item`/`decode_item`/`register_io` — storage backends call ONLY the codec, so externally-registered item types serialize with zero storage edits); the structure ops `sampleflux.ops.structure` (`SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, entry point `sampleflux-ops-structure` — the typed field-plumbing ops); and `typed_collate` (auto-dispatched for `Sample` batches) returning a BATCHED Sample (payloads stacked per field, per-item attrs as lists, roles preserved) — the ONE batch convention. **Typed storage:** all three backends write a `Sample` in the ONE field-group layout (`sampleflux_format="typedsample-v1"`; per sample one group per FIELD: `__item_type__`/`__role__` + plain attrs natively, payload as `data`, array attrs under `attrs/`, insertion order in `__field_order__`; structured attr values ride the JSON-tagged wire format in `storage/base.py::split_attrs`/`restore_attrs` — tuples SURVIVE); backends serialize ONLY through the `bag.io` codec so external item types round-trip with zero storage edits; `DirectorySink`↔`DirectorySource` (typed layout); `ZarrBatchSink` appends the PRIMARY input payload + a one-time uniform item template; typed metadata scans yield NESTED `{field: {attr: value}}` and `MetadataFilterSource.where` addresses it as `.` (`query.py::_AttrView`; a Python-keyword field name is unaddressable in an expression — use `predicate`). Pins: `tests/test_typed_storage.py`. **Typed engines:** a `Sample` passes through verbatim on every Flux route, `core._apply_op`/`_apply_op_native` apply ops to the bag verbatim, and `Use`/`Apply`/`Capture`/`_cell_field` are typed-aware (`_cell_field` on a bag = the `key`-named item or `primary()`; `Apply` gained `key=`). FlowGraph: `FlowStep.merge_from` is the fan-in (UNION of the named steps' fields+roles via `Sample.merge`, slot order, last-write-wins), lowered to the `MergeFields` context op (`ops/context.py`, `sources`/`keys`/`drop`) and lifted back by `from_ops`; `bind:` gained the field form `step[key]` (the named item; bare `step` = the primary input item), lowered to `Apply(key=...)`. The derived-field branch idiom: produce → `SelectFields([new_field])` → `merge_from` (a FULL branch bag would last-wins-overwrite shared keys — deliberate). Pins: `tests/test_typed_flow.py`. `sampleflux.bag` imports without torchvision (adapters lazy-import). Entry point `sampleflux-bag-transform`. Usage: `docs/typed-model.md`; rationale: `docs/architecture.md` → "The typed-bag model"; pins: `tests/test_bag_*.py`, `examples/typed_pipeline.py`. Follow-ups (root TASKS.md): torch-Tensor-subclass item base, confluid-native item discovery, generated `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, the `decode` path. -- **Metadata Lives PER ITEM, Never as One Flat Sample Dict:** There is no `Sample.metadata` dict — each typed item OWNS its own metadata (an `Image` knows its layout, a `Regions` its canvas, a `Signal` its samplerate, a `Label` its class names), carried as item attributes and serialized per field. Read or derive a value from the item that owns it — resolve the field via `primary(sample, role)` → `(key, item)`, then read the item's attrs / its `item_data(item)` payload — never from a string-keyed side dict. Batching is `typed_collate` (auto-dispatched for `Sample` batches): it returns a batched `Sample` with payloads stacked per field and each item's per-sample attrs collected into a list, roles preserved — the ONE batch convention (no separate `list[dict]` batched-metadata form and no `{"per_sample": [...]}` nest). Pins: `tests/test_typed_storage.py`, `tests/test_bag_*.py`. -- **Typed Bag — Full Traceability Rides on Items/Aux Fields:** All data flows through a `Sample` named bag of typed items; provenance is never dropped — everything that describes a value lives on the item that owns it, or as its own `aux`-role field, never bypassed. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` stays "no extra metadata" so it is OPT-IN and existing configs are unaffected. Keep `"*"` as the one sentinel (FluxStudio's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a Sample cell contributes its primary input item, a raw cell value is used verbatim), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' fields into the incoming sample, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells). They move data through a per-sample **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task`, `__getitem__`, and the streamed route's `(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the sample's own fields — a linear run's sample is byte-identical whether or not Context threading exists (pinned: `tests/test_context.py::test_metadata_untouched_invariant`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. -- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' fields into this step, in slot order with last-write-wins), and `bind` (`{param: step}` = the step result's `input`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-sample env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (`tests/test_flow.py::TestEngineParity`/`TestReverseParity`/`TestRoundTrip` + `examples/flow_graph.py`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `RescaleOp` checks `in_min < in_max`, `ThresholdOp` the at-least-one-bound rule, `EncodeTargetOp` the non-empty mapping — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `ThresholdOp.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. -- **Transforms Dispatch on Item TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which item TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per sample, then applies the matching kernel to EVERY field whose item type it handles, passing untouched fields through. Because the parameters are sampled once and shared, multi-field consistency is automatic — one flip moves image + mask + boxes together (the thing a flat-metadata triple could not express). Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them. A transform MAY change an item's type under the same field key (e.g. `Fourier` turns a `Signal` field into a `Spectrogram` in place). Bare library transforms (torchvision `transforms.v2`, albumentations) drop into a `Pipeline` via registered adapters (`register_adapter` / `coerce_transform`); a plain function becomes a transform via `as_transform(fn, handles=(ItemType,), only=[field])`, and a type-changing shape (read one field, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. Optional `only=[keys]` narrows a transform to specific field keys. Pins: `tests/test_bag_*.py`. -- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19):** Every op that wraps/applies OTHER ops — `TransformChain`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `sampleflux.core._apply_op(sample, op)`, NEVER `op(sample)` directly. `_apply_op` is the engine's single contract-aware chokepoint: it introspects the inner op's transform-taxonomy contract (`op_contract`) and binds the declared view (pair / input / target / `*_meta`, packed or unpacked), so a field-scoped op (e.g. a pair-scoped augmentation adapter) nests inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(sample)` call crashes on any non-sample-scoped op with a misleading "missing positional argument" `TypeError`. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Sample]`. Pins: `tests/test_augment_ops.py` (pair op inside `RandomApply`/`TransformChain`/`Enable`). -- **Augmentation = Adapter Ops + GENERATED Per-Transform Families, Never Reimplement (`sampleflux.ops.albumentations` / `.torchvision` / `.albumentations_transforms` / `.torchvision_transforms` / `._augment_bridge`, 2026-07-19):** Library augmentation enters the pipeline through two SAMPLE-scoped adapter ops — `AlbumentationsOp` (albumentations, numpy HWC in/out, core dep) and `TorchvisionTransformOp` (torchvision `transforms.v2`, CHW tensors out, `[vision]` extra, ALL torchvision imports lazy in the ADAPTER module so it imports without the library — pinned by `test_module_imports_without_torchvision`) — plus the AUTO-GENERATED per-transform op families: `sampleflux.ops._augment_bridge.generate_transform_ops` (the waivefront-helios auto-bridge pattern) walks each library's public transform classes at import time and emits one adapter SUBCLASS per transform (`Alb` ~115 ops, group `augment/albumentations`; `Tv` ~55 ops, group `augment/torchvision`) with a synthesized `__signature__`/`__annotations__`/spliced `Args:` docstring (transform params + `target`/`seed` appended LAST), the base `__call__` RE-STATED in the class dict (canvas op-classification reads `vars(cls)` — inherited-only methods are invisible), and a `raw_transform` property an adapter's `transforms` list UNWRAPS (so canvas transform nodes dock into a Compose-style adapter node). The `Alb`/`Tv` prefixes are MANDATORY (confluid's registry is flat + name-keyed; the libraries share bare names like `ColorJitter`/`Normalize`/`Resize` — the helios `Helios*` precedent); composition/container transforms are NOT generated (chaining is native); per-class generation failures skip with a DEBUG note, never break import; `torchvision_transforms` imports safely without torchvision (zero ops). Adapters: `@configurable(category="op", group="augment", random=True)`, `__call__(self, sample: Sample)` — SAMPLE-scoped deliberately, because the visual-canvas op classifier only recognises `__call__(sample: Sample)` and invokes `op(sample)` directly (pair-scoped ops are engine-legal but canvas-invisible until the kinds-grid socket stage lands); ONE library draw still moves input AND target jointly per the closed `TargetMode = Literal["none","mask","boxes"]` knob (`"boxes"` consumes the torchvision detection dict from `CocoToTorchVisionDetectionOp`/`MasksToDetectionBoxesOp`; albumentations `bbox_params` are AUTO-ADDED when the op composes — only a prebuilt `A.Compose` must carry its own, validated loudly). Config surface: `transform` (ONE transform / prebuilt Compose) XOR `transforms` (list, composed lazily; entries may be live objects, Confluid markers — flowed lazily — or generated ops); `seed` on the albumentations side maps onto `A.Compose(seed=...)` (rejected with a prebuilt Compose); NO probability knobs (gating is `RandomApply`). **YAML is Confluid-NATIVE ONLY** — nested `!class:albumentations.HorizontalFlip` / `!class:torchvision.transforms.v2.X` nodes or registered short names (`!class:AlbHorizontalFlip`), dump→load round-trips both (the engine captures foreign-class ctor kwargs); the earlier `A.to_dict()` dict-spec surface was REMOVED (user-rejected — never resurrect a library-specific serialization format as config). Don't add a third adapter without real demand, and never bake a specific augmentation as a bespoke hand-written sampleflux op — wrap the library or use the generated family. Docs: `docs/augmentation.md`; examples: `examples/augmentation_ops.py` / `examples/augmentation_training.py`; pins: `tests/test_augment_ops.py`, `tests/test_categories.py`. -- **Collation Is a Pluggable Registry (`sampleflux.collate`):** Batching a list of `Sample` bags into ONE batched `Sample` goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"typed"`** = `typed_collate`: it stacks each field's array payload, gathers per-item attrs as per-sample lists, and preserves roles (see the per-item-metadata mandate above). Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` fields, `raidar.detection_collate_fn`); their divergent conventions are deliberately NOT unified. `typed_collate` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. -- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op whose return annotation is `Iterator[...]`/`Iterable[...]`/`List[...]` (or that carries `EXPANDS = True`) is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op's contract expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `HDF5WindowSource`/`RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. Pins: `tests/test_expanding_ops.py`. +- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`sampleflux.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. +- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `sampleflux.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. +- **The RECORD Is THE Data Model (2026-07-25):** A sample is a **PLAIN `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `sampleflux.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`sampleflux.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `sampleflux.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from sampleflux import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `sampleflux/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `sampleflux.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). NEVER add a wrapper/adapter class for a library — supporting a NEW library family means adding a new branch in `_apply_op` (an MRO module-name matcher + the library's native calling convention), nothing else. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Flux._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel). +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-sample flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +- **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. +- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. +- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`sampleflux.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `sampleflux.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Collation Is a Pluggable Registry (`sampleflux.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `sampleflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into typed-bag `Sample`s — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only; the batch sink stores only the primary input payload). When you add a sink, add (or justify the absence of) its source in the same change. **The storage SINKS carry `category="sink"`** (`HDF5Sink` / `ZarrGroupSink` / `ZarrBatchSink` / `DirectorySink`) so FluxStudio surfaces them as object-member producer nodes (a `SAMPLEFLUX_OBJECT:sink` wire) that dock into a `marainer.processing.DatasetProcessor` runnable node's `sink` slot — the same model/loss/logger→trainer relationship; a canvas `Source → Flux → DatasetProcessor(sink=…)` then runs/exports exactly like the YAML `marainer convert` config. Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`) so discovery sees it. The matching SOURCES (`HDF5Source` …) stay **bare `@configurable` with no `category`** — they read a sink's layout back as YAML `!class:` nodes, NOT FluxStudio canvas nodes — so the positive `{op,source,engine,sink}` allowlist surfaces only the sink half. **HDF5 metadata storage:** scalar/string metadata is written as HDF5 *attributes*, but **array-valued metadata (`np.ndarray`/`torch.Tensor`, e.g. a segmentation mask) is written as its own dataset under a per-sample group `{prefix}_meta/`** — HDF5 caps attribute size, and the legacy str() fallback silently truncated arrays. `HDF5Source` merges the meta group back on read; files written before this layout (no `{prefix}_meta` group) read unchanged, so it is fully backward-compatible. Route array metadata to a dataset via an explicit `isinstance` check (defensive-programming), not by catching the attribute-write exception. **Tensor→array conversion is shared:** array sinks convert `Sample` fields to numpy via `to_numpy` (in `storage/base.py`, re-exported from `storage/hdf5.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. -- **Metadata Is QUERYABLE Without Array Loads (`sampleflux.storage.query`, 2026-07-17):** `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; implemented on `HDF5Source` (attrs + array-metadata shape/dtype STUBS) and `ZarrGroupSource` (`.zattrs`) — existing files queryable with NO rewrite; the protocol is STRUCTURAL, so external storage sources (e.g. waivefront's `SigMFSource`) implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration (the projection-module pattern). Entry point `sampleflux-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair (`SigMFSink`/`SigMFSource`) MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; sampleflux keeps ZERO knowledge of it. Pins: `tests/test_query.py`, `waivefront/tests/test_sigmf.py`. -- **Field Projection (`sampleflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(fields) -> Iterator[Sample]`) to yield input-only / target-only `Sample`s **without building unrequested fields** (e.g. an image dataset reads only the label column for a target-only walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The field set is the **closed `Literal`** `ProjectionField = Literal["input", "target", "metadata"]` (exported from `sampleflux.projection` and the package root), NOT a bare `str` — so a typo fails the type check and UIs / form-specs / MCP schemas enumerate the choices via `typing.get_args(ProjectionField)`; the runtime-validation tuple `_FIELDS` is `get_args(ProjectionField)` (one source of truth — never restate the values). Every `project(self, fields: Collection[ProjectionField])` implementer (the `Flux` engine, `HuggingFaceClassificationDataset`, …) MUST use this type. This is the workspace "prefer closed `Literal`s over bare strings" mandate applied. Consumers use the helpers `project()` / `iter_inputs()` / `iter_targets()`, which fall back to full iteration + field-nulling for sources that don't implement it. `num_classes(source)` is built on this — it always walks targets and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers (marainer's run-time dimension injection). -- **`LabelMap` Is the *Fittable* Companion to `EncodeTargetOp` (`sampleflux.labels`):** `EncodeTargetOp` / `DecodeTargetOp` carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing sampleflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time** (sonair's classification trainer), then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a FluxStudio canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a sampleflux dependency for this. +- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `sampleflux_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `sample_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `sampleflux/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Metadata Is QUERYABLE Without Array Loads (`sampleflux.storage.query`, 2026-07-17):** `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `sampleflux-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; sampleflux keeps ZERO knowledge of it. +- **Key Projection (`sampleflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Flux.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`sampleflux.labels`):** `EncodeTarget` / `DecodeTarget` (`sampleflux.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing sampleflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a sampleflux dependency for this. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The sampleflux buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield Samples: `HuggingFaceSource` (and waivefront's `RFUAVSource` / `RegionsJsonSource`), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource`** — each yields Samples and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:end)` slice · concatenation) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Sample → Sample` **ops**: EVERY op meant to be a canvas node MUST carry it (`RescaleOp`, `StandardizeOp`, `ThresholdOp`, the structure ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields`, `Parallel`, the modality-neutral compose helpers `Enable` (`sampleflux.ops.enable` — toggle an op-list via one named CLI flag) + `TransformChain` (`sampleflux.ops.transform_chain` — group a fixed op-sequence into one named unit; deterministic, no gate) + `SampleSinkOp` (`sampleflux.ops.sink` — adapt a `DataSink` as a pass-through op) + `ConfigureOp` (`sampleflux.ops.configure` — the helios *Configure* pattern: a `ops` compute-chain derives a value FROM the sample, writes it to `metadata[key]`, setattr's it as the `param` attribute of the wired `target` op, then applies `target` to the original sample — the sanctioned per-sample-parameter mechanism, e.g. a sample-derived `ThresholdOp.low_level`; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances; its companion `FormulaOp` (`sampleflux.ops.formula`) evaluates a restricted math formula over the sample's primary input item (`primary(sample, "input")`) — the canvas Math node's op form, emitted by the ops-export's value-chain compilation), the target shapers `MetadataToTargetOp` / `EncodeTargetOp` / `DecodeTargetOp` + the two detection-target ops `CocoToTorchVisionDetectionOp` / `MasksToDetectionBoxesOp` (`sampleflux.ops.target` — both emit the torchvision detection target `{boxes xyxy, labels}`, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK — per-instance bbox from an instance mask, or connected-components from a binary mask via the shared `sampleflux.ops.numpy.connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation; both modality-neutral image-detection counterparts of waivefront's signal-domain `RegionsToDetectionBoxesOp`), and the waivefront signal/target ops). FluxStudio uses a POSITIVE allowlist `{op, source, engine}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`): sampleflux ops use `numpy` / `torch` / `structure` (the typed field-plumbing ops `SetRole`/`RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — re-tag a field's role, rename or copy a field key, drop a field, or narrow the bag to a chosen set of fields; these are how a derived-field branch is assembled and how a snapshot is carried across a `Parallel` boundary or persisted into a sink as its own `aux`-role field) / `compose` (`Parallel`/`Enable`/`TransformChain`/`RandomApply`/`ConfigureOp`/`FormulaOp`) / `image` / `sink` (`SampleSinkOp`) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-sample summary — input/target shape+dtype + summarised metadata — to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset) (pinned in `tests/test_categories.py`). An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; publishes `image_width_px`/`image_height_px`) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across samples — the standalone `NormalizeToUint8Op` op class was DELETED in the typed purge; only the function remains), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a `Sample`, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it. Pin: `tests/test_image_ops.py::test_array_histogram_large_array_does_not_raise` (a >65536-element array). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency** (already used by `typespec.py`); matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it); pin `tests/test_image_ops.py::test_draw_text_*`. Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` / `FlowGraph` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. FluxStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`). The sampleflux groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `MetadataToTarget` / `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`sampleflux.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`sampleflux.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`SampleSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). -- **Type IS the Item's Python Class, Never a Separate Field:** A field's type is its item's Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key. A consumer reads a field's type by `type(item)` and its shape/dtype/framework off the item's own payload and attrs. A transform that changes a value's type replaces the item under the same field key (e.g. `Signal` → `Spectrogram`, `array` → `Mask` → `Regions`). Never carry a parallel type descriptor beside the bag. +- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. diff --git a/README.md b/README.md index 63b27c8..0bee9fe 100644 --- a/README.md +++ b/README.md @@ -6,68 +6,94 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `SampleF ## 🚀 Key Features -- **Functional Purity:** Transforms are simple Python callables. No complex base classes required. -- **Typed Bag of Items:** A `Sample` is a named bag of typed items (`Image`, `Mask`, `Regions`, `Label`, …), each owning its own metadata — and [transforms dispatch on item type](docs/typed-model.md), so one sampled decision moves image + mask + boxes together. -- **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-sample `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Flux` engine. +- **A sample is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. +- **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. +- **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. +- **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-record `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Flux` engine. - **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. - **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-samplefluxstoragequery) — filter stored datasets without loading a single array. -- **Passive Introspection:** transforms declare their [item-type contracts](docs/typed-model.md) (the item types they handle / consume / produce) and are discoverable by category for visual editors and schema generators. +- **Passive Introspection:** ops declare the value types they [handle / consume / produce](docs/record-model.md) and are discoverable by category for visual editors and schema generators. - **100% Reproducibility:** Entire pipelines are serializable via **Confluid** manifests. ## 🛠 Quick Start +One pipeline mixing a **bare albumentations Compose** (image + mask + boxes move together in one draw), a **bare torchvision v2 transform**, and a **native op** — no wrappers (mirrors [`examples/record_pipeline.py`](examples/record_pipeline.py)): + ```python +import albumentations as A import numpy as np -from sampleflux import Sample, Image, Flux, as_transform, primary - -# 1. A plain function becomes a transform, dispatched on item type -recenter = as_transform(lambda d: d - 0.5, handles=(Image,)) - -# 2. Build a pipeline over a source of typed samples -raw_data = [Sample({"input": Image(np.random.randn(10))}) for _ in range(100)] +from sampleflux import Flux, Image, Label, Mask, as_transform + +records = [ + { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), # typed: knows its layout + "mask": Mask((rng.random((16, 20)) > 0.5).astype(np.uint8)), + "bboxes": [[2, 3, 6, 7]], # albumentations vocabulary + "labels": ["drone"], + "class": Label("drone_x", classes=["noise", "drone_x"]), # typed: knows its vocab + "gain_db": -3.0, # a scalar is just another key + } + for rng in (np.random.default_rng(i) for i in range(100)) +] + +flux = Flux( + source=records, + ops=[ + A.Compose( # bare albumentations — as-is + [A.HorizontalFlip(p=0.5)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ), + A.GaussNoise(p=1.0), # image only (its own kwarg vocabulary) + as_transform(lambda d: d - 0.5, handles=(Image,)), # native: a plain function op + ], +).parallel(workers=4) + +for record in flux: + print(record["image"].shape, record["class"].value) # image+mask+boxes flipped together +``` -flux = ( - Flux(source=raw_data, ops=[recenter]) - .filter(lambda s: primary(s, "input")[1].mean() > 0) - .parallel(workers=4) -) +The same ops list in Confluid YAML — bare library transforms are ordinary `!class:` nodes: -# 3. Collect or stream -for sample in flux: - _, item = primary(sample, "input") # (key, item) - print(item.shape) +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.GaussNoise + p: 1.0 + - !class:sampleflux.ops.numpy.Threshold + low_level: 0.5 ``` ## 📚 Documentation | Page | Covers | |---|---| -| [docs/kinds.md](docs/kinds.md) | The transform taxonomy (field scope × call style), multi-type carriers (`Flux(native=True)`), the collate registry, 1→N expanding ops | +| [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | +| [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`), 1→N expanding ops | | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Flux.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | -| [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources, array-valued metadata, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | -| [docs/projection.md](docs/projection.md) | Field projection (`SupportsProjection`), lazy target walks, `num_classes`, the fittable `LabelMap` | +| [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | +| [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap` | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | -| [docs/configure.md](docs/configure.md) | Per-sample op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | -| [docs/augmentation.md](docs/augmentation.md) | Augmentation via albumentations / torchvision `transforms.v2` — joint input+target (mask/boxes) adapters, the generated `Alb*`/`Tv*` per-transform ops, seeding, Confluid-native YAML | -| [docs/typed-model.md](docs/typed-model.md) | The typed-bag data model: a `Sample` is a named bag of typed items (each owning its metadata), type-dispatched transforms, torchvision/albumentations adapters, custom item types | +| [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | +| [docs/augmentation.md](docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | | [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | ## 🧭 Scope: a modality-neutral engine SampleFlux deliberately contains **no domain-specific code** — every op, source and sink in this package is meaningful for any modality (arrays, tensors, images, generic metadata). Domain packages build on it and keep their own vocabulary: -- Signal/waveform work (1-D FFT + windowing ops, SigMF recording storage, spectrograms, the annotation-join source) lives in the **waivefront** package. +- Signal/waveform items and ops (spectrograms, FFT windows, recording formats) live in the domain package, which registers its item types into the same registries. - Task-specific trainers, collates and models live in their consuming projects. ## 🌐 Ecosystem Integration SampleFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: -- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into typed `Sample` bags with full metadata traceability (see [docs/sources.md](docs/sources.md)). -- **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node, every run reproducible. -- **PyTorch**: `Flux` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#multi-type-carriers--the-collate-registry-samplefluxcollate). -- **Augmentation libraries**: `AlbumentationsOp` / `TorchvisionTransformOp` wrap [albumentations](https://albumentations.ai) and torchvision `transforms.v2` as ops that augment input AND target (mask / detection boxes) jointly — plus an auto-generated op per individual library transform (`AlbHorizontalFlip`, `TvColorJitter`, …), each a graph node and a Confluid `!class:` one-liner (see [docs/augmentation.md](docs/augmentation.md); torchvision via `pip install "sampleflux[vision]"`). +- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability (see [docs/sources.md](docs/sources.md)). +- **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node — including bare library transforms — every run reproducible. +- **PyTorch**: `Flux` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#batching--collate_records--the-collate-registry-samplefluxcollate) (`collate_records` is the default). +- **Augmentation libraries**: [albumentations](https://albumentations.ai) and torchvision `transforms.v2` transforms run **as-is** in any ops list — the engine speaks each library's native convention (kwarg vocabulary vs dict walk), so there is nothing to wrap (see [docs/augmentation.md](docs/augmentation.md)). ## 🔧 Installation diff --git a/docs/architecture.md b/docs/architecture.md index bd680ba..ed55a6f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -8,11 +8,112 @@ of reverse-engineering it from git history. Each entry is a short decision record: **Context → Decision → Consequences → Example → What you may change**. When a change alters one of these mechanisms, update its record in the same change (see -the workspace `AGENTS.md` → "Architecture Decisions Are Documented"). +the workspace `AGENTS.md` → "Architecture Decisions Are Documented"). Superseded records are kept +as history, banner-marked with a pointer to their successor. --- -## Batching is two-stage; collation is a pluggable registry (`sampleflux.collate`, 2026-07-17) +## One type-dispatched op engine — plain-dict records, libraries as-is (2026-07-25) + +### Context + +The previous data model (the typed-bag `Sample`, recorded below and now superseded) got the item +half right — typed values owning their metadata — but wrapped them in a bespoke container with +per-key role tags. That container was the friction point: every external library needed an adapter +before it could touch a sample (`coerce_transform` + a matcher/factory registry + two adapter +classes + ~170 GENERATED per-transform op wrappers, all maintenance surface), the role tags +duplicated what key names already say (`"mask"` *is* the mask), and dict-native libraries — +torchvision `transforms.v2` walks dicts, albumentations takes named kwargs — were kept at arm's +length from a carrier they could have consumed directly. Meanwhile a second op-authoring surface +(the adapter/generated families) competed with the native type-dispatched `Transform`, so "where +does augmentation come from?" had three answers. + +### Decision + +Collapse to ONE carrier and ONE op engine: + +- **A sample is a plain `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **typed values** + (`Image`/`Mask`/`Regions`/`Label`, base `NDArrayItem`; open registry `register_item`; uniform + payload accessors `item_data`/`with_data`). No container class, no roles, no `primary()`: + **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"class"`), and a scalar side value + is just another key. Metadata is attrs on the typed value (`Image.layout`, `Label.classes`) or + more dict keys (`"samplerate": 30.72e6`). +- **Native ops are type-dispatched `Transform`s** (`sampleflux/transform.py`): `get_params(record)` + draws shared parameters ONCE per record, per-type kernels (`@MyOp.kernel(ItemType)`, MRO-aware + registry in `sampleflux/dispatch.py`) apply to every handled value, `field=` pins one key. The + second sanctioned shape — type-CHANGING ops (`Threshold`, `ConvertToImage`, the target ops) — + overrides `__call__`. +- **External libraries run AS-IS through the engine's op-family dispatch** + (`sampleflux.core._apply_op`, three branches): an albumentations op receives exactly its own kwarg + vocabulary (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record; one + call = one joint draw; array outputs re-wrapped in the incoming `NDArrayItem` type so + `Image`/`Mask` survive); a torchvision-v2 op is called on the dict as-is; everything else is + `op(record)` with `None` = drop. Family detection is by MRO module name — no eager imports, no + adapters, no generated wrappers. Box-carrying augmentation is the library's own + `A.Compose(..., bbox_params=...)`; seeding is the libraries' own mechanisms. +- **`Pipeline(transforms=[...])`** (`sampleflux/transform.py`) is THE sequential composer — + `TransformChain` was deleted; every composing op routes inner ops through `_apply_op`. +- **Storage is the record key-group layout** (`typedrecord-v1`): everything serializes through the + `sampleflux/io.py` codec; plain values ride the `"plain"` tag; NO backward compatibility with the + pre-record layout (an old/untagged store raises via `storage/base.py::require_record_format` — + an explicit decision: re-generate, don't accrete legacy readers). +- **Projection and collation are key-addressed**: `project(source, keys)` / `iter_key` / + `num_classes(key="class")`; the collate registry's default is `"record"` = `collate_records`. + +### Consequences + +- Zero adapter surface: the two adapter classes, the coercion registry, and both generated op + families are gone; a new library version's transforms are available the moment the library is — + nothing to regenerate. +- Cross-key consistency is the LIBRARY's own joint draw (albumentations Compose / tv2's dict walk) + for augmentation, and `get_params`-once for native ops — one mechanism per world, both automatic. +- YAML needs no special forms: a bare `!class:albumentations.HorizontalFlip {p: 0.5}` sits in an + `ops:` list like any native op (deferred markers flow at route entry). +- The albumentations vocabulary is load-bearing: a value augments only if it rides one of the + library's key names — routing is an explicit `RenameField`, never engine magic. +- Anything that used `Sample`, roles, `primary()`, `typed_collate`, `ProjectionField`, or a + `typedsample-v1` store must migrate — there are deliberately no aliases and no legacy read path. + +### Example + +One `Flux` ops list mixing both worlds, no wrappers: + +```python +import albumentations as A +from sampleflux import Flux, Image, as_transform + +flux = Flux(source=records, ops=[ + A.Compose([A.HorizontalFlip(p=0.5)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"])), + A.GaussNoise(p=1.0), # bare library op — as-is + as_transform(lambda d: d - 0.5, handles=(Image,)), # native type-dispatched op +]) +``` + +The same shape in YAML: + +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:sampleflux.ops.numpy.Threshold + low_level: 0.5 +``` + +### What you may change (and where it's documented) + +- **A new item type** — one class + `@register_item` (array-backed: subclass `NDArrayItem`, + declare `_item_attrs`); usage in [record-model.md](record-model.md). +- **A new per-type behaviour for an existing op** — `@Op.kernel(ItemType)`, no core edit. +- **A new library family** — a new branch in `core._apply_op` (MRO module-name matcher + the + library's native calling convention). Never an adapter/wrapper class; update this record when a + branch is added. +- **The `typedrecord-v1` tag and the no-back-compat rule are contracts** — changing the on-disk + layout means a NEW tag and a re-generation story, never a silent dual-read path. + +--- + +## Batching is two-stage; collation is a pluggable registry (`sampleflux.collate`, 2026-07-17, updated 2026-07-25) ### Context @@ -25,32 +126,21 @@ Turning N pipeline items into one batched carrier has two distinct halves: The engine owns grouping; it must NOT own stacking, because stacking is task-shaped: historically every consuming project shipped its own task collate (classification, segmentation, detection), -and **two divergent batched-metadata conventions** emerged — the list-form -`Sample(metadata=[...])` batch (`Sample.is_batched` True) versus a dict-nested -`metadata={"per_sample": [...]}` form. In addition, the multi-type carrier engine -(`Flux(native=True)`, see [kinds.md](kinds.md)) meant sampleflux itself needed stacking behavior -*keyed by carrier kind* — a `Sample`, a metadata-free pair, a bare value, and the -`InputMeta`/`TargetMeta` views each batch differently. +and divergent batched-metadata conventions emerged between them. ### Decision `sampleflux/collate.py` is a **pluggable registry of collate functions keyed by representation**: `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`, where an omitted key -uses the default `"typed"` collate (`typed_collate`) — batching a list of typed-bag `Sample`s into -one batched `Sample` (per-item type dispatch itself is the sibling kernel registry -`sampleflux.bag.dispatch`, which walks each item's MRO). sampleflux registers the `"typed"` -default; consuming projects may register task aliases (`"yolo"`, `"segmentation"`, …) -**additively**. Re-registering a key deliberately overwrites (logged at debug) so a consumer can -replace a default. - -Two things were deliberately **not** done: - -- **Existing task collates were not moved here.** The registry is an addressable home consumers - can opt into, not a forced migration — consuming projects keep shipping and wiring their own - collate functions directly (e.g. via a Confluid `!ref:` to the function's dotted path). -- **The divergent metadata conventions were not unified.** The dict-nested - `{"per_sample": [...]}` convention stays with the project that owns it; unification is a - tracked follow-up in the root `TASKS.md`, not a side effect of introducing the registry. +uses the default **`"record"`** collate (`collate_records`) — N plain record dicts into ONE batched +record: per key, typed values encode through the `sampleflux/io.py` codec, payloads stack +(torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a +LIST of per-record values (decoded back into one batched item of the same type), and a +`"plain"`-tagged value batches as the plain list. Batches must be key-homogeneous — a mismatch +raises. Consuming projects may register task aliases (`"yolo"`, `"segmentation"`, …) +**additively**; re-registering a key deliberately overwrites (logged at debug) so a consumer can +replace a default. The divergent consumer conventions were deliberately NOT unified here — the +registry is an addressable home consumers opt into, not a forced migration. ### Primary intended consumer: the MCP tool surface @@ -64,35 +154,31 @@ collate function directly remains the normal path; the registry never replaces i ### Consequences -- The engine stays task-agnostic: sampleflux knows *kinds*, never classification/detection/… -- `Flux(native=True)` pipelines and the examples get correct batching per carrier kind with zero - configuration (`DataLoader(flux, collate_fn=get_collate("sample"))`). +- The engine stays task-agnostic: sampleflux stacks by key + item type, never + classification/detection/… +- Item metadata batches deterministically: per-record attrs become lists on the ONE batched item + (`batch["image"].layout == ["HWC", "HWC", ...]`), plain values become plain lists — there is no + second batched-metadata convention in this package. - One addressable lookup (`get_collate("yolo")`) replaces scattered cross-package imports — once a consumer registers. Registration happens at module import, so a key exists only after its defining module has been imported. -- Batched metadata's list form (`Sample.is_batched`) is produced here, which is why the - `Sample.metadata` `dict | list[dict]` duality exists (see the sampleflux `AGENTS.md` metadata - mandate). -- **Current usage (as of 2026-07-20):** only the five kind defaults are registered; the live call - sites are one training example (`get_collate("sample")` as a `DataLoader` collate) and the test - pins. No consuming project registers or looks up yet — the open registration surface is capacity - held for the MCP tool surface above, and is provisional until that consumer lands. +- **Current usage:** only the `"record"` default is registered here; the open registration surface + is capacity held for the MCP tool surface above. ### Example ```python from torch.utils.data import DataLoader -from sampleflux import Flux, collate, get_collate, register_collate +from sampleflux import Flux, collate, collate_records, get_collate, register_collate flux = Flux(source=my_source, ops=[...]) -# Kind-dispatched: Samples stack via the "sample" default (list-form batched metadata). -batch = collate([flux[0], flux[1]]) -assert batch.is_batched +batch = collate([flux[0], flux[1]]) # the "record" default +batch["image"].shape # stacked payloads, one batched Image +batch["image"].layout # per-record attrs -> a list -# Explicit key — the DataLoader glue. -loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("sample")) +loader = DataLoader(flux, batch_size=8, collate_fn=collate_records) # A task alias registers additively (runs when the defining module is imported). @@ -108,33 +194,33 @@ loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("yolo")) - **Plugging in your own batch layout** is the supported extension point — decorate a function with `@register_collate("your-key")` and select it via `get_collate`/`collate`. Usage lives in - [kinds.md → the collate registry](kinds.md#multi-type-carriers--the-collate-registry-samplefluxcollate). -- **Changing a default collate's semantics** (e.g. how `"sample"` stacks, or the list-form metadata - convention) is an architectural change: every batch consumer (losses, predictions sinks, - `batch_meta` readers) depends on it. Update this record and the metadata mandate together. + [kinds.md](kinds.md). +- **Changing the default collate's semantics** (how `"record"` stacks, the attrs-become-lists + convention) is an architectural change: every batch consumer depends on it. Update this record + and the sampleflux `AGENTS.md` metadata mandate together. --- -## The per-sample Context is an ambient wiring plane (`sampleflux.context`, 2026-07-17) +## The per-record Context is an ambient wiring plane (`sampleflux.context`, 2026-07-17) ### Context Graph-shaped pipelines — fan-out, fan-in, cross-branch values — need somewhere to hold a value -between the op that produces it and the op that consumes it. The obvious candidate, -`sample.metadata`, was rejected: metadata is the **accumulating bus that rides inside each -sample** — it persists into sinks, crosses process boundaries, and is part of the sample's -serialized identity, while wiring data is transient scaffolding that should be gone by the end of -a well-formed graph. Three constraints shaped the mechanism: ops keep the plain -`__call__(sample)` signature (no threading a context parameter through every op), the executor -stays a bare `for op in ops` loop (graphs run on the *plain sequential engine*), and a linear -pipeline's behavior — including its metadata, byte-for-byte — must be completely untouched. +between the op that produces it and the op that consumes it. The obvious candidate, extra keys on +the record itself, was rejected: the record is the carrier that **persists** — it flows into sinks, +crosses process boundaries, and is the sample's serialized identity — while wiring data is +transient scaffolding that should be gone by the end of a well-formed graph. Three constraints +shaped the mechanism: ops keep the plain `__call__(record)` signature (no threading a context +parameter through every op), the executor stays a bare `for op in ops` loop (graphs run on the +*plain sequential engine*), and a linear pipeline's behavior — its records, byte-for-byte — must be +completely untouched. ### Decision -`sampleflux/context.py` is a **per-sample named-cell store activated ambiently**: the engine +`sampleflux/context.py` is a **per-record named-cell store activated ambiently**: the engine creates one fresh `Context` per source item and activates it around the op loop via a -`contextvars.ContextVar`; the six wiring ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`Mix` in -`sampleflux.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature +`contextvars.ContextVar`; the six wiring ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields` +in `sampleflux.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature change anywhere. Deliberate semantics: cells are stored **by reference** and copy-on-read is the *reading* op's decision (`Use` deep-copies unless `drop` frees the cell = move); a missing cell on read or delete **raises loudly** with the live-cell list (a liveness bug must never pass @@ -145,7 +231,7 @@ YAML — it is pure runtime plumbing. The public surface is two-tier by design: is a package-root export, while `activate`/`current`/`require` stay module-qualified (`sampleflux.context.…`) — reachable, but visibly plumbing. `FlowGraph` deliberately does NOT use this module: its named-step documents give the compiler full knowledge of cell lifetimes, so it -manages its own per-sample env directly, held to the context-op semantics by the pinned +manages its own per-record env directly, held to the context-op semantics by the pinned flow⇄ops execution-parity contract. ### Consequences @@ -154,7 +240,7 @@ flow⇄ops execution-parity contract. graph exporters (a visual canvas, the `flow:` compiler) lower to, so ONE executor serves both linear and graph pipelines. - Linear pipelines are provably untouched: no context op ⇒ the Context is created and never used; - the metadata-byte-identical invariant is pinned in `tests/test_context.py`. + the record-byte-identical invariant is pinned in the record-model suite under `tests/`. - Spawn-parallelism is safe by construction: contexts are created *inside* the worker and never pickled or shared across processes. - Ambient state cuts both ways: running an op list containing context ops *outside* an engine @@ -167,15 +253,15 @@ flow⇄ops execution-parity contract. ```python from sampleflux import Flux -from sampleflux.ops.context import Mix, Save +from sampleflux.ops.context import MergeFields, Save # Fan-out/fan-in on the PLAIN sequential engine: snapshot → mutate the stream → merge back. flux = Flux( source=my_source, ops=[ - Save(name="clean"), # snapshot the pristine sample into a cell - my_augment_op, # the stream mutates freely - Mix(target_from="clean", drop=["clean"]), # fan-in: target from the snapshot, cell freed + Save(name="clean"), # snapshot into a cell + my_augment_op, # the stream mutates freely + MergeFields(sources=["clean"], keys=["mask"], drop=["clean"]), # fan-in, cell freed ], ) @@ -184,7 +270,7 @@ from sampleflux.context import Context, activate, require with activate(Context()): for op in ops: - sample = op(sample) + record = op(record) # A custom op joins the wiring plane through the same seam the built-in six use: # require("MyOp").get("clean") / require("MyOp").put("my_cell", value) @@ -196,12 +282,12 @@ with activate(Context()): `require("YourOpName")` inside `__call__`, follow the by-reference/copy-on-read discipline, and free cells you consume. Usage of the six built-in ops lives in [graph.md](graph.md). - **Keep the surface narrow.** Don't root-export `activate`/`current`/`require`, and don't grow - `Context` into a general blackboard — anything that should *persist with the sample* belongs on - the metadata bus, not in a cell. + `Context` into a general blackboard — anything that should *persist with the record* belongs in + the record itself, not in a cell. - **Changing cell semantics** (by-reference storage, loud missing-cell errors, the `Parallel` boundary rule, `copy()` shallowness) is an architectural change: the flow⇄ops parity suite and - the pinned context invariants (`tests/test_context.py`, `tests/test_flow.py`) define the - contract. Update this record and the sampleflux `AGENTS.md` context mandate together. + the pinned context invariants define the contract. Update this record and the sampleflux + `AGENTS.md` context mandate together. --- @@ -254,8 +340,8 @@ resolves *a curated name/category*. importable-function targets confluid's `resolve_class` module-path branch / `!ref:` grammar can — two spellings of one job (`"module:qualname"` here vs `"module.attr"` there). The non-overlapping remainder (path *production* via `get_callable_path`, `.py`-file and `__main__` - handling, module scans, `ACCEPTS`/`PRODUCES` schemas) is why the module exists; whether the - resolution half should delegate to confluid is a tracked follow-up in the root `TASKS.md`. + handling, module scans) is why the module exists; whether the resolution half should delegate + to confluid is a tracked follow-up in the root `TASKS.md`. ### Example @@ -295,7 +381,7 @@ They stay in `core.py` because of **who constructs them and which way imports fl are the construction targets of `Flux`'s own fluent API — `.filter(pred)` appends a `FilterOp`, `.map(fn)` appends a `WrappedOp`, `Flux.joint([...])` wraps a `JointFlux` — so the engine itself instantiates them. And `core.py` is the *bottom* of the op-facing layer: every composing op in -`ops/` imports `core._apply_op` (the contract-aware chokepoint); moving `FilterOp`/`WrappedOp` +`ops/` imports `core._apply_op` (the op-family dispatch chokepoint); moving `FilterOp`/`WrappedOp` into `ops/` would make `core` import from `ops` and close an import cycle. `JointFlux` is `Flux`'s iteration-only fan-in sibling (`category="engine"`), 20 lines that exist to be `Flux.joint`'s return value — a module of its own would be structure for structure's sake @@ -320,10 +406,10 @@ off visual canvases. ```python flux = ( Flux(source=src) - .map(np.sqrt) # appends WrappedOp(f="numpy:sqrt") - .filter(lambda s: float(s.input.max()) > 0) # appends FilterOp(p=...) + .map(np.sqrt, key="image") # appends WrappedOp(f="numpy:sqrt", key="image") + .filter(lambda r: float(r["image"].max()) > 0) # appends FilterOp(p=...) ) -both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flux_a, flux_b])) +both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flux_a, flux_b])) ``` ### What you may change (and where it's documented) @@ -336,9 +422,15 @@ both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flux_a --- -## The typed-bag model: a named bag of typed items (`sampleflux.bag`, 2026-07-21) +## ~~The typed-bag model: a named bag of typed items (`sampleflux.bag`, 2026-07-21)~~ — SUPERSEDED -### Context +> **Superseded (2026-07-25)** by +> [One type-dispatched op engine — plain-dict records, libraries as-is](#one-type-dispatched-op-engine--plain-dict-records-libraries-as-is-2026-07-25). +> The `Sample` container, role tags, `primary()`, the adapter coercion registry, and the +> `sampleflux.bag` package were removed; the typed items, the kernel-dispatch idea, and the item +> codec carried forward into the record model. Kept as history — do not follow. + +### Context (historical) Before the typed model, the carrier was a fixed `(input, target, metadata)` 3-tuple where `metadata` was one flat `dict` shared by the whole sample. Everything that is not literally the model input or @@ -348,273 +440,144 @@ label's class names. Two structural costs follow. First, **metadata has no owner belongs to *the signal*, `canvas` to *the image*, but the flat dict severs that link. Second, **a transform cannot move several fields together** — flipping an image and its mask and its boxes with one shared decision is inexpressible when the fields are `input`, `target`, and `metadata["regions"]` -respectively, so today's augmentation adapters hard-code a `TargetMode = Literal["none","mask","boxes"]` -knob per op instead. `target` is also overloaded — sometimes a bare string (`"drone_x"`), sometimes a +respectively, so the era's augmentation adapters hard-coded a `TargetMode = Literal["none","mask","boxes"]` +knob per op instead. `target` was also overloaded — sometimes a bare string (`"drone_x"`), sometimes a `{boxes, labels}` dict. -### Decision +### Decision (historical) -`sampleflux.bag` models a sample as a **named bag of typed items with per-field role tags**, and -dispatches transforms on item TYPE via a kernel registry: - -- **Items own their metadata.** An item is a typed value plus the metadata that describes *it* - (`Image(arr, layout)`, `Regions(boxes, labels, canvas)`, `Label(value, classes)`). The - realization is HYBRID: array-backed items (`Image`/`Mask`) subclass `np.ndarray` with - attribute-preserving `__array_finalize__`, so a type-agnostic op touches them as an array; - structured items (`Regions`/`Label`) are dataclass wrappers. A uniform `item_data` / `with_data` - pair hides the difference from kernels. sampleflux ships only MODALITY-NEUTRAL items; signal-domain - items (`Signal`, `Spectrogram`) live in the domain package and register into the same registry (see - "Consequences"). -- **`Sample` is a named bag; `input`/`target` are role TAGS, not positions.** A field carries a - role (`input`/`target`/`aux`/`pred`); `inputs()`/`targets()`/`aux()` read them at the - train/collate/sink boundary. A field changes role without moving keys. The sample is immutable — - every mutator returns a new sample (copy-on-write). -- **Transforms sample params ONCE, then dispatch a kernel per item type** (the torchvision-v2 - `_KERNEL_REGISTRY` pattern, structurally the same registry idea as `sampleflux.collate`). Kernels - are registered per `(transform, item type)` and resolved by MRO. Targeting is by type, with an - optional `only=[keys]` filter. -- **External libraries plug in through adapters, dropped in BARE.** A `Pipeline` COERCES each element - (`coerce_transform`): a `Transform` is used as-is; a foreign object is wrapped by whichever adapter - a matcher/factory pair claims it (`register_adapter`). The built-in torchvision-v2 and albumentations - adapters register a matcher (by MRO module name — no eager library import) at package load, so - `v2.Normalize(...)` / `A.GaussNoise(...)` go straight into a `Pipeline` with no explicit wrapper. A - plain function becomes a transform via `as_transform`; a new item type is taught to an existing - transform with one `@Transform.kernel(NewType)` registration. This keeps consumer-dialect knowledge - (how to recognise/adapt a library) OUT of the core and open for any user library. - -This is **THE sampleflux data model** — the one carrier every source, op, engine and sink handles. -It deliberately introduces a `Transform` base and typed item classes; the "Functional Purity" mandate -(see `AGENTS.md`) holds because that base is a thin type-dispatch shell and the per-type kernels stay -plain callables. +`sampleflux.bag` modeled a sample as a **named bag of typed items with per-field role tags** +(`Sample`, roles `input`/`target`/`aux`/`pred`, immutable copy-on-write mutators), dispatched +transforms on item TYPE via a kernel registry, batched via `typed_collate` (a batched `Sample`), +and plugged external libraries in through a **coercion registry of adapters** +(`register_adapter`/`coerce_transform` — a `Pipeline` wrapped each bare torchvision-v2 / +albumentations transform in an adapter object at composition time). -### Consequences +### What survived, and what was undone (2026-07-25) -- **Cross-field consistency is free** — one sampled decision flips image + mask + boxes together, - the thing a flat-metadata triple could not do. -- **Names and types work together**, so the "torchvision uses types / albumentations uses names" - split is resolved by one container: the key is the name, the item is the type. -- **The subpackage is `bag`, an internal module home** — the whole typed surface is imported from - the package top level (`from sampleflux import Sample, ...`), so the module layout is never in a - consumer's import path and can move without touching consumers. -- **Batching is `typed_collate`** — it returns a batched `Sample` (payloads stacked per field, - per-item attrs collected as lists, roles preserved); there is no `list[dict]` batch-in-metadata - form. -- **Deliberately deferred** (see root `TASKS.md`): a torch-`Tensor`-subclass item base (torch - payloads ride wrapper items for now), confluid-native item-type discovery, the generated - `Tv*`/`Alb*` families in this namespace, FluxStudio typed side sockets, and the `decode` path. +- **Survived into the record model:** typed items owning their metadata (the HYBRID + ndarray-subclass / dataclass-wrapper realization, `item_data`/`with_data`, `register_item`), the + once-per-record kernel dispatch (`sampleflux.dispatch`), and the item codec idea + (`sampleflux/io.py` — storage backends never inspect item internals). +- **Undone:** the `Sample` container (a plain dict now), role tags (key names carry meaning), + `primary()` (key addressing), `typed_collate` (→ `collate_records`), and the ENTIRE adapter plane + — coercion registry, adapter classes, `only=` per-key filters (→ `field=`) — replaced by the + engine-level op-family dispatch (`core._apply_op`), which calls each library natively instead of + wrapping it. -### Example +### Example (historical shape — no longer runs) ```python -from sampleflux import Sample, Image, Mask, Regions, Label, Pipeline -from torchvision.transforms import v2 -import albumentations as A - -sample = Sample( - {"image": Image(rgb), "mask": Mask(seg), "regions": Regions(boxes, canvas=(H, W)), "class": Label("drone_x")}, - roles={"mask": "target", "regions": "target", "class": "target"}, -) -out = Pipeline([ - v2.RandomHorizontalFlip(p=1.0), # Image + Mask + Regions together (one library draw) - v2.Normalize(m, s), # Image (torchvision v2, by type) — wrapped by a registered adapter - A.GaussNoise(p=1.0), # Image (albumentations, by name) — wrapped by a registered adapter -])(sample) -# image flipped+normalized+noised; mask+regions flipped consistently; out["class"] untouched. -# sampleflux ships NO native augmentation transforms — the libraries cover that via coercion. -# Signal-domain items + the Fourier transform live in the domain package and register into the -# same registries — a bare Fourier() drops into this Pipeline with no core edit. +sample = Sample({"image": Image(rgb), "regions": Regions(boxes)}, roles={"regions": "target"}) +out = Pipeline([v2.RandomHorizontalFlip(p=1.0), A.GaussNoise(p=1.0)])(sample) # adapter-coerced ``` -### What you may change (and where it's documented) +### What you may change -- **A new item type** — add a class + `@register_item` (usage: [typed-model.md](typed-model.md)); if - it is array-backed, subclass `NDArrayItem` and declare `_item_attrs`. -- **A new per-type behaviour for an existing transform** — register a kernel - (`@Transform.kernel(ItemType)`), no core edit. -- **The typed surface is imported from the package top level** — `bag/*` is the internal module - home; never teach a `sampleflux.bag.*` import path, so the module layout can change without - touching consumers. +Nothing — superseded. Extension points live in the successor record above. --- -## Native typed transforms that change a field's TYPE (`ConvertToImage`/`Threshold`/`ConnectedComponents`, 2026-07-22) +## ~~Native typed transforms that change a field's TYPE (`ConvertToImage`/`Threshold`/`ConnectedComponents`, 2026-07-22)~~ — SUPERSEDED -### Context +> **Superseded (2026-07-25)** by +> [One type-dispatched op engine — plain-dict records, libraries as-is](#one-type-dispatched-op-engine--plain-dict-records-libraries-as-is-2026-07-25), +> which promotes this record's core insight — the type-changing `__call__`-override op as the +> second sanctioned shape — to a rule of the data model itself. The ops survive (`ConvertToImage`: +> array → `Image`, `Threshold`: array → `Mask`, `ConnectedComponents`: `Mask` → `Regions`, +> plus the target ops) but now read/write plain record KEYS (`field=` in, `output=` out) — the +> role tags, the `Sample` shims, and the legacy-op delegation described below are gone. +> Kept as history — do not follow the role/shim details. + +### Context (historical) Two shapes of typed transform exist. The first is the augmentation shape the base `Transform` was built for: it `handles` an item type and, per handled field, applies a registered kernel that returns *the same type* (a flip returns a flipped `Image`), so `image`, `mask`, and `boxes` move -together and library transforms (torchvision v2 / albumentations) drop in through the adapter -coercion registry. The workspace deliberately ships **no** native transforms of that shape — -libraries cover it. - -But a running detection/segmentation front-end needs a different shape: **read one field, write a -field of a DIFFERENT type**. Turning a numeric array into a displayable image, thresholding an -array into a boolean mask, and labelling that mask into a set of bin boxes are each a *type -change* (`array → Image`, `array → Mask`, `Mask → Regions`), not an in-place per-type edit. No -library provides them, and the earlier ops that did (`ConvertToImage`, `ThresholdOp`, -`ConnectedComponentsOp`) operated on a flat `(input, target, metadata)` triple, which the typed -model does not carry. Without typed equivalents a `Sample` pipeline could not reach `Regions` from a -raw array — the critical path for typed detection was blocked. +together. But a running detection/segmentation front-end needs a different shape: **read one +field, write a field of a DIFFERENT type**. Turning a numeric array into a displayable image, +thresholding an array into a boolean mask, and labelling that mask into a set of bin boxes are +each a *type change* (`array → Image`, `array → Mask`, `Mask → Regions`), not an in-place +per-type edit. No library provides them. -### Decision +### Decision (historical) Add native typed **twins** that subclass `Transform` and OVERRIDE `__call__` (rather than register -a kernel), reading one field and writing a different-typed item — the same shape the domain -package's `Spectrogram` twin (`Signal → Spectrogram`) already established: - -- A twin declares `handles` / `consumes` / `produces` **truthfully** as graph metadata (e.g. - `ConnectedComponents`: `consumes=(Mask,)`, `produces=(Regions,)`), but does its work in - `__call__`, not through the kernel-dispatch loop — kernel dispatch is for same-type per-field - edits, and a type change has one input field and one output field. -- The source field is resolved by a small `_find_*` helper: an explicit `field=` name, else the - first item of the natural type (a `Mask` for `ConnectedComponents`) or the first array-bearing - item — every miss raises a `ValueError` naming the sample's fields. -- The output is written with `sample.replace_field(output, item)` + `sample.set_role(output, role)` - (copy-on-write), and the role is chosen semantically: the working image is `input`, a threshold - mask and raw connected-component boxes are `aux` (intermediates, and specifically NOT `pred` — - that role is reserved for a detector's output). -- Each twin **reuses its legacy op's math verbatim** so the numbers are pinned identical: - `ConvertToImage` calls the shared `_render_rgb`/`_bound_longest_side` render core; - `ConnectedComponents` calls the shared `connected_component_bboxes` helper; `Threshold` - delegates to a legacy `ThresholdOp` instance run on a shim `Sample`. The twins are STRICTLY - ADDITIVE — the legacy ops are untouched, because many consumers still use them via the `Sample` - path. - -The generic connected-components output format is a hard contract: `Regions.boxes` is a list of -`(row_min, row_max, col_min, col_max)` inclusive integer tuples (**row bounds first, then column -bounds**). A downstream back-projection reads exactly that order to map bins to a world / signal -coordinate frame, so the tuple order is load-bearing, not incidental. +a kernel), reading one field and writing a different-typed item; resolve the source field by an +explicit `field=` name or the first item of the natural type, with every miss raising a +`ValueError` naming the sample's fields; write the output with role tags chosen semantically; and +delegate each twin to its legacy op's math verbatim for byte-parity. -### Consequences +### What survived, and what was undone (2026-07-25) -- A `Sample` carrying a raw 2-D array runs `ConvertToImage → Threshold → ConnectedComponents` - end-to-end and arrives at a `Regions` field with no legacy `Sample` anywhere — the typed - detection/segmentation front-end is unblocked. -- Parity is free and provable: because each twin reuses the legacy math, a twin's output is - byte-identical to a legacy run on the equivalent `Sample` (pinned in - `tests/test_typed_generic_ops.py`). -- `ConvertToImage` does NOT republish `image_width_px` / `image_height_px` (the legacy op wrote - them into the shared metadata dict). The `Image` item's array SHAPE carries the pixel - dimensions, and the typed model has no shared dict to write into — a consumer reads the dims off - the payload. -- `Threshold`'s `{meta_key}` expression grammar has no typed home (an item owns its own metadata; - there is no shared sample dict), so only numeric literals and `$ENV` bounds resolve in the twin; - a `{key}` bound raises loudly. Literal dB thresholds — the critical path — are unaffected. -- The twins carry `category="op"` + `group="image"`/`"numpy"`, so they are discoverable exactly - like the legacy ops (their modules were already entry-pointed; a class added to a registered - module needs no new entry point). -- The two DETECTION-TARGET twins `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` - (`sampleflux/ops/target.py`, `group="structure"`) are the SAME shape reaching one step further: - they read one source field (a `Label` carrying a COCO `objects` mapping, or a `Mask`) and write - the torchvision detection target as a `Regions` item — `boxes` = the `[N,4]` xyxy tensor, - `labels` = the class-id tensor — tagged **`target`** (not `aux`: this IS the supervised target a - loss consumes, whereas `ConnectedComponents`'s raw blobs are an intermediate). `Regions` is the - natural typed home for a bounding-box set and the batch-friendly one — `typed_collate` gathers - per-sample `Regions` into a list of targets (the variable-N detection batch convention, since - boxes can't be stacked), exactly as it gathers a classification target `Label`. Byte-parity is - again free (each delegates to its legacy `*Op` on a shim `Sample`). Pinned in - `tests/test_typed_detection_target_ops.py`. +- **Survived:** the two-shapes rule; the `field=`-or-first-natural-type source resolution with loud + `ValueError` misses; the `(row_min, row_max, col_min, col_max)` inclusive integer bin-box + contract of `connected_component_bboxes` (**still load-bearing** — a downstream back-projection + reads exactly that order); truthful `consumes`/`produces` graph metadata. +- **Undone:** role tags on outputs (an op now writes a named `output` key — `Threshold`'s default + `output="mask"`, `ConvertToImage`'s `output="image"`); the legacy `(input, target, metadata)` ops + and the shim-`Sample` delegation (the legacy ops are deleted; the math lives in the shared free + functions `threshold_array` / `connected_component_bboxes` / `value_to_image`). -### Example +### Example (current successor shape) ```python -from sampleflux import Sample, Mask -from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.numpy import Threshold, ConnectedComponents - -sample = Sample({"spec": Mask(db_spectrogram)}) # a raw 2-D array item -sample = ConvertToImage()(sample) # + Image field (role "input") -sample = Threshold(field="spec", low_level=-30.0)(sample) # + Mask field (role "aux") -sample = ConnectedComponents(field="mask")(sample) # + Regions field (role "aux") +from sampleflux.ops.numpy import ConnectedComponents, Threshold -sample["boxes"].boxes # [(row_min, row_max, col_min, col_max), ...] — the pinned bin-box contract +record = Threshold(field="spec", low_level=-30.0)(record) # + record["mask"] (a Mask) +record = ConnectedComponents(field="mask")(record) # + record["regions"] (a Regions) ``` -### What you may change (and where it's documented) +### What you may change -- **A twin's source-field resolution or output role** — keep the `_find_*` → `replace_field` → - `set_role` shape and a loud `ValueError` on a miss; `aux` vs `pred` is a semantic choice - (raw detections are `aux`). -- **The `(row_min, row_max, col_min, col_max)` bin-box order is a contract** — a back-projection - depends on it; changing it is an architectural change that must update this record and every - consumer. -- **Do not modify the legacy ops or reimplement their math in a twin** — a twin reuses the legacy - math so parity is guaranteed; the twins are additive and the legacy `Sample`-path consumers must - keep working. +The bin-box tuple order remains a contract (see the successor record); everything else here is +history. -## A typed field cannot hold a live torch tensor — `ToTensor` stores CHW-float numpy (`ToTensor`/`EncodeTarget`/`DecodeTarget`/`MetadataToTarget`, 2026-07-22) +--- -### Context +## ~~A typed field cannot hold a live torch tensor — `ToTensor` stores CHW-float numpy (2026-07-22)~~ — SUPERSEDED (decision REVERSED 2026-07-25) -The typed detection twins above reach `Regions`; a typed CLASSIFICATION front-end needs the other -two shapes: turn the working image into the model's **input tensor**, and turn the class-name label -into the encoded **target id**. The earlier ops that did this (`ToTensor`, `MetadataToTargetOp`, -`EncodeTargetOp` / `DecodeTargetOp`) operated on a flat `(input, target, metadata)` triple. Two -facts of the typed model shape the twins: (1) there is NO shared metadata dict — the label already -rides a `Label` field that owns its metadata; (2) an array item is an `np.ndarray` SUBCLASS whose -`__new__` runs `np.asarray(data)`, so **a field payload is coerced to numpy** — an `Image` cannot -hold a live `torch.Tensor` (verified: `item_data(Image(tensor))` is an `ndarray`), and a bare tensor -stored directly as a field value has no registered item type, so `typed_collate` / the storage codec -(`bag.io.encode_item`) cannot serialize it. +> **Superseded (2026-07-25, user decision)**: the constraint below was a TYPED-BAG artifact — +> every field had to be a typed item, and an `NDArrayItem` coerces its payload through +> `np.asarray`, so a live tensor could not ride a field. In the RECORD model a value can be +> ANYTHING (the `"plain"` codec tag covers storage, `collate_records._stack` stacks torch +> tensors natively, a bare torchvision-v2 op transforms them as-is), so **`ToTensor` now writes +> the LIVE CHW-float `torch.Tensor` under the key** (in place by default, `output=` for a new +> key) — no numpy round-trip, and the op's name is again the truth. `Image` itself still cannot +> hold a tensor (it IS an ndarray subclass); the torch-`Tensor`-subclass ITEM base (a typed +> tensor value with attrs) remains the documented follow-up (root `TASKS.md`). -### Decision +### Context (historical) -Add native typed twins subclassing `Transform` and overriding `__call__` (the same shape as the -detection twins), each reusing its legacy op VERBATIM on a shim `Sample` for byte-parity: - -- **`ToTensor`** (`ops/torch.py`, `group="torch"`) resolves an array-bearing field (explicit `field` - or the first array/PIL item), runs `ToTensor` (HWC→CHW + `normalize`), and writes an `Image` - with `layout="CHW"`. Because `NDArrayItem` coerces the payload, the stored value is a CHW `float32` - **numpy** array whose values equal `ToTensor(...).input.numpy()` — NOT a live tensor. By default - it REPLACES the source field in place so the field's `input` role is preserved (`output` writes a - new field tagged `input` instead). `typed_collate` stacks these payloads with `np.stack`; the - numpy→tensor conversion is the collate / model boundary's job, exactly as for any numpy dataset. A - Tensor-subclass item that would let a field carry a live tensor is the documented follow-up - (`bag/items.py` note + root TASKS.md). -- **`EncodeTarget` / `DecodeTarget`** (`ops/target.py`, `group="structure"`) resolve a `Label` field, - map its `.value` through the config-pinned `mapping` by delegating to `EncodeTargetOp` / - `DecodeTargetOp` (so the non-empty-mapping validation AND the shared `_lookup` are byte-identical), - and write a new `Label` (carrying the source label's `classes`) tagged `target`. In place by - default (`output` blank). -- **`MetadataToTarget`** is provided for PARITY / config-compat but is largely REDUNDANT in the typed - model: a source emits the label directly as a `Label` field already tagged `target`, so no - metadata→target move is needed. The twin reads a field's natural value (a `Label`'s `.value`, else - its array payload) or a named attribute (`key=`) and writes a target `Label` — the escape hatch for - a label that rode as another item's attribute. +A typed classification front-end needs to turn the working image into the model's input tensor and +the class-name label into the encoded target id. Two facts shape the ops: (1) there is no shared +metadata dict — the label already rides a `Label` value that owns its metadata; (2) an array item +is an `np.ndarray` SUBCLASS whose `__new__` runs `np.asarray(data)`, so **a payload is coerced to +numpy** — an `Image` cannot hold a live `torch.Tensor`, and a bare tensor stored directly has no +registered item type for the collate / storage codec. -### Consequences +### Decision (historical, largely still in force) -- A `Sample` carrying an HWC `Image` (role input) + a name `Label` (role target) runs - `ToTensor → EncodeTarget` into a CHW-float input field + an int-id target field, with no legacy - `Sample` anywhere — the typed classification front-end is unblocked. -- The model-input payload is CHW-float **numpy**, not a live `torch.Tensor`; a consumer / trainer - tensorizes at the collate or forward boundary. This is a deliberate current limitation, not a bug — - it disappears when the Tensor-subclass item lands. -- The twins carry `category="op"` + the legacy `group`, so they are discoverable like the legacy ops - (their modules — `sampleflux-ops-torch` / `sampleflux-ops-target` — are already entry-pointed; a - class added to a registered module needs no new entry point). +`ToTensor` resolves an array-bearing key, runs the HWC→CHW + `normalize` conversion, and writes an +`Image(layout="CHW")` whose payload is CHW `float32` numpy — NOT a live tensor; in place by +default so the working key keeps its name. `EncodeTarget` / `DecodeTarget` map a `Label`'s value +through a config-pinned `mapping` and write the encoded `Label` back (carrying the source label's +`classes`). `MetadataToTarget` stays as the escape hatch for a label that rode as another value's +attribute — largely redundant when a source emits the label as a `Label` under its own key. -### Example +### Example (current successor shape) ```python -from sampleflux import Sample, Image, Label -from sampleflux.ops.torch import ToTensor from sampleflux.ops.target import EncodeTarget +from sampleflux.ops.torch import ToTensor -sample = Sample( - {"image": Image(hwc_uint8), "class": Label("cat")}, - roles={"image": "input", "class": "target"}, -) -sample = ToTensor(field="image")(sample) # image -> CHW float32 Image (role input, in place) -sample = EncodeTarget(mapping={"cat": 0, "dog": 1}, field="class")(sample) # class -> Label(0) (role target) +record = {"image": Image(hwc_uint8), "class": Label("cat")} +record = ToTensor(field="image")(record) # record["image"] is now a LIVE CHW float32 torch.Tensor +record = EncodeTarget(mapping={"cat": 0, "dog": 1}, field="class")(record) # Label(0), classes kept ``` -### What you may change (and where it's documented) +### What you may change -- **The Tensor-subclass item follow-up** — once a field can carry a live tensor, `ToTensor` should - store it directly; update this record and the `bag/items.py` note together. -- **`ToTensor`'s replace-in-place default vs a new output field** — keep role preservation (in place) - as the default; a new `output` field is tagged `input`. -- **Do not modify the legacy ops or reimplement their math in a twin** — the twins delegate to the - legacy ops for byte-parity and are strictly additive. +- **The Tensor-subclass item follow-up** — the tensor currently rides as a PLAIN value (no item + attrs); a torch-`Tensor`-subclass item base would make it a typed value with metadata again. + Update this record and the `sampleflux/items.py` note together when it lands. diff --git a/docs/augmentation.md b/docs/augmentation.md index 49d57a0..36c1a4b 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -1,129 +1,120 @@ -# Augmentation — well-known libraries as SampleFlux ops - -SampleFlux does not reimplement augmentations. Two adapter ops wrap the established -libraries — and a **generated op family** turns every individual library transform into -its own first-class op: - -| Surface | What it is | Example | -|---|---|---| -| `sampleflux.ops.albumentations.AlbumentationsOp` | Adapter running one/many [albumentations](https://albumentations.ai) transforms | `AlbumentationsOp(transforms=[...], target="mask", seed=0)` | -| `sampleflux.ops.torchvision.TorchvisionTransformOp` | Adapter running one/many torchvision `transforms.v2` transforms | `TorchvisionTransformOp(transforms=[...], target="mask")` | -| `sampleflux.ops.albumentations_transforms` | **Auto-generated**: one `Alb` op per albumentations transform (~115) | `AlbHorizontalFlip(p=0.5, target="mask")` | -| `sampleflux.ops.torchvision_transforms` | **Auto-generated**: one `Tv` op per v2 transform (~55) | `TvRandomHorizontalFlip(p=0.5, target="mask")` | - -All are ordinary sample-scoped ops (`__call__(sample)`): they chain in a `Flux` ops list, -inside `TransformChain` / `RandomApply` / `Enable`, in Confluid YAML, and as individual -nodes on a visual canvas (palette groups `augment`, `augment/albumentations`, -`augment/torchvision`). One library draw applies jointly to the input-role field and — per the -`target` mode — its mask / boxes; other fields pass through untouched. - -Torchvision requires the `vision` extra: `pip install "sampleflux[vision]"` -(albumentations is a core dependency; without torchvision the `Tv*` family is simply -empty and everything else works). - -## Target modes - -The `target` knob is a closed `Literal["none", "mask", "boxes"]` on every op above: - -- `"none"` (default) — input-only augmentation (color jitter, noise, blur); the target-role - field passes through untouched. -- `"mask"` — the target-role field is a segmentation mask (2-D array or PIL `L` image); image - and mask receive the SAME spatial transform. -- `"boxes"` — the target-role field is the torchvision detection target - `{"boxes": [N,4] xyxy-pixel, "labels": [N]}` — exactly what `CocoToTorchVisionDetectionOp` - and `MasksToDetectionBoxesOp` emit — and boxes move with the image. The required - albumentations `bbox_params` are added automatically when the op builds the Compose; - only a prebuilt `A.Compose` must carry its own. +# Augmentation — well-known libraries run AS-IS + +SampleFlux does not reimplement augmentations, and it does not wrap them either. A bare +[albumentations](https://albumentations.ai) transform or a bare torchvision `transforms.v2` +transform drops **as-is** into any ops list — `Flux(ops=[...])`, a `Pipeline`, a `flow:` step, +inside `RandomApply` / `Enable` — and the engine's op-family dispatch +(`sampleflux.core._apply_op`) invokes it the way its own library expects. There are no adapter +classes and no generated per-transform op families. ```python import albumentations as A -from sampleflux import Flux -from sampleflux.ops.albumentations import AlbumentationsOp -from sampleflux.ops.albumentations_transforms import AlbRandomBrightnessContrast - -flux = Flux(source=samples, ops=[ - AlbumentationsOp( # several transforms, one op - transforms=[A.HorizontalFlip(p=0.5), A.Affine(translate_percent=0.1, p=1.0)], - target="mask", seed=0, - ), - AlbRandomBrightnessContrast(p=0.5), # or one generated op per transform +from torchvision.transforms import v2 +from sampleflux import Flux, Pipeline + +flux = Flux(source=records, ops=[ + A.HorizontalFlip(p=0.5), # bare albumentations + A.GaussNoise(p=1.0), # bare albumentations + my_native_op, # native sampleflux op — same list ]) + +Pipeline([v2.ToImage(), v2.RandomCrop(8)])(record) # bare torchvision v2 ``` -## YAML — Confluid-native, both directions +Torchvision is optional (`pip install "sampleflux[vision]"`); albumentations is a core +dependency. The family check is by MRO module name — neither library is imported until you +actually put one of its transforms in a pipeline. + +## How each family is invoked + +- **albumentations** dispatches by KWARG NAME: the op receives exactly its own target keys + present in the record — `image` / `mask` / `masks` / `bboxes` / `keypoints` / `labels` — and + nothing else, so extra record entries (scalars, domain items) never reach a library that would + reject them. One call = **one joint draw** across those keys: image, mask and boxes move with + the same decision. Array outputs are re-wrapped in the incoming value's item type, so an + `Image` / `Mask` keeps its type and metadata through the library. A record with none of the + known keys passes through untouched (logged at debug). +- **torchvision `transforms.v2`** natively walks dicts: the op is called on the record as-is, + samples its parameters once, transforms tensor / tv_tensor / PIL leaves and passes everything + else (labels, scalars) through. +- **everything else** is a native/wiring op `record -> Optional[Record]` (`None` drops the + record). -Transforms are ordinary nested `!class:` nodes (dotted paths or registered short names) — -no library-specific serialization formats. `confluid.dump` round-trips both forms. +## The key vocabulary — and routing into it + +Key names carry meaning: albumentations sees only its own vocabulary, so a value augments only if +it rides one of those keys. If your pipeline produced the value under another name, route it with +`RenameField` (`sampleflux.ops.structure`) before the library op: ```yaml -# Adapter with a transforms list (dotted library paths): -- !class:sampleflux.ops.albumentations.AlbumentationsOp - target: mask - seed: 0 - transforms: - - !class:albumentations.HorizontalFlip - p: 0.5 - - !class:albumentations.Affine - translate_percent: 0.1 - -# Generated per-transform ops (registered short names): -- !class:AlbHorizontalFlip - p: 0.5 - target: mask -- !class:TvRandomHorizontalFlip - p: 0.5 - target: mask +ops: + - !class:sampleflux.ops.structure.RenameField {src: spec_view, dst: image} + - !class:albumentations.GaussNoise + p: 1.0 +``` + +## Boxes: use the library's own Compose + +Box-carrying augmentation is albumentations' `Compose` job — drop a prebuilt `A.Compose` with its +own `bbox_params` into the ops list (the record supplies `bboxes` + `labels` under exactly those +keys): + +```python +import albumentations as A + +flip = A.Compose( + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), +) +out = Pipeline([flip])(record) # image + mask + bboxes flipped together, one draw ``` -## The generated op families - -`sampleflux.ops._augment_bridge` walks each library's public transform classes at import -time and generates one op per transform (the waivefront-helios auto-bridge pattern): a -subclass of the adapter whose constructor mirrors the transform's own parameters (plus -`target` / `seed`), with a synthesized signature and `Args:` docstring so form-specs, -MCP schemas, and canvas widgets see the real parameters. - -- The `Alb` / `Tv` name prefixes are MANDATORY: the confluid registry is flat and - name-keyed, and the two libraries share many bare names (`ColorJitter`, `Normalize`, - `Resize`, …). -- Zero-arg construction always works; a transform's required parameter (e.g. - `AlbRandomCrop.height`) surfaces lazily as the library's own missing-argument error on - first call. -- Composition/container transforms (`Compose`, `OneOf`, v2 `RandomApply`, …) are NOT - generated — chaining ops is native SampleFlux (`ops:` lists, `TransformChain`, - `RandomApply`). -- A generated op wired into an adapter's `transforms` list unwraps to its inner library - transform (`raw_transform`), so canvas graphs can feed transform nodes into one - Compose-style adapter node too. +Format handling (`pascal_voc` / `coco` / `yolo` / `albumentations`) is `BboxParams`' knob — the +engine adds nothing on top. The detection-target ops (`CocoToTorchVisionDetection` / +`MasksToDetectionBoxes`) produce a `Regions` item for the training boundary; the plain +`bboxes`/`labels` list keys are the augmentation-time form the library consumes. + +## YAML — bare library transforms are ordinary `!class:` nodes + +No library-specific serialization format — a transform is a Confluid `!class:` node like any op, +in mapping form or call form. `Flux` flows deferred markers at route entry, and composing ops +(`Pipeline` / `Enable` / `RandomApply`) flow theirs lazily: + +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.GaussNoise {p: 1.0} + - !class:sampleflux.ops.numpy.Threshold + low_level: 0.5 +``` ## Layout contract (the main footgun) -The two libraries disagree about layout, and the ops keep each library's native -convention instead of hiding it: +The two libraries disagree about layout, and the engine keeps each library's native convention +instead of hiding it — **conversions are always explicit library transforms, never silent**: -- **albumentations** (`AlbumentationsOp`, `Alb*`) consumes numpy **HWC** (PIL converts on - entry) and emits numpy HWC — put it BEFORE `ToTensor` in the chain. -- **torchvision** (`TorchvisionTransformOp`, `Tv*`) emits **CHW torch tensors** (numpy - HWC converts on entry, PIL passes through as PIL) — no `ToTensor` needed after it. +- **albumentations** consumes and emits numpy **HWC** — run it while your values are still numpy + arrays (an `Image`/`Mask` is an ndarray subclass, so it feeds straight in). +- **torchvision v2** wants **CHW tensors** — put the library's own `v2.ToImage()` (numpy HWC → + CHW tv_tensor) in the list first, then any v2 transform; exactly like a plain torchvision + pipeline. -Don't chain one library's output straight into the other without accounting for this. +Don't chain one library's output straight into the other without an explicit conversion step. ## Randomness & seeding -All augmentation ops carry `random=True` (the confluid stochastic mark). Stochasticity -lives where each library puts it: +Stochasticity lives where each library puts it — the engine adds no seed plumbing: -- albumentations: the `seed` knob (maps onto `A.Compose(seed=N)`); a prebuilt - `A.Compose` carries its own seed instead. +- albumentations: `A.Compose(seed=N)` on a prebuilt Compose (individual transforms keep their own + `p`). - torchvision v2: the global torch RNG — `torch.manual_seed(N)`. -- per-sample gating: wrap in `RandomApply(op=..., probability=..., random_state=N)` - (each albumentations transform also carries its own `p`). +- per-record gating of any op (native or library): `RandomApply(op=..., probability=..., + random_state=N)`. -## Examples +## Example -- [`examples/augmentation_ops.py`](../examples/augmentation_ops.py) — the tour: all - three target modes, cross-library parity, boxes mirroring, target-side encoding, the - generated op families, gated composition, and the Confluid-native YAML round-trip. -- [`examples/augmentation_training.py`](../examples/augmentation_training.py) — end to - end: synthetic images+masks → joint geometric + gated photometric augmentation → - `DataLoader` (registry collate) → a tiny CNN trained for 3 epochs with improving loss. +[`examples/record_pipeline.py`](../examples/record_pipeline.py) — the tour: a bare +`A.Compose` with `bbox_params` + `A.GaussNoise` + a native type-dispatched op in ONE `Pipeline` +(image/mask/bboxes moved jointly, types preserved), `field=` pinning, and torchvision v2 as-is +after an explicit `v2.ToImage()`. diff --git a/docs/configure.md b/docs/configure.md index 1f5c427..eb3259f 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -1,8 +1,8 @@ -# Per-sample op parameters (`ConfigureOp` / `Apply` / `Capture`) +# Per-record op parameters (`ConfigureOp` / `Apply` / `Capture`) -Some op parameters are only known *per sample*. Two mechanisms cover this: +Some op parameters are only known *per record*. Two mechanisms cover this: -- **`ConfigureOp(ops, target, param, key)`** — runs `ops` on the sample as a side-branch; the chain's final primary input value (`primary(sample, "input")`) is injected as `target.` and also recorded as an `aux`-role field named `key`, then `target` is applied. Use it when the value is *derived from the sample itself* (e.g. a threshold from the sample's own max) — the whole derivation reads as one node/YAML block. +- **`ConfigureOp(ops, target, param, source)`** — runs the `ops` compute-chain on the record as a SIDE branch (its transformations are discarded — the original record continues); the `source`-keyed entry of the chain's final record becomes the VALUE (payload-unwrapped via `item_data`), which is set as the `param` attribute of `target` — post-construction configuration, the confluid paradigm — and then `target` is applied to the original record. Use it when the value is *derived from the record itself* (e.g. a threshold from the record's own max) — the whole derivation reads as one node/YAML block. - **`Capture` + `Apply`** (`sampleflux.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. ```yaml @@ -12,24 +12,24 @@ ops: op: !class:mypackage.ops.AugmentOp {} # any op exposing a confluid @output output: applied_level name: __captured_level - # …then inject the captured value into a later op's parameter per sample. + # …then inject the captured value into a later op's parameter per record. - !class:sampleflux.ops.context.Apply op: !class:mypackage.ops.CompensateOp {} param: level source: __captured_level ``` -A self-contained `ConfigureOp` example — derive a per-sample threshold from the sample's own statistics: +A self-contained `ConfigureOp` example — derive a per-record threshold from the record's own statistics: ```yaml ops: - !class:sampleflux.ops.configure.ConfigureOp ops: - - !class:sampleflux.ops.numpy.MaxOp {} - - !class:sampleflux.ops.formula.FormulaOp { formula: "a * 0.5" } - target: !class:sampleflux.ops.numpy.ThresholdOp {} + - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "a.max() * 0.5"} + source: image + target: !class:sampleflux.ops.numpy.Threshold + low_op: ">=" param: low_level - key: derived_threshold ``` -`ConfigureOp` also records the derived value as an `aux`-role field named `key` (traceability — it persists into a sink); `Capture`/`Apply` move values through the per-sample Context, which never alters the sample's fields. +Both mechanisms leave the record's own entries untouched: `ConfigureOp`'s compute chain runs on a side-branch copy, and `Capture`/`Apply` move values through the per-record Context. All inner ops (the compute chain, `target`, the wrapped ops of `Capture`/`Apply`) are applied through the engine's op-family dispatch, so a bare library transform works in any of these slots too. diff --git a/docs/graph.md b/docs/graph.md index 02bbbcf..1e5ec23 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -6,24 +6,23 @@ The **readable authoring form** of a graph pipeline is a `flow:` document — na ```yaml flow: - scaled: !class:sampleflux.ops.numpy.RescaleOp() # input: the source sample - norm: !class:sampleflux.ops.numpy.StandardizeOp() # input: previous step - mask: !class:sampleflux.ops.numpy.ThresholdOp(low_level=0.5) {from: scaled} # 2nd reader of `scaled` = fan-out - thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: norm} - gated: !class:sampleflux.ops.numpy.ThresholdOp() - from: scaled - bind: {low_level: thresh} # per-sample param := thresh's result - out: {from: gated, target_from: mask} # pure fan-in (no op) + spec: !class:mypkg.MakeSpectrogram() # input: the source record + masked: !class:sampleflux.ops.numpy.Threshold(low_level=0.5) {from: spec} # 2nd reader of `spec` = fan-out + thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5", field=spec) {from: spec} + gated: !class:sampleflux.ops.numpy.Threshold() + from: spec + bind: {low_level: thresh[spec]} # per-record param := the `spec` entry of thresh's result + out: {from: gated, merge_from: [masked]} # fan-in (no op) outputs: out ``` -Step grammar (four reserved keys, stripped before the op is built): +Step grammar (three reserved keys, stripped before the op is built): - **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document order is the schedule and cycles are inexpressible). -- **`target_from:` / `metadata_from:`** — fan-in slots (a step result contributes its corresponding field; metadata merges last-write-wins). -- **`bind:`** — `{param: step}` per-sample parameters (a step name = its result's `input`; `step.attr` = the step op's live `@output`, stochastic-correct). +- **`merge_from:`** — fan-in: UNION the named steps' record ENTRIES into this step's incoming record, in listed order, last-write-wins on a key collision (the `MergeFields` slot semantics). +- **`bind:`** — `{param: ref}` per-record parameters: a bare `step` binds the step's WHOLE result record, `step[key]` the named ENTRY of its record, and `step.attr` the step op's live `@output` (lowered through `Capture` — stochastic-correct). -A plain-mapping step with no op (`out: {from: a, target_from: b}`) is a pure fan-in; `{}` is the identity (names the source). Cell lifetimes are **automatic** in both forms. +A plain-mapping step with no op (`out: {from: a, merge_from: [b]}`) is a pure fan-in; `{}` is the identity (names the source). Cell lifetimes are **automatic** in both forms. Steps apply their ops through the engine's op-family dispatch, so bare library transforms sit in flow steps too. Two engines, one contract — **bidirectional conversion with execution parity**: @@ -37,30 +36,32 @@ ops = to_ops(graph.steps, graph.output_step) # flow -> flat ops flow2 = from_ops(ops) # flat ops -> flow (lifting) ``` -`FlowGraph` is a `torch.utils.data.Dataset` like `Flux` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Flux's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. See `examples/flow_graph.py` for the full round-trip. +`FlowGraph` is a `torch.utils.data.Dataset` like `Flux` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Flux's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. ## Graph pipelines on a flat op list (Context ops) -A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-sample **`Context`** (a named-cell store, `sampleflux.context`) around each sample's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never mutates the sample's fields — a linear run's fields stay byte-identical whether or not context threading exists. +A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-record **`Context`** (a named-cell store, `sampleflux.context`) around each record's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never mutates the record's entries — a linear run's record stays byte-identical whether or not context threading exists. | Op | Semantics | |---|---| -| `Save(name)` | snapshot the stream sample into a cell (pass-through) — the fork point | +| `Save(name)` | snapshot the stream record into a cell (pass-through) — the fork point | | `Use(name, drop=False)` | stream := the cell's value; deep-copies unless `drop` frees the cell (move) | | `Drop(names)` | free cells explicitly | -| `Apply(op, param, source, drop=False)` | set `op.` from a cell's value, then apply `op` | +| `Apply(op, param, source, key="", drop=False)` | set `op.` from a cell (a record cell contributes its `key`-named entry, or the whole record when `key` is blank; a raw cell value verbatim), then apply `op` | | `Capture(op, output, name)` | apply `op`, record its live `@output` into a cell (stochastic-correct) | -| `Mix(input_from, target_from, metadata_from, drop)` | fan-in: compose a sample from cells + the incoming sample | +| `MergeFields(sources, keys, drop)` | fan-in: UNION the named cells' entries into the incoming record (listed order, last-write-wins; `keys` restricts the union) | ```yaml ops: - !class:sampleflux.ops.context.Save(name=fork) # fork the stream - - !class:sampleflux.ops.numpy.StandardizeOp() # branch A rides the stream + - !class:albumentations.GaussNoise {p: 1.0} # branch A rides the stream - !class:sampleflux.ops.context.Save(name=branch_a) - !class:sampleflux.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork - - !class:sampleflux.ops.numpy.ThresholdOp + - !class:sampleflux.ops.numpy.Threshold low_level: 0.5 - - !class:sampleflux.ops.context.Mix(target_from=branch_a) # fan-in + - !class:sampleflux.ops.context.MergeFields # fan-in + sources: [branch_a] + keys: [image] drop: [branch_a] ``` @@ -71,12 +72,12 @@ from sampleflux.context import Context, activate with activate(Context()): for op in ops: - sample = op(sample) + record = op(record) ``` -Cells hold whole `Sample`s (from `Save`) or raw values (from `Capture`); `Apply` reads a Sample cell's primary input item, `Mix` reads each cell's corresponding field. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-sample store instead of extra fields on the sample (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#the-per-sample-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). +Cells hold whole records (from `Save`) or raw values (from `Capture`); `Apply` reads a record cell's `key`-named entry (whole record when `key` is blank), `MergeFields` unions each cell's entries. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). On a deliberate key collision at the fan-in, rename on the producing branch first (`RenameField`, `sampleflux.ops.structure`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-record store instead of extra record keys (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). -> **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the field into its own `aux`-role field with the structure ops (`CopyField` + `SetRole`, `sampleflux.ops.structure`); the snapshot then rides the sample as a real field. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. +> **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the value under its own key with `CopyField` (`sampleflux.ops.structure`); the snapshot then rides the record as a real entry. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. ## Reattach an ops-only YAML (`Flux.from_ops_yaml`) diff --git a/docs/image.md b/docs/image.md index a8a0a48..b5d1a8e 100644 --- a/docs/image.md +++ b/docs/image.md @@ -5,28 +5,29 @@ The single, modality-agnostic "any value → image" layer — generic so every c ```python from sampleflux.ops.image import ConvertToImage, value_to_image -# Op: the sample's primary input item (2-D map / CHW tensor / PIL / bool mask) -> an Image field. +# Op: an array-bearing record value (2-D map / CHW tensor / PIL / bool mask) -> an Image item. op = ConvertToImage( colormap="viridis", # closed `Colormap` Literal -> enumerable in GUIs / schemas width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top + field="spec", # source key; blank picks the first array-bearing value + output="image", # key the HWC-uint8 Image item is written to ) -sample = op(sample) # writes an Image field; the pixel dimensions live in its array shape +record = op(record) # adds record["image"]; the pixel dimensions live in its array shape # Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): rgb = value_to_image(some_value, colormap="magma", max_size=512) -# NormalizeToUint8Op: the standalone min-max value -> uint8 quantization step +# normalize_to_uint8: the standalone min-max value -> uint8 quantization step # (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; -# set them to pin a fixed scale across samples (out-of-range values clamp). -from sampleflux.ops.image import NormalizeToUint8Op +# set them to pin a fixed scale across records (out-of-range values clamp). +from sampleflux.ops.image import normalize_to_uint8 -sample = NormalizeToUint8Op()(sample) # auto per-array min/max -sample = NormalizeToUint8Op(vmin=-80.0, vmax=0.0)(sample) # fixed dB window across a dataset -u8 = NormalizeToUint8Op.normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # the backing @staticmethod +u8 = normalize_to_uint8(arr) # auto per-array min/max +u8 = normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # fixed dB window across a dataset ``` -`Colormap` / `COLORMAPS` / `value_to_image` / `sample_to_image` are re-exported from `waivefront.visualizers` for backward compatibility. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). +`sample_to_image(record, ...)` renders a record's first array-bearing (2-D / 3-D) value the same way — the ad-hoc whole-record preview for viewer tooling. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). ## Introspection helpers diff --git a/docs/kinds.md b/docs/kinds.md index e80e4d9..894665f 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -1,55 +1,54 @@ -# Transforms, batching & expanding ops (`sampleflux.bag` / `sampleflux.collate`) +# Ops, batching & expanding ops (`sampleflux.transform` / `sampleflux.collate`) -## What a transform processes — dispatch on item type +## What an op processes — dispatch on value type -A **sample** is a named bag of typed items (`Image`, `Mask`, `Regions`, `Label`, … — see [typed-model.md](typed-model.md)). A transform declares which item TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per sample, then applies the matching kernel to every field whose item type it handles, passing untouched fields through: +A **sample** is a plain record dict of typed values (`Image`, `Mask`, `Regions`, `Label`, … — see [record-model.md](record-model.md)). A native op is a `Transform`: it declares which value TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per record (`get_params`), then applies the matching kernel to every value whose type it handles, passing untouched values through: ```python -from sampleflux import Transform, Image +from sampleflux import Record, Transform, Image class Recenter(Transform): - handles = (Image,) # which item types this transform touches + handles = (Image,) # which value types this op touches - def params(self): # sampled ONCE per sample, shared across fields - return {"mean": 0.5} + def get_params(self, record: Record) -> dict: + return {"mean": 0.5} # sampled ONCE per record, shared across values @Recenter.kernel(Image) # per-type behaviour -def _(item, params): - return item - params["mean"] +def _(value, params): + return value - params["mean"] ``` -Because the parameters are sampled once and shared, a transform that handles several types moves those fields **consistently** — one flip decision applies to `Image`, `Mask` and `Regions` together, the thing a flat `(input, target, metadata)` triple could not express. Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them. +Because the parameters are sampled once and shared, an op that handles several types moves those values **consistently** — one drawn decision applies to every handled value in the record. Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them. Two smaller shapes round it out: -- **A plain function** becomes a transform via `as_transform(fn, handles=(Image,), only=["image"])` — `only=` narrows a transform to specific field keys. -- **A type-changing transform** — read one field, write a differently-typed item (`array → Image`, `Signal → Spectrogram`, `Mask → Regions`) — subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. +- **A plain function** becomes an op via `as_transform(fn, handles=(Image,), field="image")` — `field=` pins the op to one named key (still type-gated). +- **A type-changing op** — read one key, write a differently-typed item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: `Mask` → `Regions`) — subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. -Bare library transforms (torchvision `transforms.v2` dispatching by type, albumentations by keyword name) drop straight into a `Pipeline` through registered adapters — each one hits only the field(s) it handles. See [typed-model.md](typed-model.md#mixing-libraries--one-pipeline-many-worlds). +Bare library transforms (torchvision `transforms.v2` walking the dict natively, albumentations dispatching by keyword name) drop straight into any ops list **as-is** — the engine's op-family dispatch invokes each one the way its own library expects. See [record-model.md](record-model.md#mixing-libraries--as-is-no-adapters) and [augmentation.md](augmentation.md). ```python -from sampleflux import Sample, Image, Mask, Regions, Label, Pipeline -from torchvision.transforms import v2 import albumentations as A +from sampleflux import Pipeline out = Pipeline([ - v2.RandomHorizontalFlip(p=1.0), # Image + Mask + Regions together (one library draw) - v2.Normalize(mean, std), # Image only — wrapped by a registered adapter - A.GaussNoise(p=1.0), # Image only — wrapped by a registered adapter -])(sample) -# a Label field is untouched (no kernel handles it); roles are preserved. + A.HorizontalFlip(p=1.0), # image + mask + bboxes together (one library draw) + A.GaussNoise(p=1.0), # image only — its own kwarg vocabulary + Recenter(), # native op — same list +])(record) +# record["class"] (a Label) is untouched: no kernel handles it, no library key names it. ``` -## Batching — `typed_collate` & the collate registry (`sampleflux.collate`) +## Batching — `collate_records` & the collate registry (`sampleflux.collate`) -Transforms are per-sample; batching is a separate stage. **`typed_collate`** (auto-dispatched for `Sample` batches) stacks each field's payload and collects each item's per-sample attributes into a list, preserving roles — the ONE batch convention: +Ops are per-record; batching is a separate stage. **`collate_records`** (the registry's `"record"` default) stacks N record dicts into ONE batched record: per key, typed payloads stack (torch → stacked tensor, numpy → stacked array, else a list) and each item's declared attrs become per-record lists, decoded back into one batched item of the same type; plain values batch as plain lists. Batches must carry the same keys — a mismatch raises. ```python -from sampleflux import typed_collate +from sampleflux import collate_records from torch.utils.data import DataLoader -batch = typed_collate(list(flux)) # a batched Sample: payloads stacked per field -loader = DataLoader(flux, collate_fn=typed_collate) +batch = collate_records(list(flux)) # ONE batched record: payloads stacked per key +loader = DataLoader(flux, collate_fn=collate_records) ``` Collation is a pluggable registry keyed by name, so a task can register its own convention additively: @@ -62,24 +61,28 @@ def yolo_collate(items): ... loader = DataLoader(flux, collate_fn=get_collate("yolo")) ``` -The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17). +The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17-updated-2026-07-25). ## 1→N expanding ops (iterable-only pipelines) -An op may return **several** carriers — a windowing op splitting one capture into N windows is just a generator-returning op: +An op may return **several** carriers — a windowing op splitting one capture into N windows marks itself with `EXPANDS = True` and returns an iterable of records: ```python from typing import Iterator -from sampleflux import Sample, Transform, primary, with_data - -@configurable -class SlidingWindowOp(Transform): - def __call__(self, sample: Sample) -> Iterator[Sample]: - key, item = primary(sample, "input") - for w in sliding_windows(item, self.size, self.stride): - yield sample.replace_field(key, with_data(item, w)) +from confluid import configurable +from sampleflux import Record +from sampleflux.items import item_data, with_data + +@configurable(category="op") +class SlidingWindow: + EXPANDS = True # the explicit 1→N marker + + def __call__(self, record: Record) -> Iterator[Record]: + item = record["signal"] + for w in sliding_windows(item_data(item), self.size, self.stride): + yield {**record, "signal": with_data(item, w)} ``` -Expansion is detected from the return annotation (`Iterator[...]` / `Iterable[...]` / `List[...]`; or the explicit `EXPANDS = True` marker) and flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. +Expansion is flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(flux)` / `flux[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(flux)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Flux` engine. diff --git a/docs/projection.md b/docs/projection.md index b2bc582..558a522 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -1,45 +1,38 @@ -# Field projection, class counting & label maps (`sampleflux.projection` / `sampleflux.labels`) +# Key projection, class counting & label maps (`sampleflux.projection` / `sampleflux.labels`) -## Field projection +## Key projection -Walking a source for a single field (the classic case: counting classes from *targets*) shouldn't pay to build the fields you don't need. `sampleflux.projection` adds an opt-in protocol plus lazy helpers: +Walking a source for a single record key (the classic case: counting classes from the label key) shouldn't pay to build the values you don't need. `sampleflux.projection` adds an opt-in protocol plus lazy helpers, all **key-addressed** — any subset of record keys: ```python -from sampleflux import project, iter_targets, num_classes -from sampleflux import ProjectionField # Literal["input", "target", "metadata"] +from sampleflux import project, iter_key, num_classes -# A source MAY implement SupportsProjection (`project(fields)`) to skip building -# unrequested fields — e.g. an image dataset reads only the label column for a -# target-only walk, never decoding an image. -for sample in project(my_source, ("target",)): - ... # only target-role fields are built; input-role fields are skipped +# A source MAY implement SupportsProjection (`project(keys)`) to skip building +# unrequested values — e.g. an image dataset reads only the label column for a +# class-count walk, never decoding an image. +for record in project(my_source, ("class",)): + ... # partial records carrying only the "class" entry -labels = list(iter_targets(my_source)) # lazy -n = num_classes(my_source) # max(class_id) + 1 — always walks +labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, + # other items to their payload, plain values verbatim +n = num_classes(my_source, key="class") # max(class_id) + 1 — always walks ``` -The field set is a **closed `Literal`**, `ProjectionField`, not a bare `str` — so a typo is a type error, and a UI / form-spec / MCP schema enumerates the choices straight from the annotation instead of hard-coding a parallel list: - -```python -from typing import get_args -get_args(ProjectionField) # ('input', 'target', 'metadata') -``` - -Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup). `num_classes` is a free function, not a `Flux` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. +Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Flux.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Flux` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. ## `LabelMap` — fittable name↔id encoding -When a dataset's `target` is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTargetOp` / `DecodeTargetOp` need — the *fittable* companion to those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: +When a dataset's label is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTarget` / `DecodeTarget` ops need — the *fittable* companion to those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: ```python -from sampleflux import LabelMap, Flux +from sampleflux import LabelMap, Flux, iter_key -lm = LabelMap.fit(iter_targets(train_source)) # {"bird": 0, "cat": 1, "dog": 2} +lm = LabelMap.fit(iter_key(train_source, "class")) # {"bird": 0, "cat": 1, "dog": 2} lm.num_classes # 3 lm.label_names # ["bird", "cat", "dog"] (id -> name) lm.save("class_names.json") # {"class_names": [...], "num_classes": N} -encoded = Flux(source=train_source, ops=[lm.encode_op()]) # targets are now ints +encoded = Flux(source=train_source, ops=[lm.encode_op()]) # "class" Labels now carry int ids # Later, at eval time — reload the SAME ordering instead of refitting: lm2 = LabelMap.load("class_names.json") diff --git a/docs/record-model.md b/docs/record-model.md new file mode 100644 index 0000000..8253717 --- /dev/null +++ b/docs/record-model.md @@ -0,0 +1,425 @@ +# The record model — THE sampleflux data model + +A sample is a **plain `dict`** of **typed values**. Import the whole surface from the PACKAGE TOP +LEVEL (`from sampleflux import Record, Image, Mask, Regions, Label, Transform, Pipeline, +as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). +The design rationale is recorded in +[architecture.md](architecture.md#one-type-dispatched-op-engine--plain-dict-records-libraries-as-is-2026-07-25). + +## Why + +If everything that is not literally the model input or target — a segmentation mask, +region boxes, a signal's samplerate, an image's layout, a label's class names — is jammed into one +flat `metadata` dict keyed by string, it is disconnected from the value it describes. And if the +carrier is a bespoke container class, every external library needs an adapter before it can touch it. + +The record model fixes both. **A sample is a plain dict, values are typed, and metadata lives on the +value it describes** — an `Image` carries its `layout`, a `Label` its `classes`. **Key names carry +meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the same convention as every torch +batch dict and albumentations' keyword vocabulary), so libraries that already understand dicts or +named kwargs run **as-is**, with no wrapper anywhere. A scalar side value is just another key: + +```python +record = { + "image": Image(rgb_hwc), # typed: knows its layout + "mask": Mask(seg_hw), # shares the image's frame + "bboxes": [[2, 3, 6, 7]], # albumentations vocabulary + "labels": ["drone"], + "class": Label("drone_x", classes=["noise", "drone_x"]), + "samplerate": 30.72e6, # a plain value is just another key +} +``` + +There is deliberately **no container class** — `Record` is a type alias (`Dict[str, Any]` in +`sampleflux.items`), ops receive and return ordinary dicts, and `None` means "drop this record" +(filter semantics). + +## The pieces + +### Items — typed values that own their metadata + +sampleflux is **modality-neutral**, so its core ships only generic items — images, masks, boxes, +labels. (Domain items — a signal, a spectrogram — live in the domain package; see below.) + +```python +from sampleflux import Image, Mask, Regions, Label + +Image(rgb_hwc, layout="HWC") # an image knows its layout ("HWC" default / "CHW") +Mask(seg_hw) # a mask shares its image's frame +Regions(boxes=[[1,1,4,4]], labels=["drone"], canvas=(8, 10), extras={"snr_db": [12.5]}) +Label("drone_x", classes=["noise", "drone_x"]) +``` + +Items are **hybrid**: array-backed items (`Image`, `Mask`) subclass `NDArrayItem` — an `np.ndarray` +subclass whose declared `_item_attrs` survive numpy operations via `__array_finalize__` — so a +type-agnostic operation touches them as an array; structured items (`Regions`, `Label`) are dataclass +wrappers (a bounding-box set is not an array). A uniform payload accessor hides the difference from +kernels: + +```python +from sampleflux import item_data, with_data +item_data(Image(arr)) # -> the plain ndarray +with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved +``` + +`register_item` / `is_item` / `item_types` / `get_item_type` / `item_type_names` are the open item +registry — the extensibility surface a domain package or user type plugs into (one class + one +decorator, no core edit). + +### Ops — type dispatch with once-per-record parameters + +A `Transform` (`sampleflux.transform`) samples its parameters ONCE per record +(`get_params(record)`), then applies a per-type **kernel** to every value whose type it handles +(`@MyOp.kernel(ItemType)`, resolved MRO-aware by `sampleflux.dispatch`). Values it does not handle +pass through. Because the parameters are sampled once and shared, one op moves every handled value +with the SAME decision — the torchvision-v2 model. Targeting is by TYPE; the `field=` constructor +parameter pins an op to one named key when a record holds several values of a handled type. + +```python +import numpy as np +from sampleflux import Image, Record, Transform + +class Brighten(Transform): + handles = (Image,) + + def __init__(self, strength: float = 0.1, field: str | None = None) -> None: + super().__init__(field=field) + self.strength = strength + self._rng = np.random.default_rng(7) + + def get_params(self, record: Record) -> dict: + return {"offset": self._rng.uniform(0.0, self.strength)} # drawn ONCE per record + +@Brighten.kernel(Image) +def _brighten_image(value: Image, params: dict) -> Image: + return Image(np.asarray(value) + params["offset"], layout=value.layout) +``` + +The second sanctioned op shape is the **type-changing op** — read one key, write a differently-typed +item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: +`Mask` → `Regions`, the target ops). It subclasses `Transform` and overrides `__call__` instead of +registering a same-type kernel, declaring `handles` / `consumes` / `produces` truthfully as graph +metadata (next section). + +### Declaring an op's type interface — `handles` / `consumes` / `optional` / `produces` + +Every `Transform` carries four class-level tuples of item types. They are the op's **type +interface**: what a reader (or a machine — a visual editor's typed sockets, a pipeline linter) +learns about the op without executing it or loading the kernel registry. + +| Attribute | Meaning | Enforced at runtime? | +|---|---|---| +| `handles` | The value types this op processes — every record value of one of these types is touched, everything else passes through. | Only by `FunctionTransform` / `as_transform` (`isinstance(value, self.handles)` is its application gate). For a kernel op, actual dispatch is the kernel registry (`dispatch(type(self), type(value))`) — `handles` must MIRROR the registered kernels. | +| `consumes` | The input types the op NEEDS to do useful work (its required inputs). Convention: an empty `consumes` means "same as `handles`". | No — declarative. | +| `optional` | Input types the op uses when present but works without (e.g. a geometric op that also moves a `Mask` if the record has one). | No — declarative. | +| `produces` | The types the op ADDS or CHANGES — its output contract (what a downstream op can rely on finding). | No — declarative. | + +Concretely, `ToTensor` declares: + +```python +class ToTensor(Transform): + handles = (NDArrayItem,) # touches array-backed values + consumes = (NDArrayItem,) # needs at least one array-bearing key to act on + produces = (torch.Tensor,) # writes a LIVE CHW float tensor under `output` (or in place) +``` + +(A `produces` entry need not be a registered item type — `ToTensor`'s output is a plain +record value, which is exactly what the declaration should say.) + +**When `handles` and `consumes` differ.** They coincide for a simple one-type op (`Threshold`, +the FFT ops), and diverge in two directions: + +- **Optional riders — `handles` ⊃ `consumes`.** A joint geometric op MAY move several types with + one draw but only REQUIRES one of them: + + ```python + class JointFlip(Transform): + handles = (Image, Mask, Regions) # everything ONE draw may move + consumes = (Image,) # the only input it needs to be useful + optional = (Mask, Regions) # moved together with the image when present + ``` + + A record with just an `Image` is fine; a record that also carries a `Mask`/`Regions` gets them + moved consistently. Declaring `consumes = handles` here would wrongly tell a reader (or a + pipeline linter) that a mask is required. + +- **Read-only reference inputs — `consumes` ⊃ `handles`.** An op may NEED a value it never + changes. A denoiser that estimates the noise floor from the signal but excludes the + ground-truth ON regions when a mask is available follows the same logic with `optional` + (a real op: `handles = consumes = (Signal,)`, `optional = (Mask, GridMask)`, `produces = + (Signal,)` — the mask is read, never written). The required-reference variant looks like: + + ```python + class ScaleBoxesToImage(Transform): + handles = (Regions,) # the only type it CHANGES + consumes = (Regions, Image) # ...but it cannot run without the reference Image (its shape) + ``` + +In short: `handles` = "what I write", `consumes` = "what must be present", `optional` = "what I +use when present" — the three answer different questions, and only collapse into one tuple for +the simplest ops. + +**Limiting a multi-input op to NAMED keys.** The type interface says *what kinds* of values an +op works with; *which record entry* each input comes from is CONFIG. A single-input op uses the +base `field=` param (one key, still type-gated). A multi-input op declares **one `_field` +constructor param per input slot** — defaulting to the conventional key name, resolved and +validated lazily in `__call__`: + +```python +class KeepRegionsOnMask(Transform): + """Drop regions whose center pixel is OFF in the activity mask. + + Args: + mask_field: Record key of the activity Mask to test against. Defaults to "mask". + regions_field: Record key of the Regions to filter. Defaults to "regions". + output: Key the filtered Regions are written to; blank (default) replaces regions_field in place. + """ + + handles = (Regions,) # the only type it CHANGES + consumes = (Mask, Regions) # both inputs must be present + produces = (Regions,) + + def __init__(self, mask_field: str = "mask", regions_field: str = "regions", output: str = "") -> None: + super().__init__() + self.mask_field = mask_field + self.regions_field = regions_field + self.output = output + + def __call__(self, record: Record) -> Record: + for name, want in ((self.mask_field, Mask), (self.regions_field, Regions)): + if name not in record: + raise ValueError(f"{type(self).__name__}: no {name!r} key in record (keys: {list(record)})") + if not isinstance(record[name], want): + raise TypeError(f"{type(self).__name__}: {name!r} is {type(record[name]).__name__}, expected {want.__name__}") + mask, regions = record[self.mask_field], record[self.regions_field] + keep = [b for b in regions.boxes if mask[int((b[1] + b[3]) / 2), int((b[0] + b[2]) / 2)]] + out = Regions(boxes=keep, labels=regions.labels, scores=regions.scores, canvas=regions.canvas) + return {**record, (self.output or self.regions_field): out} +``` + +So a record carrying several masks and several region sets is disambiguated entirely in config — +the op looks ONLY at the named entries: + +```yaml +- !class:mypkg.KeepRegionsOnMask + mask_field: activity_mask # not the segmentation mask under "mask" + regions_field: predictions # not the ground truth under "regions" +``` + +This is the established pattern for every shipped multi-input op (e.g. the region→target ops +take `image_field="image"` + `regions_field="regions"` + `output="target"`). Two rules keep it +predictable: the defaults are the CONVENTIONAL key names (so the common record shape needs zero +config), and a wrong/missing key fails lazily in `__call__` with the key list in the message — +never silently falls back to a different entry when an explicit name was given. + +Rules of use: + +- **Declare truthfully or not at all.** Nothing validates these tuples against the op's behavior, + so wrong metadata is worse than missing metadata — it misleads both readers and any tool that + consumes it. A kernel op's `handles` changes when its kernel registrations change; keep them in + sync (an externally-registered kernel widens the REAL dispatch without widening `handles` — that + is fine, `handles` documents the op author's contract, the registry documents the deployment). +- **Kernel ops rarely need more than `handles`** — dispatch and pass-through already follow from + the registry; `consumes`/`produces` earn their keep on type-CHANGING ops, where the `__call__` + override hides the type flow that kernels would have made explicit. +- **These tuples never gate execution** (except the `FunctionTransform` case above). If an op must + refuse to run without an input, validate lazily in `__call__` with a clear error — the same + lazy-validation convention every op follows. + +**sampleflux ships no native augmentation ops** — geometric/photometric augmentation comes from +torchvision `transforms.v2` / albumentations run as-is (next section); native ops exist only where +no library covers them. + +### Mixing libraries — as-is, no adapters + +The engine's single op-application chokepoint, `sampleflux.core._apply_op(record, op)`, dispatches +on the op's FAMILY (by MRO module name, no eager import) and invokes each family the way its own +library expects: + +- **albumentations** — the op receives exactly its own kwarg vocabulary: the + `image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record, nothing else. One + call = one joint draw across them; array outputs are re-wrapped in the incoming value's item type, + so an `Image`/`Mask` keeps its type and metadata through the library. +- **torchvision `transforms.v2`** — called on the record dict as-is (tv2 walks dicts natively). + Layout conversions are the library's own transforms (`v2.ToImage()`) — the engine never converts + silently. +- **everything else** — `op(record)`; `None` drops the record. + +So bare library transforms sit in one list with native ops — in `Flux(ops=[...])`, in a `Pipeline`, +in a `flow:` step: + +```python +import albumentations as A +from sampleflux import Pipeline + +Pipeline([ + A.Compose( # box-carrying augmentation: the library's own Compose + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ), + A.GaussNoise(p=1.0), # image only — its own kwarg vocabulary + Brighten(strength=0.2), # native type-dispatched op +])(record) +# image + mask + bboxes flipped together (one joint draw); record["class"] untouched. +``` + +The same holds in YAML — a bare library transform is an ordinary `!class:` node in an `ops:` list +(the engine flows deferred markers at route entry): + +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.GaussNoise + p: 1.0 +``` + +See [augmentation.md](augmentation.md) for the full key-vocabulary / bbox / seeding recipes. +Runnable end-to-end: [`examples/record_pipeline.py`](../examples/record_pipeline.py). + +### `Pipeline` — the sequential composer + +`Pipeline(transforms=[...])` (`sampleflux.transform`, `@configurable(category="op", +group="compose")`) wraps an ordered op list so it appears as one named block in a config and one +node on a visual canvas: zero-arg/lazy (config-deferred markers flow on first call), entries applied +through `_apply_op` (so bare library transforms nest exactly as in a bare ops list), `None` +propagation (a filter-drop stops the chain), and `close()` propagation to inner ops that own +resources. + +## Extending it + +### A custom op from a plain function + +```python +from sampleflux import as_transform, Image +brighten = as_transform(lambda d: d + 0.1, handles=(Image,), field="image") +``` + +### A custom item type + a kernel for an existing op — no core edit + +```python +from dataclasses import dataclass, field +from sampleflux import register_item +from mypkg.transforms import MyGeoTransform # any Transform subclass + +@register_item +@dataclass +class Keypoints: + data: list = field(default_factory=list) # a `data` field = the payload slot + +@MyGeoTransform.kernel(Keypoints) +def _(value, params): + return move_points(value, params) +``` + +Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a +subclass transform inherits its base's kernels until it overrides them. + +### Domain items live in the domain package + +The same mechanism, applied across packages: because sampleflux is modality-neutral, a signal-domain +package defines its own items (a signal, a spectrogram) and its own type-changing ops, registers +them with `register_item`, and they become first-class record values — dispatchable, collatable, +storable — with no core edit. + +### A new library family + +Supporting a new external transform library is NOT an adapter class — it is one new branch in +`core._apply_op` (an MRO module-name matcher plus the library's native calling convention), so every +engine route and composing op picks it up at once. + +## Engines — Flux and FlowGraph carry the record + +Every carrier is a plain record dict, and every route applies ops through `_apply_op` — sequential, +spawn-parallel, streamed, and random-access (`__getitem__`) alike, in `Flux` and in `FlowGraph`. +Composing ops (`Pipeline`, `RandomApply`, `Enable`, `Parallel`, `ConfigureOp`, the context ops +`Apply`/`Capture`) route their inner ops through the same chokepoint, so a bare library transform +nests anywhere a native op does. + +```python +Flux(source=my_source, ops=[A.GaussNoise(p=1.0), Brighten()]).to_sink(HDF5Sink(path="out.h5")) +``` + +`Flux.map(func, key=None)` lifts a plain function over one record entry (`key=None` hands it the +whole dict — internally a `WrappedOp`, which stores the callable as its importable path so it +pickles across `spawn` workers); `Flux.project(keys)` yields partial records restricted to the +requested keys (see [projection.md](projection.md)). + +### Graph fan-in (`merge_from`) and entry binds (`step[key]`) + +In a `flow:` document, the fan-in is **`merge_from`** — the UNION of the named steps' record +entries, in slot order, last-write-wins on a key collision. The idiom for a derived-entry branch: +produce, `SelectFields` the new key(s), merge: + +```yaml +flow: + start: {} + masked: {op: !class:sampleflux.ops.numpy.Threshold(low_level=0.5), from: start} + mask_only: {op: !class:sampleflux.ops.structure.SelectFields(keys: [mask]), from: masked} + boosted: {op: !class:mypkg.Boost(), from: start} + out: {from: boosted, merge_from: [mask_only]} +``` + +`bind:` references have three shapes: a bare `step` binds the step's WHOLE result record, +`step[key]` binds the named ENTRY of that step's record, and `step.attr` binds the step op's live +`@output`. Lowering (`to_ops`) compiles `merge_from` to the `MergeFields` context op and entry-binds +to `Apply(key=...)`; lifting (`from_ops`) round-trips both. See [graph.md](graph.md). + +## Storage — the record key-group layout + +All four backends (`HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, +`ZarrBatchSink`↔`ZarrBatchSource`, `DirectorySink`↔`DirectorySource`) write a record in ONE logical +schema: per record, one group per KEY carrying the value's registered type name (`__item_type__`), +the payload as a `data` dataset, and its attrs (scalars natively — queryable; arrays as sub-datasets +under `attrs/`; structured values JSON-tagged so tuples survive). A plain (non-item) value rides the +`"plain"` type tag — an array payload as `data`, a scalar under the `value` attr. Key order is +preserved in `__field_order__`; the store is stamped `sampleflux_format = "typedrecord-v1"`. + +Backends never inspect item internals — everything serializes through the item codec +(`sampleflux/io.py`: `encode_item` / `decode_item` / `encode_record` / `decode_record`), so an +externally-registered item type round-trips with zero storage edits; +`register_io(MyItem, encode=..., decode=...)` overrides the default structural codec when needed. +Decoding requires the item type to be registered (imported) in the reading process — the same +contract as Confluid's `!class:`. + +```python +sink = HDF5Sink(path="out.h5", overwrite=True) +with sink: + for record in flux: + sink.write(record) +back = list(HDF5Source(path="out.h5")) # exact records: keys, types, order, tuple attrs +``` + +`ZarrBatchSink` (the uniform single-array sink) appends the FIRST record entry's payload per row and +stores a one-time item template — per-record attr variation needs `ZarrGroupSink`. + +**No backward compatibility:** a store stamped with the pre-record `typedsample-v1` tag (or carrying +no tag) raises a `ValueError` telling you to re-generate it with a current sink +(`storage/base.py::require_record_format`) — there is no legacy read path. + +### Querying record stores without loading arrays + +The metadata scans yield the nested `{key: {attr: value}}` shape, and a `where` expression +addresses it as `.` (a plain scalar entry appears under its `value` attr): + +```python +fast = MetadataFilterSource(source=HDF5Source(path="out.h5"), where="signal.samplerate > 1e6") +``` + +Array-valued attrs appear as shape/dtype stubs (presence/shape testable, never loaded). A key named +like a Python keyword (e.g. `class`) can't be addressed in an expression — use the programmatic +`predicate` or a non-keyword key name. Live records expose the same nested shape via +`sampleflux.storage.query.record_metadata(record)`. See [storage.md](storage.md). + +## Batching — `collate_records` + +`collate_records` (the collate registry's `"record"` default) turns N record dicts into ONE batched +record: per key, typed payloads stack (torch → stacked tensor, numpy → stacked array, else a list) +and each declared item attr becomes a LIST of per-record values, decoded back into one batched item +of the same type; a plain value batches as the plain list. Batches must be key-homogeneous — a +mismatch raises. See [kinds.md](kinds.md). + +## What is NOT here yet (follow-ups) + +A torch-`Tensor`-subclass item base (torch payloads currently ride in wrapper items or as plain +values) and confluid-native item-type discovery. See the root `TASKS.md`. diff --git a/docs/sources.md b/docs/sources.md index 78c3861..61d4297 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -2,23 +2,23 @@ ## Hugging Face datasets -`HuggingFaceSource` turns any `datasets.Dataset` (a Hub repo id or a local imagefolder path) into standardized `Sample` triplets, preserving metadata traceability that often goes missing in simple dictionary-based records. +`HuggingFaceSource` turns any `datasets.Dataset` (a Hub repo id or a local imagefolder path) into plain record dicts of typed values: the `input_feature` column becomes an `Image` under the record key `"image"`, the `target_feature` column a `Label` under `"class"`, and each kept metadata column its own `Label` entry keyed by the column name (plus the source-provenance `hf_path` / `hf_split` entries) — traceability that often goes missing in bare dictionary loading. -- **`metadata_features` (which columns ride along on `Sample.metadata`):** `None` / `[]` keep none (the default); an explicit list keeps exactly those columns; and the sentinel **`"*"`** (or `["*"]`) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load. It stays opt-in so existing configs are unchanged. +- **`metadata_features` (which extra columns become record entries):** the sentinel **`"*"`** (or `["*"]`, the default) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load; an explicit list keeps exactly those columns; `None` / `[]` keep none. ```yaml hf_train: !class:sampleflux.sources.HuggingFaceSource() path: mnist input_feature: image target_feature: label - metadata_features: ["*"] # keep every other column as metadata + metadata_features: ["*"] # keep every other column as its own record entry (the default) ``` > **Lazy & zero-arg construction** — `HuggingFaceSource` follows the workspace lazy-init convention: the constructor does no work (no network), so `HuggingFaceSource()` is valid and building one is free. The dataset is downloaded only on first access to the read-only `.dataset` property (cached thereafter; reset `_dataset` to reload), and `.resolved_metadata_features` (the `"*"` expansion) is derived lazily from the loaded columns. `path` is therefore optional at construction and validated lazily — accessing `.dataset` with an empty `path` raises a clear `ValueError`. ## Train / val / test splitting (`DatasetSplit`) -`DatasetSplit` partitions any indexable source (implementing `__len__` and `__getitem__`) into reproducible **train / val / test** views. It is a `source` (`category="source"`) — it yields `Sample`s and is wired into a trainer's `source:` slot — and it applies no ops, so it's a source, not an engine. +`DatasetSplit` partitions any indexable source (implementing `__len__` and `__getitem__`) into reproducible **train / val / test** views. It is a `source` (`category="source"`) — it yields records and is wired into a trainer's `source:` slot — and it applies no ops, so it's a source, not an engine. **Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: @@ -60,13 +60,13 @@ val_set: !class:sampleflux.sources.DatasetSplit() ## Range & concatenation sources -- **`RangeSource(source, start, end)`** — a contiguous index slice `[start:end)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. +- **`RangeSource(source, start, stop)`** — a contiguous index slice `[start:stop)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. ```yaml first_half: !class:sampleflux.sources.RangeSource() source: !ref:hf_train start: 0 - end: 5000 + stop: 5000 ``` - **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointFlux`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. diff --git a/docs/storage.md b/docs/storage.md index 8328e2f..5df3cc1 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -9,55 +9,86 @@ from sampleflux.storage.zarr import ZarrGroupSink # Stream from HDF5 to Zarr in parallel Flux.from_source(HDF5Source("input.h5")) \ .parallel(workers=8) \ - .map(heavy_op) \ + .map(heavy_op, key="image") \ .to_sink(ZarrGroupSink("output.zarr")) ``` ## Sinks and their matching sources -Every sink has a source that reads its layout back into typed `Sample` bags: +Every sink has a source that reads its layout back into record dicts of typed values: | Backend | Sink | Source | Round-trips | |---|---|---|---| -| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | all fields + roles | -| Zarr group (one group / sample) | `ZarrGroupSink` | `ZarrGroupSource` | all fields + roles | -| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | primary input field only (uniform shape) | -| Directory (one dir / sample) | `DirectorySink` | `DirectorySource` | all fields + roles | +| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | all keys + item types + attrs | +| Zarr group (one group / record) | `ZarrGroupSink` | `ZarrGroupSource` | all keys + item types + attrs | +| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | first record entry only (uniform shape) | +| Directory (one dir / record) | `DirectorySink` | `DirectorySource` | all keys + item types + attrs | ```python from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource -Flux(samples).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) -for sample in ZarrGroupSource("ds.zarr"): # exact fields, roles and item attrs reconstructed +Flux(records).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) +for record in ZarrGroupSource("ds.zarr"): # exact keys, item types and attrs reconstructed ... ``` -> Domain-specific storage formats implement the same `DataSink`/`DataSource` protocols in their own package — e.g. the SigMF waveform-recording pair (`SigMFSink`/`SigMFSource`) lives in `waivefront.sigmf`, not here. The engine never couples to a specific format. +All four backends share ONE logical schema — the **record key-group layout**, stamped +`sampleflux_format = "typedrecord-v1"`: per record, one group per KEY carrying the value's +registered type name, the payload as a `data` dataset, and its attrs (scalars natively; structured +values JSON-tagged so tuples survive); a plain (non-item) value rides the `"plain"` type tag — an +array payload as `data`, a scalar under the `value` attr. Everything serializes through the item +codec (`sampleflux/io.py`), so an externally-registered item type round-trips with zero storage +edits (see [record-model.md](record-model.md#storage--the-record-key-group-layout)). + +> **No backward compatibility.** A store whose format tag is missing or pre-record +> (`typedsample-v1`) raises a `ValueError` telling you to re-generate it with a current sink — +> there is no legacy read path. + +> Domain-specific storage formats implement the same `DataSink`/`DataSource` protocols in their own +> package — e.g. a waveform-recording format pair lives in the signal-domain package, not here. The +> engine never couples to a specific format. ## Array-valued item attributes -Each field is stored as its own group: the item's payload as a `data` dataset and its scalar attributes as HDF5 **attributes**. HDF5 caps attribute size, so any **array-valued attribute** (`np.ndarray` / `torch.Tensor`, e.g. a per-sample weight map) is written as its own sub-dataset under `attrs/` instead — a large array never overflows the attribute limit, and `HDF5Source` restores every attribute on read. A segmentation mask is not an attribute at all: it is a first-class `Mask` field with its own payload. +Each key is stored as its own group: the item's payload as a `data` dataset and its scalar +attributes as HDF5 **attributes**. HDF5 caps attribute size, so any **array-valued attribute** +(`np.ndarray` / `torch.Tensor`, e.g. a per-record weight map riding an item's attrs) is written as +its own sub-dataset under `attrs/` instead — a large array never overflows the attribute limit, and +`HDF5Source` restores every attribute on read. A segmentation mask is not an attribute at all: it +is a first-class `Mask` value under its own key with its own payload. ```python -from sampleflux import Sample, Image, Mask, item_data +from sampleflux import Image, Mask, item_data -sample = Sample({"image": Image(data), "mask": Mask(mask_2d)}, roles={"mask": "target"}) -Flux([sample]).to_sink(HDF5Sink("ds.h5", overwrite=True)) +record = {"image": Image(data), "mask": Mask(mask_2d)} +Flux([record]).to_sink(HDF5Sink("ds.h5", overwrite=True)) loaded = next(iter(HDF5Source("ds.h5"))) item_data(loaded["mask"]) # the full mask array, byte-exact (not a truncated repr) -loaded.role_of("mask") # "target" — roles round-trip too +loaded["image"].layout # item attrs round-trip too ``` ## Queryable metadata (`sampleflux.storage.query`) -Filter stored samples by metadata predicates *without loading arrays*: sources implementing the `SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs / `.zattrs` / sidecar JSON — `HDF5Source` and `ZarrGroupSource` both do (and external storage sources can implement the structural protocol without importing this module), so **existing HDF5/Zarr files are queryable with no rewrite**: +Filter stored records by metadata predicates *without loading arrays*: sources implementing the +`SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs / `.zattrs` / sidecar JSON — +`HDF5Source` and `ZarrGroupSource` both do (and external storage sources can implement the +structural protocol without importing this module), so **record-layout HDF5/Zarr files are +queryable with no extra index**: ```python from sampleflux.storage.query import MetadataFilterSource -view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="snr_db > 10 and drone == 'DJI'") +view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="signal.samplerate > 1e6") len(view) # matches counted from a metadata-only scan -flux = Flux(source=view, ops=[...]) # arrays load ONLY for matching samples +flux = Flux(source=view, ops=[...]) # arrays load ONLY for matching records ``` -`where` uses the FormulaOp restricted namespace with metadata keys as variables (a missing key = non-matching, a malformed expression fails loudly); a programmatic `predicate=` composes with AND; sources without the protocol fall back to full-iteration filtering. Array-valued HDF5 metadata appears in the scan as shape/dtype stub strings (`""`), so queries can test presence without a single array read. +The scans yield the nested `{key: {attr: value}}` shape, and a `where` expression addresses it as +`.` (a plain scalar entry appears under its `value` attr — `"snr_db.value > 10"`). +`where` uses the FormulaOp restricted namespace with metadata keys as variables (a missing key = +non-matching, a malformed expression fails loudly); a programmatic `predicate=` composes with AND; +sources without the protocol fall back to full-iteration filtering via `record_metadata(record)` — +the same nested shape derived from a live record. Array-valued attrs appear in the scan as +shape/dtype stub strings (`""`), so queries can test presence +without a single array read. A key named like a Python keyword (e.g. `class`) can't be addressed +in an expression — use the programmatic `predicate` or a non-keyword key name. diff --git a/docs/typed-model.md b/docs/typed-model.md deleted file mode 100644 index 7173af2..0000000 --- a/docs/typed-model.md +++ /dev/null @@ -1,227 +0,0 @@ -# The typed-bag model — THE sampleflux data model - -Import the typed surface from the PACKAGE TOP LEVEL -(`from sampleflux import Sample, Image, Mask, Regions, Label, Transform, Pipeline, primary, item_data, typed_collate, register_item, register_kernel, register_adapter, register_io, ...`). -The design rationale is recorded in -[architecture.md](architecture.md#the-typed-bag-model-a-named-bag-of-typed-items-sampleflux-bag-2026-07-21). - -## Why - -If everything that is not literally the model input or target — a segmentation mask, -`[f0,f1,t0,t1]` regions, a signal's samplerate, an image's canvas size, a label's class names — is -jammed into one flat `metadata` dict keyed by string, it is disconnected from the value it -describes. That makes two things hard: metadata has no natural home, and a transform cannot move -several fields together consistently (flip an image → flip its mask → flip its boxes). - -The typed-bag model fixes both: **a sample is a named bag of typed items, and metadata lives on the -item it describes.** Transforms dispatch on item *type*. - -## The pieces - -### Items — typed values that own their metadata - -sampleflux is **modality-neutral**, so its core ships only generic items — images, masks, boxes, -labels. (Signal-domain items live in the domain package; see below.) - -```python -from sampleflux import Image, Mask, Regions, Label - -Image(rgb_hwc, layout="HWC") # an image knows its layout -Mask(seg_hw) # a mask shares its image's frame -Regions(boxes=[[1,1,4,4]], labels=["drone"], canvas=(8, 10)) -Label("drone_x", classes=["noise", "drone_x"]) -``` - -Items are **hybrid**: array-backed items (`Image`, `Mask`) subclass `np.ndarray`, so a -type-agnostic operation touches them as an array and their extra attributes survive numpy ops; -structured items (`Regions`, `Label`) are dataclass wrappers. A uniform payload accessor hides the -difference from kernels: - -```python -from sampleflux import item_data, with_data -item_data(Image(arr)) # -> the plain ndarray -with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved -``` - -### `Sample` — a named bag with role tags - -```python -from sampleflux import Sample - -sample = Sample( - {"image": Image(rgb), "regions": Regions(boxes), "class": Label("drone_x")}, - roles={"regions": "target", "class": "target"}, # default role is "input" -) -sample.inputs() # {"image": Image(...)} -sample.targets() # {"regions": Regions(...), "class": Label(...)} -sample.set_role("regions", "aux") # copy-on-write; a field's role changes without moving keys -``` - -`input` / `target` / `aux` / `pred` are **tags read at the train/collate/sink boundary**, not tuple -positions. `Sample` is immutable — every mutator returns a new sample. - -### Transforms — type dispatch with once-per-sample parameters - -A transform samples its parameters once, then applies a per-type kernel to each handled field. -Fields it does not handle pass through. Because the parameters are sampled **once** and shared, -image / mask / boxes move consistently — the thing the flat-metadata model could not express. A -transform may also CHANGE an item's type under the same key (e.g. the domain `Fourier` turns a -`Signal` field into a `Spectrogram` in place). - -**sampleflux ships no native augmentation transforms** — geometric/photometric augmentation comes -from torchvision `transforms.v2` / albumentations through the coercion registry below; native -transforms exist only where no library covers them (domain packages register their own). - -### Mixing libraries — one pipeline, many worlds - -torchvision `transforms.v2` dispatches by type, albumentations by keyword name. **Bare library -transforms drop straight into a `Pipeline`** — a registered adapter wraps each one automatically, and -each transform hits only the field(s) it handles: - -```python -from torchvision.transforms import v2 -import albumentations as A - -Pipeline([ - v2.RandomHorizontalFlip(p=0.5), # torchvision v2: Image + Mask + Regions together (one draw) - v2.Normalize(mean, std), # torchvision v2: Image (wrapped automatically) - A.GaussNoise(p=1.0), # albumentations: Image (wrapped automatically) -])(sample) -``` - -The coercion is a small **registry** (`register_adapter` / `coerce_transform`, both top-level): the -built-in torchvision-v2 and albumentations adapters register a matcher (by MRO module name, no eager -import) at package load. Teach a `Pipeline` about your own library's transforms with one call: - -```python -from sampleflux import register_adapter -register_adapter(lambda o: type(o).__module__.startswith("mylib"), lambda o: MyLibAdapter(o)) -``` - -For surgical control — target one field key with a library transform — construct the adapter -explicitly: `TorchvisionV2Adapter(v2.Normalize(...), only=["image"])`. - -Runnable end-to-end: [`examples/typed_pipeline.py`](../examples/typed_pipeline.py). - -## Extending it - -### A custom transform from a plain function - -```python -from sampleflux import as_transform, Image -brighten = as_transform(lambda d: d + 0.1, handles=(Image,), only=["image"]) -``` - -### A custom item type + a kernel for an existing transform — no core edit - -```python -from sampleflux import register_item -from mypkg.transforms import MyGeoTransform # any Transform subclass - -@register_item -class Keypoints: - def __init__(self, points): self.points = points - -@MyGeoTransform.kernel(Keypoints) -def _(item, params): - return move_points(item, params) -``` - -Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a -subclass transform inherits its base's kernels until it overrides them. - -### Signal-domain items live in the domain package (`waivefront.bag`) - -This is the same mechanism, applied across packages: because sampleflux is modality-neutral, the -signal-domain `Signal` / `Spectrogram` items and the `Fourier` transform (`Signal` → `Spectrogram`) -live in `waivefront.bag` and register into the SAME registries on import — so a bare `Fourier()` -drops into a `sampleflux.bag.Pipeline` alongside the generic transforms with no core edit. See -`waivefront/examples/05_typed_bag_signal.py`. - -## Engines — Flux and FlowGraph carry the typed bag - -A `Sample` is **never coerced**: on every `Flux` route (sequential / parallel / streamed / -`__getitem__`) and in `FlowGraph`, a typed source item passes through verbatim and each op receives -the whole bag (`Pipeline` transforms, structure ops, and the compose plane — `TransformChain`, -`RandomApply`, `Enable`, `Apply`, `Capture` — all route typed carriers correctly). - -```python -Flux(source=typed_source, ops=[v2.RandomHorizontalFlip(p=0.5), Fourier()]).to_sink(HDF5Sink(...)) -``` - -### Typed fan-in (`merge_from`) and field binds (`step[key]`) - -In a `flow:` document, the fan-in is **`merge_from`** — the UNION of the named steps' fields -and roles, in slot order, last-write-wins on a key collision. The idiom for a derived-field branch: -produce, `SelectFields` the new field(s), merge: - -```yaml -flow: - start: {} - masked: {op: !class:mypkg.MakeMask(), from: start} - mask_only: {op: !class:sampleflux.ops.structure.SelectFields(keys: [mask]), from: masked} - boosted: {op: !class:mypkg.Boost(), from: start} - out: {from: boosted, merge_from: [mask_only]} -``` - -`bind:` references gain a field form: `step[key]` binds the named ITEM of that step's bag as an op -parameter; a bare `step` reference binds the step's PRIMARY input-role item -(`sampleflux.primary`). Lowering (`to_ops`) compiles `merge_from` to the `MergeFields` context op -and key-binds to `Apply(key=...)`; lifting (`from_ops`) round-trips both. - -## Storage — the typed field-group layout - -All three backends (`HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, -`DirectorySink`↔`DirectorySource`) write a `Sample` in ONE logical schema: per sample, one -group per FIELD carrying the item's registered type name, its role, the payload as a dataset, and -its attrs (scalars natively — queryable; arrays as sub-datasets; structured values JSON-tagged so -tuples survive). The store is stamped `sampleflux_format = "typedsample-v1"`. Backends never -inspect item internals — everything serializes through the item codec (`encode_item` / `decode_item`, -top-level), so an externally-registered item type round-trips with zero storage -edits; `register_io(MyItem, encode=..., decode=...)` overrides the default structural codec when -needed. - -```python -sink = HDF5Sink(path="out.h5", overwrite=True) -with sink: - for sample in flux: # Samples - sink.write(sample) -back = list(HDF5Source(path="out.h5")) # exact Samples: fields, roles, order, tuple attrs -``` - -`ZarrBatchSink` (the uniform single-array sink) appends the PRIMARY input field's payload per row -and stores a one-time item template — per-sample attr variation needs `ZarrGroupSink`. - -### Querying typed stores without loading arrays - -The metadata scans yield the nested `{field: {attr: value}}` shape, and a `where` expression -addresses it as `.`: - -```python -fast = MetadataFilterSource(source=HDF5Source(path="out.h5"), where="signal.samplerate > 1e6") -``` - -Array-valued attrs appear as shape/dtype stubs (presence/shape testable, never loaded). A field -named like a Python keyword (e.g. `class`) can't be addressed in an expression — use the -programmatic `predicate` or a non-keyword field name. - -## Interop with plain `(input, target, metadata)` tuples - -Bridge to and from a plain 3-tuple — for an external consumer that expects one, or when adopting a -non-typed dataset. The bridge is lossless: `to_legacy` flattens the bag into an -`(input, target, metadata)` tuple (the whole bag encoded in the metadata, while `input` / `target` -still expose the primary payloads), and `to_typed` reconstructs the exact bag: - -```python -from sampleflux import to_legacy, to_typed -plain = to_legacy(sample) # (input, target, metadata); to_typed(plain) == sample -typed = to_typed(plain) # exact reconstruction -# adopting an arbitrary external dataset needs a per-dataset builder: -to_typed(record, builder=lambda r: Sample({"image": Image(r.image), "class": Label(r.label)})) -``` - -## What is NOT here yet (follow-ups) - -A torch-`Tensor`-subclass item base (torch payloads currently ride in wrapper items), confluid-native -item-type discovery, the generated per-transform families (`Tv*` / `Alb*`) in this namespace, -FluxStudio typed side sockets, and the `decode` (inverse) path. See the root `TASKS.md`. diff --git a/examples/dataset_split.yaml b/examples/dataset_split.yaml index 13f08c8..bac2838 100644 --- a/examples/dataset_split.yaml +++ b/examples/dataset_split.yaml @@ -29,12 +29,11 @@ val_set: !class:sampleflux.core.Flux() test_set: !class:sampleflux.core.Flux() source: !ref:my_split.test -# === RangeSource: a contiguous [start:end) slice over any indexable source === +# === RangeSource: a contiguous [start:stop) slice over any indexable source === first_1000: !class:sampleflux.core.Flux() source: !class:sampleflux.sources.RangeSource() source: !ref:hf_train - start: 0 - end: 1000 + stop: 1000 # === ConcatSource: join several indexable sources into one (then optionally split) === hf_test: !class:sampleflux.sources.HuggingFaceSource() diff --git a/examples/record_pipeline.py b/examples/record_pipeline.py new file mode 100644 index 0000000..ff4ff26 --- /dev/null +++ b/examples/record_pipeline.py @@ -0,0 +1,114 @@ +"""The record data model: a plain dict of typed values, type-dispatched ops, libraries as-is. + +Demonstrates the modality-neutral core of the engine: + +1. a sample is a PLAIN ``dict`` of TYPED values, each owning its metadata — an ``Image`` + carries its layout, a ``Label`` its classes; scalar side values are just more keys; +2. the HEADLINE — ONE pipeline mixing a BARE albumentations transform (invoked natively by + the engine's op-family dispatch: it receives exactly its own ``image``/``mask``/``bboxes`` + keys, one call = one joint draw) with native ops. sampleflux ships NO augmentation of its + own and NO adapter classes — the libraries run as-is; +3. cross-key consistency — ONE ``A.Compose`` draw moves image, mask and bboxes together, + the Label untouched; +4. a native op in the torchvision-v2 authoring style: params drawn once per record in + ``get_params``, a kernel per value type, ``field=`` pinning it to one key; +5. a torchvision ``transforms.v2`` transform as-is — after the EXPLICIT ``v2.ToImage()`` + conversion, exactly like a plain torchvision pipeline (the engine never converts silently). + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import albumentations as A +import numpy as np +import torch +from torchvision.transforms import v2 + +from sampleflux import Image, Label, Mask, Pipeline, Record, Transform, as_transform + + +def make_record(rng: np.random.Generator) -> Record: + """A detection record: image, mask, boxes (albumentations vocabulary), and a class label.""" + return { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), + "mask": Mask((rng.random((16, 20)) > 0.5).astype(np.uint8)), + "bboxes": [[2, 3, 6, 7]], + "labels": ["drone"], + "class": Label("drone_x", classes=["noise", "drone_x"]), + "gain_db": -3.0, # a scalar side value is just another key + } + + +class Brighten(Transform): + """Add a per-record random offset to every Image value (the tv2 authoring pattern). + + Args: + strength: Maximum brightness offset drawn per record. + field: Apply only to this record key. None (default) = every Image value. + """ + + handles = (Image,) + + def __init__(self, strength: float = 0.1, field: str = None) -> None: # type: ignore[assignment] + super().__init__(field=field) + self.strength = strength + self._rng = np.random.default_rng(7) + + def get_params(self, record: Record) -> dict: + return {"offset": self._rng.uniform(0.0, self.strength)} # drawn ONCE per record + + +@Brighten.kernel(Image) +def _brighten_image(value: Image, params: dict) -> Image: + return Image(np.asarray(value) + params["offset"], layout=value.layout) + + +def main() -> None: + rng = np.random.default_rng(0) + + # 1. The record: a plain dict of typed values. + record = make_record(rng) + print("record keys: ", list(record)) + print("image meta: ", f"layout={record['image'].layout} class vocab={record['class'].classes}") + + # 2+3. HEADLINE — bare albumentations (its OWN Compose carries bbox_params) + a native + # op in ONE Pipeline. The engine invokes each op family natively — no wrappers. + flip = A.Compose( + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ) + out = Pipeline([flip, A.GaussNoise(p=1.0), Brighten(strength=0.2)])(record) + assert out is not None + print("\n--- bare albumentations + native op in one pipeline ---") + print("image ->", type(out["image"]).__name__, np.asarray(out["image"]).shape, "(flipped + noised + brightened)") + print("mask ->", type(out["mask"]).__name__, "(flipped with the image — one joint draw)") + print("bboxes ->", record["bboxes"], "->", [[round(v) for v in b] for b in out["bboxes"]], "(W=20)") + print("class ->", type(out["class"]).__name__, repr(out["class"].value), "(no handler — untouched)") + assert np.array_equal(np.asarray(out["mask"]), np.asarray(record["mask"])[:, ::-1]) + assert [round(v) for v in out["bboxes"][0]] == [14, 3, 18, 7] + assert out["class"].value == "drone_x" and out["gain_db"] == -3.0 + assert isinstance(out["image"], Image) and isinstance(out["mask"], Mask) # types survive the library + + # 4. field= pins a type-dispatched op to ONE key (here a no-op: "class" is not an Image). + untouched = Brighten(strength=0.2, field="class")(record) + assert untouched is not None and np.array_equal(np.asarray(untouched["image"]), np.asarray(record["image"])) + + # 5. torchvision v2 as-is: the EXPLICIT conversion first (v2.ToImage: numpy HWC -> CHW + # tv_tensor), then any v2 transform — the engine passes the dict straight through. + tv_out = Pipeline([v2.ToImage()])({"image": np.asarray(record["image"])}) + assert tv_out is not None and isinstance(tv_out["image"], torch.Tensor) and tv_out["image"].shape == (3, 16, 20) + cropped = Pipeline([v2.RandomCrop(8)])(tv_out) + assert cropped is not None and tuple(cropped["image"].shape) == (3, 8, 8) + print("\n--- torchvision v2 as-is (explicit ToImage conversion) ---") + print("image ->", type(cropped["image"]).__name__, tuple(cropped["image"].shape)) + + # 6. A custom function op — no library, no core edit. + doubled = as_transform(lambda d: d * 2, handles=(Mask,), field="mask")(record) + assert doubled is not None + print("\n--- custom function op ---") + print("mask doubled:", np.array_equal(np.asarray(doubled["mask"]), np.asarray(record["mask"]) * 2)) + + print("\nOK") + + +if __name__ == "__main__": + main() diff --git a/examples/typed_pipeline.py b/examples/typed_pipeline.py deleted file mode 100644 index 0eead45..0000000 --- a/examples/typed_pipeline.py +++ /dev/null @@ -1,81 +0,0 @@ -"""The typed-bag data model (proof of concept): a named bag of typed items, type-dispatched transforms. - -Demonstrates the modality-neutral core of the redesign that steps away from -``Sample(input, target, metadata)``: - -1. a ``Sample`` is a NAMED BAG of TYPED ITEMS, each owning its metadata — an ``Image`` - carries its layout, a ``Regions`` its canvas, a ``Label`` its classes; ``input`` / - ``target`` are ROLE TAGS, not fixed positions; -2. the HEADLINE — ONE pipeline of BARE library transforms (each wrapped by its registered - adapter): two torchvision ``transforms.v2`` transforms and an albumentations transform, - each hitting only the field(s) of a type it handles. sampleflux ships NO native - augmentation transforms — the libraries cover that through adapter coercion; -3. cross-field consistency — ONE library flip draw moves Image, Mask and Regions together, - the Label untouched; -4. a custom transform from a plain function (``as_transform``), no library, no core edit. - -Signal-domain items (``Signal`` / ``Spectrogram``) and the ``Fourier`` transform are NOT here — -sampleflux is modality-neutral. They live in ``waivefront.bag`` and register into the SAME -registry; see ``waivefront/examples/typed_signal_pipeline.py`` for the signal + image mix. - -Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). -""" - -import albumentations as A -import numpy as np -from torchvision.transforms import v2 - -from sampleflux import Image, Label, Mask, Pipeline, Regions, Sample, as_transform - - -def make_sample(rng: np.random.Generator) -> Sample: - """A detection sample: an image, its mask, its boxes (targets), and a class label (target).""" - return Sample( - { - "image": Image(rng.random((16, 20, 3)).astype(np.float32)), - "mask": Mask(rng.random((16, 20)) > 0.5), - "regions": Regions(boxes=[[2, 3, 6, 7]], labels=["drone"], canvas=(16, 20)), - "class": Label("drone_x", classes=["noise", "drone_x"]), - }, - roles={"mask": "target", "regions": "target", "class": "target"}, - ) - - -def main() -> None: - rng = np.random.default_rng(0) - - # 1. The named typed bag with role tags. - sample = make_sample(rng) - print("sample: ", sample) - print("inputs: ", list(sample.inputs()), " targets:", list(sample.targets())) - print("image meta: ", f"layout={sample['image'].layout} regions canvas={sample['regions'].canvas}") - - # 2. HEADLINE — one pipeline of BARE library transforms; a registered adapter wraps each, - # and every transform hits only the field(s) of its type. - out = Pipeline( - [ - v2.RandomHorizontalFlip(p=1.0), # torchvision v2: Image + Mask + Regions together (one draw) - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.25, 0.25, 0.25]), # torchvision v2: Image - A.GaussNoise(p=1.0), # albumentations: Image - ] - )(sample) - print("\n--- mixed cross-library pipeline (flip + normalize + noise) ---") - print("image ->", type(out["image"]).__name__, np.asarray(out["image"]).shape, "(flipped + normalized + noised)") - print("mask ->", type(out["mask"]).__name__, "(flipped with the image)") - print("regions->", sample["regions"].boxes, "->", [[round(v) for v in b] for b in out["regions"].boxes], "(W=20)") - print("class ->", type(out["class"]).__name__, repr(out["class"].value), "(no handler — untouched)") - assert np.array_equal(np.asarray(out["mask"]), np.asarray(sample["mask"])[:, ::-1]) - assert [round(v) for v in out["regions"].boxes[0]] == [14, 3, 18, 7] - assert out["class"].value == "drone_x" and out.roles == sample.roles - - # 3. A custom transform from a plain function — no library, no core edit. - brighten = as_transform(lambda d: d + 0.1, handles=(Image,), only=["image"]) - brightened = brighten(sample) - print("\n--- custom function transform ---") - print("image brightened:", np.allclose(np.asarray(brightened["image"]), np.asarray(sample["image"]) + 0.1)) - - print("\nOK") - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 4bf435a..6da9aa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dev = [ "pytest>=7.0.0,<9.0.0", "pytest-cov>=4.0.0,<7.0.0", # CI installs `.[dev]` (unit-tests AND verify-examples), so torchvision here keeps the - # torchvision augmentation adapter + examples exercised in CI without a workflow edit. + # bare torchvision-v2 op family + examples exercised in CI without a workflow edit. "torchvision", ] notebook = [ @@ -45,8 +45,9 @@ notebook = [ ] vision = [ "scipy", - # TorchvisionTransformOp (sampleflux.ops.torchvision) lazy-imports torchvision; the - # extra makes `pip install sampleflux[vision]` the documented way to enable it. + # Bare torchvision transforms.v2 ops run as-is through the engine's op-family dispatch + # (sampleflux.core._apply_op); the extra makes `pip install sampleflux[vision]` the + # documented way to enable them. "torchvision", ] @@ -69,7 +70,6 @@ sampleflux-ops-random-apply = "sampleflux.ops.random_apply" # changes need an editable reinstall before FluxStudio/navigaitor discovery sees the module. sampleflux-ops-configure = "sampleflux.ops.configure" sampleflux-ops-formula = "sampleflux.ops.formula" -sampleflux-ops-transform-chain = "sampleflux.ops.transform_chain" # Context ops (Save/Use/Drop/Apply/Capture/MergeFields) — the graph-plane building blocks lowered from flow: docs sampleflux-ops-context = "sampleflux.ops.context" # The FlowGraph engine (flow: named-step documents + the flow<->ops converters) @@ -81,16 +81,6 @@ sampleflux-ops-numpy = "sampleflux.ops.numpy" sampleflux-ops-torch = "sampleflux.ops.torch" sampleflux-ops-target = "sampleflux.ops.target" sampleflux-ops-image = "sampleflux.ops.image" -# Augmentation adapters over well-known libraries (pair-scoped input+target ops). -# Both modules import WITHOUT their library (lazy imports) so discovery stays safe on -# hosts missing torchvision. Entry-point changes need an editable reinstall -# (`aisland setup`, never --reinstall) before FluxStudio/navigaitor discovery sees them. -sampleflux-ops-albumentations = "sampleflux.ops.albumentations" -sampleflux-ops-torchvision = "sampleflux.ops.torchvision" -# The auto-generated per-transform op families (Alb / Tv) — one node per -# library transform, generated at import time by sampleflux.ops._augment_bridge. -sampleflux-ops-albumentations-transforms = "sampleflux.ops.albumentations_transforms" -sampleflux-ops-torchvision-transforms = "sampleflux.ops.torchvision_transforms" # PrintSampleOp (log/print a per-sample summary). Entry-point changes need an editable reinstall # before FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). sampleflux-ops-debug = "sampleflux.ops.debug" @@ -103,13 +93,11 @@ sampleflux-ops-debug = "sampleflux.ops.debug" sampleflux-storage-hdf5 = "sampleflux.storage.hdf5" sampleflux-storage-zarr = "sampleflux.storage.zarr" sampleflux-storage-directory = "sampleflux.storage.directory" -# The typed-bag model (sampleflux.bag): the Transform/Pipeline/coercion machinery lives in -# sampleflux.bag.transform, which imports without torchvision (adapters lazy-import their -# library). Entry-point changes need an editable reinstall before FluxStudio/navigaitor -# discovery sees the module (`aisland setup`, never --reinstall). -sampleflux-bag-transform = "sampleflux.bag.transform" -# Typed-bag structure ops (SetRole/RenameField/DropField/CopyField/SelectFields) — reshape a -# Sample's named fields; the typed replacement for the classic triple-slot plumbing. +# The record model's op surface: Transform (type-dispatched record ops) + Pipeline (THE +# compose op) live in sampleflux.transform. Entry-point changes need an editable reinstall +# before FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). +sampleflux-transform = "sampleflux.transform" +# Structure ops (RenameField/DropField/CopyField/SelectFields) — reshape a record's entries. sampleflux-ops-structure = "sampleflux.ops.structure" # The runnable orchestration layer: DatasetProcessor (generic source→sink runner) and the # workflow combinators (Sequence/Conditional/Switch + PathExists/Not/AllOf/AnyOf predicates). diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py index 8d70896..29704d8 100644 --- a/sampleflux/__init__.py +++ b/sampleflux/__init__.py @@ -1,56 +1,41 @@ """ SampleFlux: Modular, functional data pipelines. -The data model is the TYPED BAG: a :class:`Sample` is a named bag of typed items (each -owning its metadata), ``input`` / ``target`` are ROLE TAGS on fields, and transforms -dispatch on item TYPE. Import the whole surface from the package top level -(``from sampleflux import Sample, Image, Transform, primary, ...``); the internal module -layout (``sampleflux.bag.*``) is transitional and may be promoted to the package root. +The data model is the RECORD: a sample is a plain ``dict`` of typed values (each value +owning its metadata — an ``Image`` its layout, a ``Label`` its classes), and ops dispatch +on value TYPE (the torchvision-v2 model). Bare albumentations / torchvision ``transforms.v2`` +transforms drop into any ops list AS-IS — the engine invokes each op family natively +(``sampleflux.core._apply_op``). Import the whole surface from the package top level +(``from sampleflux import Record, Image, Transform, Pipeline, ...``). """ -# --- the typed-bag data model + transforms + item codec ------------------------------------ -from sampleflux.bag import ( - ROLES, - EncodedField, - EncodedItem, - FunctionTransform, +# --- shared infrastructure ----------------------------------------------------------------- +from sampleflux.collate import collate, collate_records, get_collate, register_collate, registered_collates +from sampleflux.context import Context +from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp + +# --- the record data model + transforms + item codec ---------------------------------------- +from sampleflux.dispatch import dispatch, register_kernel, registered_kernels +from sampleflux.flow import FlowGraph, from_ops, to_ops +from sampleflux.io import EncodedField, EncodedItem, decode_item, decode_record, encode_item, encode_record, register_io +from sampleflux.items import ( Image, Label, Mask, NDArrayItem, - Pipeline, + Record, Regions, - Role, - Sample, - Transform, - as_transform, - coerce_transform, - decode_item, - decode_sample, - dispatch, - encode_item, - encode_sample, get_item_type, is_item, item_data, item_type_names, item_types, - primary, - register_adapter, - register_io, register_item, - register_kernel, with_data, ) - -# --- shared infrastructure ----------------------------------------------------------------- -from sampleflux.collate import collate, get_collate, register_collate, registered_collates, typed_collate -from sampleflux.context import Context -from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.flow import FlowGraph, from_ops, to_ops from sampleflux.labels import LabelMap from sampleflux.processing import DatasetProcessor -from sampleflux.projection import ProjectionField, SupportsProjection, iter_inputs, iter_targets, num_classes, project +from sampleflux.projection import SupportsProjection, iter_key, num_classes, project from sampleflux.runnable import ( ProgressCallback, ProgressReporting, @@ -60,14 +45,12 @@ runnable_entrypoints, ) from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName +from sampleflux.transform import FunctionTransform, Pipeline, Transform, as_transform from sampleflux.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch __all__ = [ - # ---- typed-bag data model ---- - "Sample", - "Role", - "ROLES", - "primary", + # ---- record data model ---- + "Record", "NDArrayItem", "Image", "Mask", @@ -84,17 +67,16 @@ "Pipeline", "FunctionTransform", "as_transform", - "register_adapter", - "coerce_transform", "dispatch", "register_kernel", + "registered_kernels", "EncodedItem", "EncodedField", "register_io", "encode_item", "decode_item", - "encode_sample", - "decode_sample", + "encode_record", + "decode_record", # ---- shared infrastructure ---- "Context", "Flux", @@ -105,10 +87,10 @@ "from_ops", "to_ops", "collate", + "collate_records", "get_collate", "register_collate", "registered_collates", - "typed_collate", "LabelMap", # ---- sources ---- "HuggingFaceSource", @@ -117,10 +99,8 @@ "ConcatSource", "SplitName", # ---- projection ---- - "ProjectionField", "SupportsProjection", - "iter_inputs", - "iter_targets", + "iter_key", "num_classes", "project", # ---- runnable protocol + orchestration ---- diff --git a/sampleflux/bag/__init__.py b/sampleflux/bag/__init__.py deleted file mode 100644 index 08e98e9..0000000 --- a/sampleflux/bag/__init__.py +++ /dev/null @@ -1,96 +0,0 @@ -"""``sampleflux.bag`` — the typed-bag data model with type-dispatched transforms. - -A sample is a NAMED BAG of TYPED ITEMS (:class:`Sample`), each item owning its own -metadata; ``input``/``target`` are ROLE TAGS on fields, not tuple positions. Transforms -dispatch on item TYPE via a kernel registry, sampling their parameters once per sample so -multi-field consistency (flip image + mask + boxes together) is automatic. External libraries -(torchvision ``transforms.v2``, albumentations) drop into a :class:`Pipeline` bare — a -registered adapter wraps each — and user/domain packages register their own item types, -kernels, adapters, and storage codecs from outside (``register_item`` / ``@Transform.kernel`` -/ ``register_adapter`` / ``register_io``). - -This is THE sampleflux data model (the legacy ``Sample`` triple is being migrated out; it -survives only until every consumer has flipped). Import the public surface from the PACKAGE -TOP LEVEL (``from sampleflux import Sample, Image, Transform, ...``) — the ``bag`` -module path is a transitional home. See ``docs/typed-model.md`` (usage) and -``docs/architecture.md`` (rationale). -""" - -# Import the adapters for their SIDE EFFECT: each registers a coercion matcher so a bare -# torchvision v2 / albumentations transform can be dropped straight into a Pipeline. This does -# NOT import torchvision/albumentations (the adapters lazy-import their library inside method -# bodies), so `import sampleflux.bag` stays library-free — pinned by test_bag_pipeline.py. -from sampleflux.bag import adapters as _adapters # noqa: F401,E402 (registration side effect) -from sampleflux.bag.dispatch import dispatch, register_kernel, registered_kernels -from sampleflux.bag.io import ( - EncodedField, - EncodedItem, - decode_item, - decode_sample, - encode_item, - encode_sample, - register_io, -) -from sampleflux.bag.items import ( - Image, - Label, - Mask, - NDArrayItem, - Regions, - get_item_type, - is_item, - item_data, - item_type_names, - item_types, - register_item, - with_data, -) -from sampleflux.bag.sample import ROLES, Role, Sample, primary -from sampleflux.bag.transform import ( - FunctionTransform, - Pipeline, - Transform, - as_transform, - coerce_transform, - register_adapter, -) - -__all__ = [ - # data model - "Sample", - "Role", - "ROLES", - "primary", - # items - "NDArrayItem", - "Image", - "Mask", - "Regions", - "Label", - "register_item", - "item_types", - "item_type_names", - "get_item_type", - "is_item", - "item_data", - "with_data", - # transforms - "Transform", - "Pipeline", - "FunctionTransform", - "as_transform", - "register_adapter", - "coerce_transform", - # dispatch - "dispatch", - "register_kernel", - "registered_kernels", - # storage codec - "EncodedItem", - "EncodedField", - "register_io", - "encode_item", - "decode_item", - "encode_sample", - "decode_sample", -] diff --git a/sampleflux/bag/adapters/__init__.py b/sampleflux/bag/adapters/__init__.py deleted file mode 100644 index 886061c..0000000 --- a/sampleflux/bag/adapters/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Adapters that run external augmentation libraries as typed-bag transforms. - -These are deliberately NOT imported by :mod:`sampleflux.bag`'s top-level ``__init__`` — each -lazy-imports its library inside method bodies, so ``import sampleflux.bag`` stays safe on a -host without torchvision. Import an adapter directly:: - - from sampleflux.bag.adapters import TorchvisionV2Adapter, AlbumentationsAdapter -""" - -from sampleflux.bag.adapters.albumentations import AlbumentationsAdapter -from sampleflux.bag.adapters.torchvision import TorchvisionV2Adapter - -__all__ = ["TorchvisionV2Adapter", "AlbumentationsAdapter"] diff --git a/sampleflux/bag/adapters/albumentations.py b/sampleflux/bag/adapters/albumentations.py deleted file mode 100644 index 1abcb61..0000000 --- a/sampleflux/bag/adapters/albumentations.py +++ /dev/null @@ -1,102 +0,0 @@ -"""``AlbumentationsAdapter`` — run an albumentations transform over a typed bag. - -albumentations dispatches by keyword NAME (``image=`` / ``mask=`` / ``bboxes=``) rather than by -type, so this adapter maps typed items to those named arguments, calls the transform once (one -draw applied jointly), and maps the result back. It targets ONE image field (the first, or the -one selected via ``only``), plus an optional mask and an optional regions field. - -albumentations operates on numpy HWC images and stays numpy HWC. It is a hard dependency but -imported lazily so module import stays light. -""" - -from typing import Any, List, Optional - -import numpy as np - -from sampleflux.bag.items import Image, Mask, Regions, item_data, with_data -from sampleflux.bag.sample import Sample -from sampleflux.bag.transform import Transform, register_adapter - - -class AlbumentationsAdapter(Transform): - """Wrap one albumentations transform (or ``A.Compose``) as a typed-bag transform. - - Args: - transform: An albumentations transform / ``A.Compose``. Validated lazily on first call. - only: Restrict to these field keys (still type-gated). - """ - - handles = (Image, Mask, Regions) - consumes = (Image,) - optional = (Mask, Regions) - produces = (Image, Mask, Regions) - - def __init__(self, transform: Optional[Any] = None, only: Optional[List[str]] = None) -> None: - super().__init__(only=only) - self.transform = transform - - def __call__(self, sample: Sample) -> Sample: - if self.transform is None: - raise ValueError("AlbumentationsAdapter: 'transform' must be set before calling.") - - img_key = self._pick(sample, Image) - if img_key is None: - return sample # albumentations needs an image; nothing to do - mask_key = self._pick(sample, Mask) - reg_key = self._pick(sample, Regions) - - kwargs: dict = {"image": np.asarray(item_data(sample[img_key]))} - if mask_key is not None: - kwargs["mask"] = np.asarray(item_data(sample[mask_key])) - if reg_key is not None: - regions = sample[reg_key] - kwargs["bboxes"] = [list(box) for box in regions.boxes] - kwargs["labels"] = list(regions.labels) if regions.labels is not None else [0] * len(regions.boxes) - - out = self._compose(need_bbox=reg_key is not None)(**kwargs) - - result = sample.replace_field(img_key, with_data(sample[img_key], out["image"])) - if mask_key is not None: - result = result.replace_field(mask_key, with_data(sample[mask_key], out["mask"])) - if reg_key is not None: - regions = sample[reg_key] - result = result.replace_field( - reg_key, - Regions( - boxes=[list(box) for box in out["bboxes"]], - labels=list(out["labels"]), - scores=regions.scores, - canvas=regions.canvas, - ), - ) - return result - - def _pick(self, sample: Sample, item_type: type) -> Optional[str]: - """The first field of ``item_type`` (honoring ``only``), or ``None``.""" - for key, item in sample.items(): - if self.only is not None and key not in self.only: - continue - if isinstance(item, item_type): - return key - return None - - def _compose(self, need_bbox: bool) -> Any: - """The live ``A.Compose`` — a prebuilt Compose is used as-is; a bare transform is wrapped.""" - import albumentations as A - from albumentations.core.composition import BaseCompose - - if isinstance(self.transform, BaseCompose): - return self.transform - bbox_params = A.BboxParams(format="pascal_voc", label_fields=["labels"]) if need_bbox else None - return A.Compose([self.transform], bbox_params=bbox_params) - - -def is_albumentations_transform(obj: Any) -> bool: - """True for an albumentations transform / ``Compose`` — by MRO module name (no import here).""" - return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(obj).__mro__) - - -# Drop a bare albumentations transform straight into a Pipeline — wrapped in an AlbumentationsAdapter. -register_adapter(is_albumentations_transform, AlbumentationsAdapter) - -__all__ = ["AlbumentationsAdapter", "is_albumentations_transform"] diff --git a/sampleflux/bag/adapters/torchvision.py b/sampleflux/bag/adapters/torchvision.py deleted file mode 100644 index a67e13f..0000000 --- a/sampleflux/bag/adapters/torchvision.py +++ /dev/null @@ -1,143 +0,0 @@ -"""``TorchvisionV2Adapter`` — run a torchvision ``transforms.v2`` transform over a typed bag. - -The typed bag and torchvision's ``tv_tensors`` are the SAME shape — a heterogeneous structure -of typed leaves — so this adapter is thin: it maps our items to ``tv_tensors`` -(:class:`~sampleflux.bag.items.Image`\\ →``Image``, -:class:`~sampleflux.bag.items.Mask`\\ →``Mask``, -:class:`~sampleflux.bag.items.Regions`\\ →``BoundingBoxes``), hands the WHOLE dict to the v2 -transform (v2 draws its random parameters once and applies them across every leaf, so a -geometric augmentation stays consistent across image / mask / boxes), and maps the result -back into typed items with their metadata preserved. - -torchvision is lazy-imported; this module imports without it installed (a missing install -raises a clear error pointing at the ``sampleflux[vision]`` extra). -""" - -from typing import Any, List, Optional, Tuple - -import numpy as np - -from sampleflux.bag.items import Image, Mask, Regions, item_data, with_data -from sampleflux.bag.sample import Sample -from sampleflux.bag.transform import Transform, register_adapter - - -def _import_v2() -> Any: - try: - from torchvision.transforms import v2 - except ImportError as exc: # pragma: no cover - exercised only without torchvision - raise ImportError( - "TorchvisionV2Adapter requires torchvision (transforms.v2 / tv_tensors). " - 'Install it via `pip install "sampleflux[vision]"`.' - ) from exc - return v2 - - -class TorchvisionV2Adapter(Transform): - """Wrap one ``transforms.v2`` transform (or ``v2.Compose``) as a typed-bag transform. - - Args: - transform: A ``transforms.v2`` transform / ``v2.Compose``. Validated lazily on first call. - only: Restrict to these field keys (still type-gated). - """ - - handles = (Image, Mask, Regions) - consumes = (Image,) - optional = (Mask, Regions) - produces = (Image, Mask, Regions) - - def __init__(self, transform: Optional[Any] = None, only: Optional[List[str]] = None) -> None: - super().__init__(only=only) - self.transform = transform - - def __call__(self, sample: Sample) -> Sample: - import torch - from torchvision import tv_tensors - - _import_v2() # raise the actionable extra hint before any torchvision use - if self.transform is None: - raise ValueError("TorchvisionV2Adapter: 'transform' must be set before calling.") - - canvas = _canvas_size(sample) - structure: dict = {} - for key, item in sample.items(): - if self.only is not None and key not in self.only: - continue - wrapped = _wrap(item, tv_tensors, torch, canvas) - if wrapped is not None: - structure[key] = wrapped - if not structure: - return sample - - out_structure = self.transform(structure) - out = sample - for key, wrapped_out in out_structure.items(): - out = out.replace_field(key, _unwrap(sample[key], wrapped_out, torch)) - return out - - -def _canvas_size(sample: Sample) -> Optional[Tuple[int, int]]: - """``(H, W)`` from the first Image/Mask field — the reference frame for bounding boxes.""" - for _, item in sample.items(): - if isinstance(item, (Image, Mask)): - arr = item_data(item) - if isinstance(item, Image) and getattr(item, "layout", "HWC") == "CHW" and arr.ndim == 3: - return int(arr.shape[1]), int(arr.shape[2]) - if arr.ndim >= 2: - return int(arr.shape[0]), int(arr.shape[1]) - return None - - -def _wrap(item: Any, tv_tensors: Any, torch: Any, canvas: Optional[Tuple[int, int]]) -> Any: - """Our item → a ``tv_tensors`` carrier (``None`` for a type torchvision does not handle).""" - if isinstance(item, Image): - arr = item_data(item) - tensor = torch.as_tensor(np.ascontiguousarray(arr)) - if getattr(item, "layout", "HWC") == "HWC" and tensor.ndim == 3: - tensor = tensor.permute(2, 0, 1) - if tensor.ndim == 2: - tensor = tensor.unsqueeze(0) - return tv_tensors.Image(tensor) - if isinstance(item, Mask): - return tv_tensors.Mask(torch.as_tensor(np.ascontiguousarray(item_data(item)))) - if isinstance(item, Regions): - size = item.canvas or canvas - if size is None: - raise ValueError( - "TorchvisionV2Adapter: Regions need a canvas (H, W) — set Regions.canvas or include an Image field." - ) - boxes = torch.as_tensor(np.asarray(item.boxes, dtype=np.float32).reshape(-1, 4)) - return tv_tensors.BoundingBoxes(boxes, format="XYXY", canvas_size=size) - return None - - -def _unwrap(original: Any, wrapped_out: Any, torch: Any) -> Any: - """A ``tv_tensors`` result → our item type, metadata preserved.""" - if isinstance(original, Image): - tensor = wrapped_out.as_subclass(torch.Tensor) - arr = tensor.detach().cpu().numpy() - if getattr(original, "layout", "HWC") == "HWC" and arr.ndim == 3: - arr = np.transpose(arr, (1, 2, 0)) - return with_data(original, arr) - if isinstance(original, Mask): - return with_data(original, wrapped_out.as_subclass(torch.Tensor).detach().cpu().numpy()) - if isinstance(original, Regions): - boxes = wrapped_out.as_subclass(torch.Tensor).detach().cpu().numpy().reshape(-1, 4).tolist() - return Regions(boxes=boxes, labels=original.labels, scores=original.scores, canvas=original.canvas) - return original # pragma: no cover - only wrapped types reach here - - -def is_torchvision_v2_transform(obj: Any) -> bool: - """True for a torchvision ``transforms.v2`` transform / ``Compose`` — by MRO module name. - - Inspects the object's own class MRO (which the caller already imported), so it recognises v2 - objects WITHOUT importing torchvision here; v1 ``torchvision.transforms.transforms`` objects do - not match (they don't handle ``tv_tensors``). - """ - return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(obj).__mro__) - - -# Drop a bare v2 transform straight into a Pipeline — it is wrapped in a TorchvisionV2Adapter. -register_adapter(is_torchvision_v2_transform, TorchvisionV2Adapter) - -__all__ = ["TorchvisionV2Adapter", "is_torchvision_v2_transform"] diff --git a/sampleflux/bag/sample.py b/sampleflux/bag/sample.py deleted file mode 100644 index c2152d6..0000000 --- a/sampleflux/bag/sample.py +++ /dev/null @@ -1,211 +0,0 @@ -"""``Sample`` — the named bag of typed items that replaces ``Sample(input, target, metadata)``. - -A sample is an ordered mapping ``name -> item`` (see :mod:`sampleflux.bag.items`), plus a -per-key ROLE tag. This gives every field BOTH a name (the key — the albumentations dispatch -axis) and a type (the item — the torchvision dispatch axis), and it makes ``input`` / ``target`` -ordinary tags read only at the train / collate / sink boundary rather than fixed tuple -positions. A field can change role without moving keys; auxiliary items (masks, derived -params) are simply tagged ``aux`` and excluded from both ``inputs()`` and ``targets()``. - -``Sample`` is immutable — every mutator returns a NEW sample (copy-on-write), mirroring -the ``Sample._replace`` idiom the legacy engine already relies on, so a transform never -aliases its input. -""" - -from typing import Any, Dict, Iterator, Mapping, Optional, Tuple - -import numpy as np -from typing_extensions import Literal, get_args - -__all__ = ["Sample", "Role", "ROLES", "primary"] - -#: The closed set of field roles. ``input`` / ``target`` drive the train boundary; ``aux`` is -#: a helper field (mask, derived param) in neither; ``pred`` is a model prediction. Closed -#: ``Literal`` so a typo fails at the call site and UIs enumerate the choices via ``get_args``. -Role = Literal["input", "target", "aux", "pred"] -ROLES: Tuple[str, ...] = get_args(Role) - -_DEFAULT_ROLE: Role = "input" - - -class Sample: - """An ordered, immutable bag of typed items with per-field role tags. - - Construct from a mapping of items (roles default to ``input``); pass ``roles`` to tag - specific keys:: - - s = Sample( - {"image": Image(rgb), "regions": Regions(boxes), "class": Label("drone")}, - roles={"regions": "target", "class": "target"}, - ) - s.inputs() # {"image": Image(...)} - s.targets() # {"regions": Regions(...), "class": Label(...)} - s2 = s.set_role("regions", "aux") # copy-on-write - """ - - __slots__ = ("_fields", "_roles") - - def __init__( - self, - fields: Optional[Mapping[str, Any]] = None, - roles: Optional[Mapping[str, Role]] = None, - ) -> None: - self._fields: Dict[str, Any] = dict(fields or {}) - roles = roles or {} - for key, role in roles.items(): - if key not in self._fields: - raise KeyError(f"Sample: role given for unknown field {key!r}") - if role not in ROLES: - raise ValueError(f"Sample: invalid role {role!r} for {key!r} (allowed: {list(ROLES)})") - self._roles: Dict[str, Role] = {key: roles.get(key, _DEFAULT_ROLE) for key in self._fields} - - # --- read views ------------------------------------------------------- - @property - def fields(self) -> Dict[str, Any]: - """A shallow copy of the ``name -> item`` mapping (mutating it does not touch the sample).""" - return dict(self._fields) - - @property - def roles(self) -> Dict[str, Role]: - """A shallow copy of the ``name -> role`` mapping.""" - return dict(self._roles) - - def __getitem__(self, key: str) -> Any: - return self._fields[key] - - def __contains__(self, key: object) -> bool: - return key in self._fields - - def __iter__(self) -> Iterator[str]: - return iter(self._fields) - - def __len__(self) -> int: - return len(self._fields) - - def keys(self) -> Iterator[str]: - return iter(self._fields) - - def items(self) -> Iterator[Tuple[str, Any]]: - return iter(self._fields.items()) - - def role_of(self, key: str) -> Role: - """The role tag of ``key``.""" - return self._roles[key] - - def of_role(self, role: Role) -> Dict[str, Any]: - """The ``name -> item`` fields tagged ``role`` (insertion order preserved).""" - return {key: item for key, item in self._fields.items() if self._roles[key] == role} - - def inputs(self) -> Dict[str, Any]: - """The fields tagged ``input`` — what the model consumes.""" - return self.of_role("input") - - def targets(self) -> Dict[str, Any]: - """The fields tagged ``target`` — what the loss consumes.""" - return self.of_role("target") - - def aux(self) -> Dict[str, Any]: - """The fields tagged ``aux`` — helpers in neither inputs nor targets.""" - return self.of_role("aux") - - def items_of_type(self, *types: type) -> Iterator[Tuple[str, Any]]: - """Yield ``(key, item)`` for every field whose item is an instance of one of ``types``.""" - for key, item in self._fields.items(): - if isinstance(item, types): - yield key, item - - # --- copy-on-write mutators ------------------------------------------ - def replace_field(self, key: str, item: Any) -> "Sample": - """A copy with ``key`` set to ``item`` (added if new; role preserved, else ``input``).""" - fields = dict(self._fields) - fields[key] = item - return Sample(fields, {**self._roles, key: self._roles.get(key, _DEFAULT_ROLE)}) - - def set_role(self, key: str, role: Role) -> "Sample": - """A copy with ``key``'s role set to ``role``.""" - if key not in self._fields: - raise KeyError(f"Sample.set_role: unknown field {key!r}") - if role not in ROLES: - raise ValueError(f"Sample.set_role: invalid role {role!r} (allowed: {list(ROLES)})") - return Sample(dict(self._fields), {**self._roles, key: role}) - - def drop(self, key: str) -> "Sample": - """A copy without ``key``.""" - fields = dict(self._fields) - roles = dict(self._roles) - fields.pop(key, None) - roles.pop(key, None) - return Sample(fields, roles) - - def rename(self, src: str, dst: str) -> "Sample": - """A copy with field ``src`` renamed to ``dst`` (role travels; position moves to the end). - - Renaming onto an existing ``dst`` replaces it (last-write-wins, consistent with - :meth:`merge`). Unknown ``src`` raises. - """ - if src not in self._fields: - raise KeyError(f"Sample.rename: unknown field {src!r}") - fields = dict(self._fields) - roles = dict(self._roles) - item = fields.pop(src) - role = roles.pop(src) - fields.pop(dst, None) - roles.pop(dst, None) - fields[dst] = item - roles[dst] = role - return Sample(fields, roles) - - # --- fan-in ------------------------------------------------------------ - @classmethod - def merge(cls, *samples: "Sample") -> "Sample": - """The ordered UNION of several samples' fields — the typed fan-in primitive. - - Fields AND their roles are united in listed order; on a key collision the - LAST-listed sample wins (value and role) — the deterministic slot-order rule that - replaces the classic metadata dict-merge. Avoid a deliberate collision by renaming - on the producing branch (:meth:`rename` / the ``RenameField`` op), not with merge - policy knobs. - """ - fields: Dict[str, Any] = {} - roles: Dict[str, Role] = {} - for sample in samples: - if not isinstance(sample, Sample): - raise TypeError(f"Sample.merge: expected Sample, got {type(sample).__name__}") - fields.update(sample._fields) - roles.update(sample._roles) - return cls(fields, roles) - - # --- equality / repr -------------------------------------------------- - def __eq__(self, other: object) -> bool: - if not isinstance(other, Sample): - return NotImplemented - if self._roles != other._roles or list(self._fields) != list(other._fields): - return False - return all(_field_equal(self._fields[k], other._fields[k]) for k in self._fields) - - def __repr__(self) -> str: - parts = ", ".join(f"{key}={type(item).__name__}[{self._roles[key]}]" for key, item in self._fields.items()) - return f"Sample({parts})" - - -def primary(sample: Sample, role: Role = "input") -> Tuple[str, Any]: - """The FIRST field of ``role`` in insertion order, as ``(key, item)``. - - The sanctioned answer to "the input" / "the target" of a bag: engines, ``bind``, and - ``Apply``-style parameter injection use it when no explicit field key is given. Raises - ``KeyError`` (naming the sample's fields) when no field carries the role. - """ - for key, item in sample.items(): - if sample.role_of(key) == role: - return key, item - raise KeyError(f"primary: no field with role {role!r} (fields: {list(sample.keys()) or ''})") - - -def _field_equal(a: Any, b: Any) -> bool: - """Value equality that is robust to array-valued items (elementwise ``==`` is not a bool).""" - if isinstance(a, np.ndarray) or isinstance(b, np.ndarray): - return type(a) is type(b) and np.array_equal(np.asarray(a), np.asarray(b)) - try: - return bool(a == b) - except Exception: # pragma: no cover - exotic payloads fall back to identity - return a is b diff --git a/sampleflux/bag/transform.py b/sampleflux/bag/transform.py deleted file mode 100644 index 8f9f4f3..0000000 --- a/sampleflux/bag/transform.py +++ /dev/null @@ -1,198 +0,0 @@ -"""``Transform`` — type-dispatched sample transforms with once-per-sample parameters. - -A transform samples its random / configured parameters ONCE per sample (:meth:`Transform.get_params`), -then walks the bag and, for each field whose item type it handles, applies the registered -kernel (:mod:`sampleflux.bag.dispatch`). Fields it does not handle pass through untouched. - -Two properties fall out of this shape for free: - -* **Cross-field consistency.** Because params are sampled once and shared, one transform - moves every spatial field with the SAME decision (a torchvision-v2 flip dropped into a - :class:`Pipeline` flips an :class:`~sampleflux.bag.items.Image`, its - :class:`~sampleflux.bag.items.Mask`, and its :class:`~sampleflux.bag.items.Regions` - together) — the thing the old flat-metadata model could not express. -* **Open extension.** A new item type is taught to an existing transform with one - ``@Transform.kernel(NewType)`` registration and no core edit. - -Targeting is by TYPE, with an optional ``only=[keys]`` filter for surgical control (touch -only the named fields even if others share a handled type). - -sampleflux ships NO native augmentation transforms — geometric/photometric augmentation -comes from the libraries (torchvision ``transforms.v2`` / albumentations) through the -adapter coercion registry below; a domain package registers its own transforms (e.g. a -signal FFT) via the same ``Transform`` + kernel machinery from outside. - -Graph annotations (``consumes`` / ``optional`` / ``produces`` — item-type tuples) describe a -transform's item-level inputs/outputs for a visual editor's typed side sockets; they are -declarative metadata, not enforced at runtime here. -""" - -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple - -from sampleflux.bag.dispatch import Kernel, dispatch, register_kernel -from sampleflux.bag.items import item_data, with_data -from sampleflux.bag.sample import Sample - -__all__ = [ - "Transform", - "Pipeline", - "FunctionTransform", - "as_transform", - "register_adapter", - "coerce_transform", -] - - -class Transform: - """Base class for type-dispatched transforms (see the module docstring). - - Subclasses declare ``handles`` (the item types they process) and register a kernel per - type via ``@MyTransform.kernel(ItemType)``. Override :meth:`get_params` to sample shared - parameters once per sample. - - Args: - only: Restrict the transform to these field keys (still type-gated). ``None`` = every - field of a handled type. - """ - - #: Item types this transform processes (a field of another type passes through). - handles: Tuple[type, ...] = () - #: Graph metadata — required input item types (defaults to ``handles`` when empty). - consumes: Tuple[type, ...] = () - #: Graph metadata — optional input item types. - optional: Tuple[type, ...] = () - #: Graph metadata — item types this transform adds or changes. - produces: Tuple[type, ...] = () - - def __init__(self, only: Optional[List[str]] = None) -> None: - self.only = list(only) if only else None - - @classmethod - def kernel(cls, item_type: type) -> Callable[[Kernel], Kernel]: - """Register a kernel for ``item_type`` on this transform (decorator over :func:`register_kernel`).""" - return register_kernel(cls, item_type) - - def get_params(self, sample: Sample) -> Dict[str, Any]: - """Sample the shared parameters for one call. Default: no params.""" - return {} - - def __call__(self, sample: Sample) -> Sample: - params = self.get_params(sample) - out = sample - for key, item in sample.items(): - if self.only is not None and key not in self.only: - continue - kernel = dispatch(type(self), type(item)) - if kernel is None: - continue - out = out.replace_field(key, kernel(item, params)) - return out - - def decode(self, sample: Sample) -> Sample: - """The inverse transform (for visualization / back-projection). Not defined by default.""" - raise NotImplementedError(f"{type(self).__name__} defines no decode (inverse)") - - -# --------------------------------------------------------------------------- -# Adapter coercion registry — drop a FOREIGN transform (a torchvision v2 transform, -# an albumentations transform, a user library object) straight into a Pipeline and the -# right adapter wraps it. Open for extension: register a matcher + factory for any type. -# --------------------------------------------------------------------------- -#: A matcher decides whether an object is adaptable; a factory wraps it into a Transform. -AdapterMatcher = Callable[[Any], bool] -AdapterFactory = Callable[[Any], "Transform"] - -_ADAPTERS: List[Tuple[AdapterMatcher, AdapterFactory]] = [] - - -def register_adapter(matcher: AdapterMatcher, factory: AdapterFactory) -> AdapterFactory: - """Teach :func:`coerce_transform` (and thus ``Pipeline``) to adapt a foreign transform type. - - ``matcher(obj) -> bool`` recognises the objects this adapter handles (keep it import-free — - inspect ``type(obj).__mro__`` module names rather than importing the library); ``factory(obj)`` - returns a :class:`Transform` wrapping it. Later registrations win on ties (checked last-first). - - Example — make a user's library transforms droppable into a ``Pipeline``:: - - register_adapter( - lambda o: type(o).__module__.startswith("mylib"), - lambda o: MyLibAdapter(o), - ) - """ - _ADAPTERS.append((matcher, factory)) - return factory - - -def coerce_transform(obj: Any) -> "Transform": - """Return ``obj`` if it is already a :class:`Transform`, else adapt it via a registered adapter. - - Raises a clear ``TypeError`` naming the object when no adapter matches (wrap it with - :func:`as_transform` / an explicit adapter, or :func:`register_adapter`). - """ - if isinstance(obj, Transform): - return obj - for matcher, factory in reversed(_ADAPTERS): - try: - matched = matcher(obj) - except Exception: # pragma: no cover - a defensive matcher never breaks coercion - matched = False - if matched: - return factory(obj) - raise TypeError( - f"Pipeline: don't know how to adapt {type(obj).__module__}.{type(obj).__name__} into a " - "Transform. Wrap it with as_transform(...) or an adapter, or register one via " - "sampleflux.bag.register_adapter(matcher, factory)." - ) - - -class Pipeline: - """Sequential application of transforms — ``Pipeline([a, b, c])(sample)`` is ``c(b(a(sample)))``. - - Elements are COERCED (:func:`coerce_transform`): a :class:`Transform` is used as-is, and a - foreign transform (a torchvision ``transforms.v2`` transform, an albumentations transform, a - registered user type) is wrapped by its adapter automatically — so libraries drop straight in:: - - Pipeline([v2.RandomHorizontalFlip(p=0.5), v2.Normalize(mean, std), A.GaussNoise(p=1.0)])(sample) - - For surgical control (targeting one field key), construct the adapter explicitly with ``only=``. - """ - - def __init__(self, transforms: Sequence[Any]) -> None: - self.transforms: List[Transform] = [coerce_transform(t) for t in transforms] - - def __call__(self, sample: Sample) -> Sample: - for transform in self.transforms: - sample = transform(sample) - return sample - - def __repr__(self) -> str: - return f"Pipeline([{', '.join(type(t).__name__ for t in self.transforms)}])" - - -class FunctionTransform(Transform): - """A transform that applies one plain function ``fn(data) -> data`` to every handled field. - - The escape hatch for custom transforms: no kernel registration, no subclass — wrap a - function and say which item types it applies to (via :func:`as_transform`). - """ - - def __init__(self, fn: Callable[[Any], Any], handles: Sequence[type], only: Optional[List[str]] = None) -> None: - super().__init__(only=only) - self._fn = fn - self.handles = tuple(handles) - - def __call__(self, sample: Sample) -> Sample: - out = sample - for key, item in sample.items(): - if self.only is not None and key not in self.only: - continue - if isinstance(item, self.handles): - out = out.replace_field(key, with_data(item, self._fn(item_data(item)))) - return out - - -def as_transform( - fn: Callable[[Any], Any], handles: Sequence[type], only: Optional[List[str]] = None -) -> FunctionTransform: - """Wrap a plain ``fn(data) -> data`` as a :class:`FunctionTransform` over ``handles``.""" - return FunctionTransform(fn, handles, only=only) diff --git a/sampleflux/collate.py b/sampleflux/collate.py index 6fcdfeb..5f15fa4 100644 --- a/sampleflux/collate.py +++ b/sampleflux/collate.py @@ -4,23 +4,25 @@ ``FlowGraph.batch`` yield ``list``\\ s of N items) and a COLLATE function stacks a group into one batched carrier. This registry gives consumer packages ONE addressable home for their task collates — consumers ``register_collate`` their task collates additively, and -callers dispatch by key or by the default typed collate. +callers dispatch by key or by the default record collate. The string keys primarily serve AI-callable (MCP) tool surfaces, which pass JSON-serializable names — never function objects — and enumerate the legal values via :func:`registered_collates`; in Python (and in YAML via a dotted ``!ref:`` to the function), passing a collate function directly remains the normal path. -The default registered here is ``"typed"`` — N :class:`~sampleflux.bag.sample.Sample` bags -collated into ONE batched bag (payloads stacked per field, per-item attrs as lists, roles -preserved). Consumer conventions are deliberately NOT unified here; the registry is additive. +The default registered here is ``"record"`` — N plain record dicts collated into ONE +batched record (array payloads stacked per key, per-record item attrs as lists, plain +values gathered into lists). Consumer conventions are deliberately NOT unified here; the +registry is additive. """ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from loggair import get_logger -from sampleflux.bag.sample import Sample +from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from sampleflux.items import Record logger = get_logger(__name__) @@ -28,7 +30,7 @@ _REGISTRY: Dict[str, CollateFn] = {} -__all__ = ["CollateFn", "collate", "get_collate", "register_collate", "registered_collates", "typed_collate"] +__all__ = ["CollateFn", "collate", "collate_records", "get_collate", "register_collate", "registered_collates"] def register_collate(key: str) -> Callable[[CollateFn], CollateFn]: @@ -69,12 +71,12 @@ def registered_collates() -> Tuple[str, ...]: def collate(items: Sequence[Any], key: Optional[str] = None) -> Any: """Collate ``items`` into one batched carrier. - ``key`` picks a registered collate explicitly; omitted, the default ``"typed"`` collate - is used (every carrier is a :class:`~sampleflux.bag.sample.Sample` bag). An empty batch raises. + ``key`` picks a registered collate explicitly; omitted, the default ``"record"`` collate + is used (every carrier is a plain record dict). An empty batch raises. """ if not items: raise ValueError("collate: cannot collate an empty batch") - return get_collate(key or "typed")(items) + return get_collate(key or "record")(items) def _stack(values: List[Any]) -> Any: @@ -99,36 +101,37 @@ def _stack(values: List[Any]) -> Any: return list(values) -@register_collate("typed") -def typed_collate(items: Sequence[Any]) -> Any: - """The typed-bag collate: N ``Sample``\\ s → ONE batched ``Sample``. +@register_collate("record") +def collate_records(items: Sequence[Record]) -> Record: + """The record collate: N record dicts → ONE batched record dict. - Per field (union of keys is NOT taken — every sample must carry the same fields, a - mismatch raises): payloads are stacked via :func:`_stack` (torch → stacked tensor, - numpy → stacked array, else a list) and each declared item attr becomes a LIST of - per-item values. Array items come back as the SAME item type over the stacked payload; - wrapper items likewise (attrs as lists). Roles are preserved. Trainers read - ``primary(batch)`` / ``batch.targets()``. + Per key (union of keys is NOT taken — every record must carry the same keys, a + mismatch raises): typed values encode through :func:`~sampleflux.io.encode_item`, + payloads are stacked via :func:`_stack` (torch → stacked tensor, numpy → stacked + array, else a list) and each declared item attr becomes a LIST of per-record values, + decoding back into ONE batched item of the same type. A ``"plain"``-tagged value + (a scalar / string / bare value) batches as the plain LIST of per-record values. """ - from sampleflux.bag.io import EncodedItem, decode_item, encode_item - if not items: - raise ValueError("typed_collate: cannot collate an empty batch") + raise ValueError("collate_records: cannot collate an empty batch") first = items[0] - if not isinstance(first, Sample): - raise TypeError(f"typed_collate: expected Sample items, got {type(first).__name__}") + if not isinstance(first, dict): + raise TypeError(f"collate_records: expected record dicts, got {type(first).__name__}") keys = list(first.keys()) - for i, sample in enumerate(items): - if not isinstance(sample, Sample) or list(sample.keys()) != keys: + for i, record in enumerate(items): + if not isinstance(record, dict) or list(record.keys()) != keys: raise ValueError( - f"typed_collate: item {i} fields {list(sample.keys()) if isinstance(sample, Sample) else '?'} " - f"do not match the batch fields {keys} — collate requires a homogeneous batch." + f"collate_records: item {i} keys {list(record.keys()) if isinstance(record, dict) else '?'} " + f"do not match the batch keys {keys} — collate requires a homogeneous batch." ) - fields: Dict[str, Any] = {} + batched: Record = {} for key in keys: - encoded = [encode_item(sample[key]) for sample in items] + encoded = [encode_item(record[key]) for record in items] type_name = encoded[0].type_name + if type_name == PLAIN_TYPE: + batched[key] = [e.payload for e in encoded] + continue stacked_payload = _stack([e.payload for e in encoded]) if encoded[0].payload is not None else None batched_attrs = {name: [e.attrs.get(name) for e in encoded] for name in encoded[0].attrs} - fields[key] = decode_item(EncodedItem(type_name=type_name, payload=stacked_payload, attrs=batched_attrs)) - return Sample(fields, {key: first.role_of(key) for key in keys}) + batched[key] = decode_item(EncodedItem(type_name=type_name, payload=stacked_payload, attrs=batched_attrs)) + return batched diff --git a/sampleflux/context.py b/sampleflux/context.py index d519642..9032f7c 100644 --- a/sampleflux/context.py +++ b/sampleflux/context.py @@ -1,7 +1,7 @@ """Per-sample named-cell store — the graph data plane for graph-shaped pipelines. A :class:`Context` holds named **cells** for exactly one sample's trip through the op -list: branch snapshots (a cell holding a :class:`~sampleflux.sample.Sample`), captured +list: branch snapshots (a cell holding a record dict), captured ``@output`` values, and per-sample parameters. The context ops in :mod:`sampleflux.ops.context` (``Save`` / ``Use`` / ``Drop`` / ``Apply`` / ``Capture`` / ``Mix``) move data between the linear sample stream and these cells, which is what lets diff --git a/sampleflux/core.py b/sampleflux/core.py index eb5da29..5ef0688 100644 --- a/sampleflux/core.py +++ b/sampleflux/core.py @@ -10,32 +10,72 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.bag.items import item_data, with_data -from sampleflux.bag.sample import Role, Sample, primary from sampleflux.context import Context, activate -from sampleflux.projection import ProjectionField +from sampleflux.items import NDArrayItem, Record, item_data, with_data logger = get_logger(__name__) -# Role a projection field maps onto in the typed bag (``metadata`` -> the ``aux`` role). -_PROJECTION_ROLES: Dict[str, str] = {"input": "input", "target": "target", "metadata": "aux"} - def _op_expands(op: Any) -> bool: """True when an op is a 1→N expanding op (explicit ``EXPANDS = True`` class attribute).""" return bool(getattr(op, "EXPANDS", False)) -def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: - """Apply one op to the typed :class:`Sample` bag verbatim. +#: The record keys albumentations understands — its OWN target vocabulary. An albumentations +#: op receives exactly these keys (the ones present) and nothing else, so extra record +#: entries (scalars, domain items) never reach a library that would reject them. +_ALB_KEYS: Tuple[str, ...] = ("image", "mask", "masks", "bboxes", "keypoints", "labels") + + +def _is_albumentations(op: Any) -> bool: + """True for an albumentations transform / ``Compose`` — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(op).__mro__) + + +def _is_torchvision_v2(op: Any) -> bool: + """True for a torchvision ``transforms.v2`` transform — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(op).__mro__) + + +def _apply_op(record: Record, op: Any) -> Optional[Record]: + """Apply one op to the record dict — the engine's op-FAMILY dispatch. The single op-application chokepoint shared by the sequential, parallel (via - :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths. A transform - takes the whole bag and returns a new bag (or ``None`` to drop the sample); composing - ops (``Parallel`` / ``Enable`` / ``TransformChain`` / ``RandomApply`` / the context ops) - route their inner ops through here so every op is applied identically. + :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths; composing + ops (``Pipeline`` / ``Parallel`` / ``Enable`` / ``RandomApply`` / the context ops) + route their inner ops through here so every op is applied identically. Each op family + is invoked the way its library expects — no wrapper/adapter classes: + + * **albumentations** — dispatches by KWARG NAME: the op receives exactly its own target + keys present in the record (``image``/``mask``/``bboxes``/…), one call = one joint + draw across them. Array outputs are re-wrapped in the incoming value's item type + (``with_data``) so an ``Image``/``Mask`` keeps its type and metadata. Box-carrying + augmentation belongs in albumentations' own ``A.Compose(..., bbox_params=...)`` + (dropped into the ops list bare) — format handling is Compose's job in that library. + * **torchvision v2** — natively walks the dict, samples params once, transforms + tensor/tv_tensor/PIL leaves and passes everything else through: called as-is. + * **anything else** — a native/wiring op ``record -> Optional[Record]`` (``None`` drops + the record — filter semantics). """ - return cast(Optional[Sample], op(sample)) + if _is_albumentations(op): + kwargs = {k: record[k] for k in _ALB_KEYS if k in record} + if not kwargs: + logger.debug( + f"albumentations op {type(op).__name__} received no known keys " + f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." + ) + return record + out = op(**kwargs) + merged = dict(record) + for key, value in out.items(): + original = record.get(key) + if isinstance(original, NDArrayItem) and not isinstance(value, NDArrayItem): + value = with_data(original, value) + merged[key] = value + return merged + if _is_torchvision_v2(op): + return cast(Record, op(record)) + return cast(Optional[Record], op(record)) def _describe_deferred_source(source: Any) -> str: @@ -61,50 +101,62 @@ def _fluid_source_guidance(source: Any) -> str: def _fluid_op_guidance(op: Any, index: int) -> str: - """Build an actionable message when a Flux op is still a Confluid Fluid.""" + """Build an actionable message when a Flux op marker cannot be materialized.""" return ( - f"Flux.ops[{index}] is still a deferred Confluid marker: {_describe_deferred_source(op)}. " - "Ops must be live callables at iteration time. Fixes: (a) in YAML, write each op as " - "`!class:X()` (with parens) so it becomes an Instance and is materialized at load " - "time; (b) or call `flow(op)` on the op before handing it to Flux." + f"Flux.ops[{index}] is a deferred Confluid marker that could not be materialized: " + f"{_describe_deferred_source(op)}. Fixes: (a) in YAML, write the op as `!class:X()` " + "(with parens) so it becomes an Instance and is materialized at load time; (b) or " + "call `flow(op)` on the op before handing it to Flux." ) def _check_ops_materialized(ops: List[Any]) -> None: - """Raise a single actionable error if any op is still a Confluid Fluid marker.""" + """Flow any still-deferred Confluid op markers IN PLACE at engine-route entry. + + The same lazy-flow convention the composing ops (``Pipeline`` / ``Enable`` / + ``RandomApply``) use — so a YAML ops doc may list bare ``!class:`` mapping-form + entries (e.g. a bare albumentations transform) directly under ``ops:``. The in-place + write is the cache: later routes (and the spawn pickler) see live ops. A marker that + cannot build raises ONE actionable error naming the offending index. + """ + from confluid import flow + for i, op in enumerate(ops): if isinstance(op, _ConfluidFluid): - raise TypeError(_fluid_op_guidance(op, i)) + try: + ops[i] = flow(op) + except Exception as exc: + raise TypeError(_fluid_op_guidance(op, i)) from exc @configurable class FilterOp: """Configurable filter operation. - The op form of :meth:`Flux.filter` — a predicate gate over the stream: the sample + The op form of :meth:`Flux.filter` — a predicate gate over the stream: the record passes when the predicate returns ``True`` and is dropped otherwise (``__call__`` - returns ``None``, which every engine route treats as "skip this sample"). + returns ``None``, which every engine route treats as "skip this record"). Args: - p: Predicate ``Sample -> bool``; the sample passes through when it returns ``True``, else is dropped. + p: Predicate ``record -> bool``; the record passes through when it returns ``True``, else is dropped. Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. """ - def __init__(self, p: Optional[Callable[[Sample], bool]] = None): + def __init__(self, p: Optional[Callable[[Record], bool]] = None): # Lazy / zero-arg: store config only; a missing predicate is validated lazily in __call__. self.p = p - def __call__(self, s: Sample) -> Optional[Sample]: + def __call__(self, record: Record) -> Optional[Record]: if self.p is None: - raise ValueError("FilterOp.p (predicate) is not set — provide a Sample->bool callable before use.") - return s if self.p(s) else None + raise ValueError("FilterOp.p (predicate) is not set — provide a record->bool callable before use.") + return record if self.p(record) else None @configurable class WrappedOp: """Configurable transformation wrapper with smart mapping. - The op form of :meth:`Flux.map` — lifts a plain function over one Sample field. The + The op form of :meth:`Flux.map` — lifts a plain function over one record value. The callable is ALWAYS stored as its importable ``module:function`` path (via :mod:`sampleflux.discovery`), so the op pickles across ``spawn`` workers and serializes into Confluid YAML verbatim; the live function resolves lazily on first @@ -113,18 +165,18 @@ class WrappedOp: Args: f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. - s: Which field to transform — ``"input"`` (default, the primary input field's payload), - ``"target"`` (the primary target field's payload), or ``"all"`` (the whole ``Sample`` bag). + key: The record key whose value payload the function transforms (item metadata preserved). + ``None`` (default) = the function receives the WHOLE record dict and returns the new record. kw: Extra keyword arguments forwarded to the wrapped callable on every call (defaults to none). """ - def __init__(self, f: Union[str, Callable] = "", s: str = "input", kw: Optional[Dict[str, Any]] = None): + def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: Optional[Dict[str, Any]] = None): from sampleflux.discovery import get_callable_path # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` # property). EXPLICIT: always store the string path for serialization. self.f = get_callable_path(f) if callable(f) else f - self.s = s + self.key = key self.kw = dict(kw) if kw else {} # Internal cache for the live callable self._func_cache: Optional[Callable] = None @@ -137,17 +189,22 @@ def func(self) -> Callable: self._func_cache = resolve_callable(self.f) return self._func_cache - def __call__(self, sample: Sample) -> Optional[Sample]: - if self.s == "all": - return cast(Sample, self.func(sample, **self.kw)) - role: Role = "input" if self.s == "input" else "target" - key, item = primary(sample, role) - new_data = self.func(item_data(item), **self.kw) - return sample.replace_field(key, with_data(item, new_data)) + def __call__(self, record: Record) -> Optional[Record]: + if self.key is None: + return cast(Optional[Record], self.func(record, **self.kw)) + if self.key not in record: + raise KeyError(f"WrappedOp: record has no key {self.key!r} (keys: {list(record)})") + value = record[self.key] + new_data = self.func(item_data(value), **self.kw) + try: + new_value = with_data(value, new_data) + except TypeError: + new_value = new_data # a plain (non-item) value is replaced verbatim + return {**record, self.key: new_value} class _Carried(NamedTuple): - """A :class:`Sample` travelling the streamed route together with its per-sample Context.""" + """A record travelling the streamed route together with its per-record Context.""" sample: Any ctx: Context @@ -243,7 +300,7 @@ def __init__(self, fluxes: Optional[List["Flux"]] = None) -> None: # Lazy / zero-arg: store config only; no sub-fluxes ⇒ an empty stream. self.fluxes = fluxes if fluxes is not None else [] - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: """Iterate through all sub-fluxes sequentially.""" for flux in self.fluxes: yield from flux @@ -254,19 +311,20 @@ def __len__(self) -> int: @configurable(category="engine") -class Flux(torch.utils.data.Dataset[Sample]): +class Flux(torch.utils.data.Dataset[Record]): """ The primary stream engine for SampleFlux. Wraps any iterable or indexed dataset and provides a functional API. - Every carrier is a typed :class:`~sampleflux.bag.sample.Sample` bag, passed through the op - chain verbatim (no coercion). ``source`` is duck-typed (any iterable; the Indexable - protocol if ``__getitem__``/``__len__`` are present) and ``ops`` is a list of bare - transforms ``Sample -> Optional[Sample]``. + Every carrier is a plain record ``dict`` of typed values, and every op is applied + through the op-FAMILY dispatch (:func:`_apply_op`) — so native sampleflux ops, + bare albumentations transforms, and bare torchvision ``transforms.v2`` transforms + all sit in ONE ``ops`` list as-is. ``source`` is duck-typed (any iterable; the + Indexable protocol if ``__getitem__``/``__len__`` are present). Args: - source: Any iterable or indexable dataset (duck-typed) yielding ``Sample`` bags; ``None`` = empty stream. - ops: Ordered transforms ``Sample -> Optional[Sample]`` applied lazily on access (``None`` = no ops). + source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. + ops: Ordered ops applied lazily on access — native ops and bare library transforms alike (``None`` = no ops). chunk_size: Parallel-processing chunk size; ``0`` (the default) processes sequentially. """ @@ -377,7 +435,7 @@ def __getitem__(self, index: int) -> Any: if result is None: raise IndexError(f"Sample {index} filtered out by {op}") sample = result - return cast(Sample, sample) + return cast(Record, sample) def to_sink(self, sink: Any) -> None: """Write the entire flux to a DataSink.""" @@ -400,13 +458,17 @@ def batch(self, chunk_size: int) -> "Flux": self._chunk_size = chunk_size return self - def map(self, func: Callable, select: str = "input", **kwargs: Any) -> "Flux": - """Append a transformation to the flux.""" - op = WrappedOp(func, select, kwargs) + def map(self, func: Callable, key: Optional[str] = None, **kwargs: Any) -> "Flux": + """Append a transformation to the flux. + + ``key`` names the record entry whose payload ``func`` transforms; ``None`` hands + ``func`` the whole record dict. + """ + op = WrappedOp(func, key, kwargs) self.ops.append(op) return self - def filter(self, predicate: Callable[[Sample], bool]) -> "Flux": + def filter(self, predicate: Callable[[Record], bool]) -> "Flux": """Filter the flux based on a predicate.""" self.ops.append(FilterOp(predicate)) return self @@ -435,7 +497,7 @@ def __iter__(self) -> Iterator[Any]: else: yield from it - def _iter_streamed(self) -> Iterator[Sample]: + def _iter_streamed(self) -> Iterator[Record]: """Mixed per-sample / stream-level op chain (a stream-level op exposes ``.stream``).""" source = self._guard_live_source() if source is None: @@ -462,7 +524,7 @@ def per_sample(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Option else: yield None if s is None else _Carried(s, c.ctx) - def strip(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[Sample]]: + def strip(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[Record]]: for c in stream: if c is None: yield None @@ -475,7 +537,7 @@ def strip(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[Sa ) yield c.sample - def wrap(stream: Iterator[Optional[Sample]]) -> Iterator[Optional[_Carried]]: + def wrap(stream: Iterator[Optional[Record]]) -> Iterator[Optional[_Carried]]: for s in stream: yield None if s is None else _Carried(s, Context()) @@ -490,7 +552,7 @@ def wrap(stream: Iterator[Optional[Sample]]) -> Iterator[Optional[_Carried]]: if c is not None: yield c.sample - def _iter_sequential(self) -> Iterator[Sample]: + def _iter_sequential(self) -> Iterator[Record]: """Standard single-threaded execution.""" source = self._guard_live_source() if source is None: @@ -499,7 +561,7 @@ def _iter_sequential(self) -> Iterator[Sample]: for item in source: yield from _worker_task_multi(item, self.ops) - def _iter_parallel(self) -> Iterator[Sample]: + def _iter_parallel(self) -> Iterator[Record]: """Multiprocess execution engine.""" source = self._guard_live_source() if source is None: @@ -517,20 +579,17 @@ def _iter_parallel(self) -> Iterator[Sample]: for future in futures: yield from future.result() - def collect(self) -> List[Sample]: + def collect(self) -> List[Record]: """Materialize the full flux into a list.""" return list(self) - def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: - """Yield pipeline-output Samples carrying only ``fields`` (the projection primitive). + def project(self, keys: Collection[str]) -> Iterator[Record]: + """Yield pipeline-output records carrying only ``keys`` (the projection primitive). Implements :class:`sampleflux.projection.SupportsProjection`. Flux must run its op - chain to produce each Sample (an op may consume the input), so this is the generic - "iterate, then keep only fields of the requested roles" form. ``fields`` is a subset - of ``{"input", "target", "metadata"}`` (mapped onto the ``input`` / ``target`` / - ``aux`` roles). Lazy: a generator. + chain to produce each record (an op may consume the input), so this is the generic + "iterate, then keep only the requested keys" form. Lazy: a generator. """ - want_roles = {_PROJECTION_ROLES[f] for f in fields} - for sample in self: - keep = [k for k in sample.keys() if sample.role_of(k) in want_roles] - yield Sample({k: sample[k] for k in keep}, {k: sample.role_of(k) for k in keep}) + want = set(keys) + for record in self: + yield {k: v for k, v in record.items() if k in want} diff --git a/sampleflux/bag/dispatch.py b/sampleflux/dispatch.py similarity index 79% rename from sampleflux/bag/dispatch.py rename to sampleflux/dispatch.py index e51b90b..6da114f 100644 --- a/sampleflux/bag/dispatch.py +++ b/sampleflux/dispatch.py @@ -1,20 +1,20 @@ -"""The kernel registry — type dispatch for transforms (the torchvision-v2 ``_KERNEL_REGISTRY`` pattern). +"""The kernel registry — type dispatch for ops (the torchvision-v2 ``_KERNEL_REGISTRY`` pattern). -A transform does not hard-code how to handle each item type. Instead a kernel is registered +An op does not hard-code how to handle each value type. Instead a kernel is registered per ``(transform class, item type)`` pair, and :func:`dispatch` looks one up — walking the -item's MRO so a kernel registered for a base item type also serves its subclasses. This is +value's MRO so a kernel registered for a base item type also serves its subclasses. This is the same registry idea as :mod:`sampleflux.collate` (batching keyed by representation), -applied to per-type transform behaviour. +applied to per-type op behaviour. -Registration is open: a downstream package teaches an existing transform about a new item +Registration is open: a downstream package teaches an existing op about a new value type with one decorator and NO core edit — from mypkg.transforms import Denoise # any Transform subclass from mypkg.items import IQSignal # any registered item type @Denoise.kernel(IQSignal) - def _(item, params): - return denoise_iq(item, strength=params["strength"]) + def _(value, params): + return denoise_iq(value, strength=params["strength"]) The transform base exposes ``.kernel(item_type)`` as a thin wrapper over :func:`register_kernel`; both are documented so either entry point works. @@ -24,7 +24,7 @@ def _(item, params): __all__ = ["Kernel", "register_kernel", "get_kernel", "dispatch", "registered_kernels"] -#: A kernel maps ``(item, params) -> item`` — the per-type behaviour of one transform. +#: A kernel maps ``(value, params) -> value`` — the per-type behaviour of one op. Kernel = Callable[[Any, Dict[str, Any]], Any] _KERNEL_REGISTRY: Dict[Tuple[type, type], Kernel] = {} @@ -57,8 +57,8 @@ def dispatch(transform_cls: type, item_cls: type) -> Optional[Kernel]: Resolution walks the transform's MRO (a subclass transform inherits its base's kernels unless it overrides them) and, for each, the item's MRO (a kernel on a base item type serves subclasses). The MOST specific transform wins; within a transform, the most - specific item type wins. ``None`` means "this transform does not handle this item" — - the caller passes the field through untouched. Results are memoized (see + specific item type wins. ``None`` means "this op does not handle this value" — + the caller passes the entry through untouched. Results are memoized (see :data:`_DISPATCH_CACHE`), invalidated on every :func:`register_kernel`. """ key = (transform_cls, item_cls) diff --git a/sampleflux/flow.py b/sampleflux/flow.py index fb8ce4d..cd71a19 100644 --- a/sampleflux/flow.py +++ b/sampleflux/flow.py @@ -23,11 +23,11 @@ - ``from:`` — the step supplying this step's input sample. Omitted = the previous step (the first step reads the source sample). Must name an EARLIER step: document order is the schedule, so forward references are errors and cycles are inexpressible. -- ``merge_from:`` — typed fan-in: UNION another step's fields into this step's incoming - sample before the op runs (the ``MergeFields`` slot semantics — last-write-wins on a +- ``merge_from:`` — fan-in: UNION another step's record entries into this step's incoming + record before the op runs (the ``MergeFields`` slot semantics — last-write-wins on a key collision, in listed order). -- ``bind:`` — ``{param: ref}`` per-sample parameters: ``ref`` is a step name (its result - sample's primary input, ``step[key]`` for a named field, or the raw value) or +- ``bind:`` — ``{param: ref}`` per-record parameters: ``ref`` is a step name (the step's + whole result record, ``step[key]`` for a named entry, or the raw value) or ``step.attr`` (the step op's live ``@output`` after it ran — lowered through ``Capture``). A step may be a plain mapping with no op (``out: {from: a, merge_from: [b]}``) — a pure @@ -48,7 +48,8 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.bag.sample import Sample, primary +from sampleflux.core import _apply_op +from sampleflux.items import Record from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Save, Use, _read_output logger = get_logger(__name__) @@ -74,7 +75,7 @@ class _BindRef(NamedTuple): step: str attr: Optional[str] # "step.attr" = the step op's @output attribute - key: Optional[str] # "step[key]" = the named FIELD of the step's Sample result + key: Optional[str] # "step[key]" = the named ENTRY of the step's record result def _split_bind_ref(ref: str) -> _BindRef: @@ -245,7 +246,7 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T @configurable(category="engine") -class FlowGraph(torch.utils.data.Dataset[Sample]): +class FlowGraph(torch.utils.data.Dataset[Record]): """Named-step graph engine — executes a ``flow:`` document natively. The readable twin of :class:`~sampleflux.core.Flux`: steps run in document order over @@ -255,7 +256,7 @@ class FlowGraph(torch.utils.data.Dataset[Sample]): between the two is a pinned contract. Args: - source: Any iterable or indexable dataset (duck-typed) yielding ``Sample`` bags; ``None`` = empty stream. + source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. outputs: Name of the step whose result is yielded. Blank (default) = the last step. chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single samples. @@ -353,23 +354,23 @@ def read_result(name: str, *, copy: bool) -> Any: else: sample = seed - # 2. typed fan-in: UNION the merge_from steps' fields (slot order, last wins) + # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) if step.merge_from: - if not isinstance(sample, Sample): + if not isinstance(sample, dict): raise TypeError( - f"flow step {step.name!r}: merge_from is the typed fan-in but the carrier is " - f"{type(sample).__name__} — expected a Sample." + f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " + f"{type(sample).__name__} — expected a record dict." ) - merged = [sample] + merged = dict(sample) for ref in step.merge_from: value = read_result(ref, copy=True) - if not isinstance(value, Sample): + if not isinstance(value, dict): raise TypeError( f"flow step {step.name!r}: merge_from step {ref!r} holds " - f"{type(value).__name__}, expected a Sample" + f"{type(value).__name__}, expected a record" ) - merged.append(value) - sample = Sample.merge(*merged) + merged.update(value) + sample = merged # 3. per-sample parameter binds if step.op is not None: @@ -392,11 +393,11 @@ def read_result(name: str, *, copy: bool) -> Any: ) else: value = read_result(parsed.step, copy=False) - if isinstance(value, Sample): - # "step[key]" = the named field; bare "step" = the primary input. - value = value[parsed.key] if parsed.key else primary(value)[1] + if isinstance(value, dict) and parsed.key: + # "step[key]" = the named entry; bare "step" = the whole record. + value = value[parsed.key] setattr(op, param, value) - result = op(sample) + result = _apply_op(sample, op) if result is None: return None sample = result @@ -404,14 +405,14 @@ def read_result(name: str, *, copy: bool) -> Any: env[step.name] = sample prev = step.name - return cast(Optional[Sample], env.get(outputs)) if outputs in env else None + return cast(Optional[Record], env.get(outputs)) if outputs in env else None def __iter__(self) -> Iterator[Any]: if self.source is None: return it = self._iter_samples() if self._chunk_size > 0: - batch: List[Sample] = [] + batch: List[Record] = [] for sample in it: batch.append(sample) if len(batch) == self._chunk_size: @@ -422,7 +423,7 @@ def __iter__(self) -> Iterator[Any]: else: yield from it - def _iter_samples(self) -> Iterator[Sample]: + def _iter_samples(self) -> Iterator[Record]: if self._workers > 1: yield from self._iter_parallel() return @@ -432,7 +433,7 @@ def _iter_samples(self) -> Iterator[Sample]: if result is not None: yield result - def _iter_parallel(self) -> Iterator[Sample]: + def _iter_parallel(self) -> Iterator[Record]: """Multiprocess execution — delegates to the serial engine over the LOWERED op list.""" from sampleflux.core import Flux diff --git a/sampleflux/bag/io.py b/sampleflux/io.py similarity index 67% rename from sampleflux/bag/io.py rename to sampleflux/io.py index 52c822b..8333ae6 100644 --- a/sampleflux/bag/io.py +++ b/sampleflux/io.py @@ -1,16 +1,20 @@ -"""The item codec registry — how a typed item serializes, for EVERY storage backend. +"""The item codec registry — how a typed value serializes, for EVERY storage backend. Storage backends never inspect item internals: they call :func:`encode_item` to get a flat :class:`EncodedItem` (registered type name + array payload + scalar attrs) and :func:`decode_item` to rebuild the item. The DEFAULT structural codec covers both item -shapes (an :class:`~sampleflux.bag.items.NDArrayItem` subclass → the array + its declared +shapes (an :class:`~sampleflux.items.NDArrayItem` subclass → the array + its declared attrs; a dataclass wrapper with a ``data`` field → the payload + the remaining fields), so an externally-registered item type — a domain package's signal item, a user type — serializes with ZERO storage-code changes. :func:`register_io` overrides the codec for types whose structure the default cannot capture (e.g. a payload-less wrapper with non-scalar fields). -The registered TYPE NAME (via :func:`~sampleflux.bag.items.register_item` / -:func:`~sampleflux.bag.items.get_item_type`) is the on-disk type tag — decoding requires the +A PLAIN (non-item) record value — a float, a string, a bare array — encodes under the +pseudo type tag ``"plain"`` and decodes back verbatim, so scalar metadata keys ride the +same layout as typed values. + +The registered TYPE NAME (via :func:`~sampleflux.items.register_item` / +:func:`~sampleflux.items.get_item_type`) is the on-disk type tag — decoding requires the item type to be registered (imported) in the reading process, exactly like the confluid ``!class:`` contract. """ @@ -20,8 +24,7 @@ from dataclasses import is_dataclass from typing import Any, Callable, Dict, Tuple, cast -from sampleflux.bag.items import NDArrayItem, get_item_type, item_data -from sampleflux.bag.sample import Role, Sample +from sampleflux.items import NDArrayItem, Record, get_item_type, is_item, item_data __all__ = [ "EncodedItem", @@ -29,14 +32,17 @@ "register_io", "encode_item", "decode_item", - "encode_sample", - "decode_sample", + "encode_record", + "decode_record", ] +#: The on-disk type tag for a plain (non-item) record value — stored and restored verbatim. +PLAIN_TYPE = "plain" + @dataclass(frozen=True) class EncodedItem: - """One item, flattened for storage: registered type name + payload + scalar attrs.""" + """One value, flattened for storage: registered type name (or ``"plain"``) + payload + scalar attrs.""" type_name: str payload: Any # ndarray / tensor / scalar / None @@ -45,10 +51,9 @@ class EncodedItem: @dataclass(frozen=True) class EncodedField: - """One named field of a sample: the encoded item plus its key and role.""" + """One named entry of a record: the encoded value plus its key.""" key: str - role: Role item: EncodedItem @@ -69,16 +74,23 @@ def register_io(item_cls: type, *, encode: Encoder, decode: Decoder) -> None: def encode_item(item: Any) -> EncodedItem: - """Flatten one item for storage (registered codec first, else the default structural codec).""" + """Flatten one value for storage (registered codec first, else the default structural codec). + + A value that is not a registered item type encodes as ``"plain"`` — payload verbatim. + """ codec = _CODECS.get(type(item)) if codec is not None: payload, attrs = codec[0](item) return EncodedItem(type_name=type(item).__name__, payload=payload, attrs=attrs) + if not is_item(item): + return EncodedItem(type_name=PLAIN_TYPE, payload=item, attrs={}) return EncodedItem(type_name=type(item).__name__, payload=_payload(item), attrs=_attrs(item)) def decode_item(encoded: EncodedItem) -> Any: - """Rebuild an item from its encoded form (the type must be registered in this process).""" + """Rebuild a value from its encoded form (an item type must be registered in this process).""" + if encoded.type_name == PLAIN_TYPE: + return encoded.payload cls = cast(Any, get_item_type(encoded.type_name)) codec = _CODECS.get(cls) if codec is not None: @@ -90,21 +102,14 @@ def decode_item(encoded: EncodedItem) -> Any: return cls(**encoded.attrs) -def encode_sample(sample: Sample) -> Tuple[EncodedField, ...]: - """Encode every field of a sample, in insertion order.""" - return tuple( - EncodedField(key=key, role=sample.role_of(key), item=encode_item(item)) for key, item in sample.items() - ) +def encode_record(record: Record) -> Tuple[EncodedField, ...]: + """Encode every entry of a record, in insertion order.""" + return tuple(EncodedField(key=key, item=encode_item(value)) for key, value in record.items()) -def decode_sample(fields: Tuple[EncodedField, ...]) -> Sample: - """Rebuild a :class:`Sample` from encoded fields (order preserved).""" - items: Dict[str, Any] = {} - roles: Dict[str, Role] = {} - for field in fields: - items[field.key] = decode_item(field.item) - roles[field.key] = field.role - return Sample(items, roles) +def decode_record(fields: Tuple[EncodedField, ...]) -> Record: + """Rebuild a record dict from encoded fields (order preserved).""" + return {field.key: decode_item(field.item) for field in fields} # --- the default structural codec ------------------------------------------- diff --git a/sampleflux/bag/items.py b/sampleflux/items.py similarity index 83% rename from sampleflux/bag/items.py rename to sampleflux/items.py index 057c55b..d01ec0a 100644 --- a/sampleflux/bag/items.py +++ b/sampleflux/items.py @@ -1,9 +1,10 @@ -"""Typed items — the leaves of the typed-bag model, each a value that OWNS its metadata. +"""Typed values — the vocabulary a record is made of, each value OWNING its metadata. -This is the answer to *"metadata belongs to input or target"*: instead of a shared flat -``Sample.metadata`` dict keyed by string, a sample is a bag of typed items and every piece -of metadata lives ON the item it describes — an :class:`Image` carries its ``layout``, a -:class:`Label` its ``classes``, a :class:`Regions` its ``canvas`` reference frame. +A sample is a plain ``dict`` (the :data:`Record` alias) whose values are TYPED: an +:class:`Image` carries its ``layout``, a :class:`Label` its ``classes``, a +:class:`Regions` its ``canvas`` reference frame. Ops dispatch on these types (the +torchvision-v2 ``tv_tensors`` idea) — there is no wrapper container and no role tags; +key names ("image", "mask", "label") carry meaning, exactly like every torch batch dict. The item model is HYBRID (the workspace decision): @@ -18,24 +19,23 @@ fragile). This module is MODALITY-NEUTRAL — only generic items live here (images, masks, boxes, -labels). Signal-domain items (a signal, a spectrogram) live in the domain package -(``waivefront.bag``) and register into the SAME registry, per the workspace modality-neutral -mandate. That IS the extensibility story below. +labels). Domain items (a signal, a spectrogram) live in the domain package and register +into the SAME registry, per the workspace modality-neutral mandate. That IS the +extensibility story below. Both shapes present a uniform payload accessor via :func:`item_data` / :func:`with_data`, so a transform kernel never has to special-case "is this a subclass or a wrapper". Extensibility: any type decorated with :func:`register_item` becomes a first-class item — -the dispatch registry (:mod:`sampleflux.bag.dispatch`) and the graph socket-type map can -see it. A downstream package (a signal item, a torchsig-shaped item, a SigMF recording, a -user type) adds one class + one decorator, no core edit. +the dispatch registry (:mod:`sampleflux.dispatch`) and a visual editor's socket-type map can +see it. A downstream package (a signal item, a user type) adds one class + one decorator, +no core edit. -NOTE (PoC scope): array items are ``np.ndarray`` subclasses only; a torch-``Tensor``-subclass +NOTE (scope): array items are ``np.ndarray`` subclasses only; a torch-``Tensor``-subclass item base (via ``__torch_function__``) is a documented follow-up — torch payloads ride in -wrapper items in the proof-of-concept. Items are registered in the local -:func:`register_item` registry rather than carried on the confluid ``@configurable`` -registry (an ``np.ndarray`` subclass builds through ``__new__``, which fights confluid's -``__init__`` validation wrap); confluid-native item discovery is a follow-up. +wrapper items. Items are registered in the local :func:`register_item` registry rather than +carried on the confluid ``@configurable`` registry (an ``np.ndarray`` subclass builds +through ``__new__``, which fights confluid's ``__init__`` validation wrap). """ from dataclasses import dataclass, field, fields, is_dataclass, replace @@ -45,7 +45,13 @@ _ItemT = TypeVar("_ItemT") +#: A sample record — a PLAIN dict of typed values. There is deliberately no container +#: class: ops receive and return ordinary dicts, so library transforms that already +#: understand dicts (torchvision v2) or named kwargs (albumentations) run as-is. +Record = Dict[str, Any] + __all__ = [ + "Record", "NDArrayItem", "Image", "Mask", @@ -62,7 +68,7 @@ # --------------------------------------------------------------------------- # Item registry — the extensibility surface. A registered type is a first-class -# item the dispatch registry and the (design-only) FluxStudio socket-type map see. +# item the dispatch registry and a visual editor's socket-type map see. # --------------------------------------------------------------------------- _ITEM_TYPES: Dict[str, type] = {} diff --git a/sampleflux/labels.py b/sampleflux/labels.py index 0021761..bcfd4ee 100644 --- a/sampleflux/labels.py +++ b/sampleflux/labels.py @@ -1,7 +1,7 @@ """``LabelMap`` — a bidirectional class-name ↔ integer-id map. -The *fittable* companion to the config-pinned :class:`~sampleflux.ops.target.EncodeTargetOp` / -:class:`~sampleflux.ops.target.DecodeTargetOp`. Those ops carry an explicit ``mapping`` that is +The *fittable* companion to the config-pinned :class:`~sampleflux.ops.target.EncodeTarget` / +:class:`~sampleflux.ops.target.DecodeTarget`. Those ops carry an explicit ``mapping`` that is **pinned in config, NOT fitted** at run time, so train / eval / predict share one identical label→id ordering. :class:`LabelMap` is the piece that *produces* such a pinned mapping: @@ -31,11 +31,11 @@ @configurable class LabelMap: - """Bidirectional class-name ↔ integer-id map (the fittable companion to ``EncodeTargetOp``). + """Bidirectional class-name ↔ integer-id map (the fittable companion to ``EncodeTarget``). Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`label_names`, builds the - :class:`~sampleflux.ops.target.EncodeTargetOp` / :class:`~sampleflux.ops.target.DecodeTargetOp` + :class:`~sampleflux.ops.target.EncodeTarget` / :class:`~sampleflux.ops.target.DecodeTarget` that apply it, and round-trips to disk in marainer's ``class_names.json`` format. Args: diff --git a/sampleflux/ops/__init__.py b/sampleflux/ops/__init__.py index 2c69b7a..d5873d2 100644 --- a/sampleflux/ops/__init__.py +++ b/sampleflux/ops/__init__.py @@ -1,5 +1,5 @@ """ -SampleFlux operations (typed-bag :class:`~sampleflux.Sample` transforms). +SampleFlux operations (record-dict ops). Submodules: - sampleflux.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / @@ -8,17 +8,19 @@ - sampleflux.ops.image: ConvertToImage (+ value_to_image / normalize_to_uint8 …) - sampleflux.ops.target: MetadataToTarget, EncodeTarget, DecodeTarget, CocoToTorchVisionDetection, MasksToDetectionBoxes - - sampleflux.ops.structure: SetRole, RenameField, DropField, CopyField, SelectFields + - sampleflux.ops.structure: RenameField, DropField, CopyField, SelectFields - sampleflux.ops.parallel: Parallel (worker-pool sub-pipeline) - sampleflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - sampleflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - - sampleflux.ops.configure: ConfigureOp (per-sample parameter injection) - - sampleflux.ops.formula: FormulaOp (math formula over the primary input) + - sampleflux.ops.configure: ConfigureOp (per-record parameter injection) + - sampleflux.ops.formula: FormulaOp (math formula over one record entry) - sampleflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) - - sampleflux.ops.transform_chain: TransformChain (sequential op-chain grouping) - - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-sample + - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-record Context graph plane — the flat-list building blocks a branchy flow: document lowers to) - - sampleflux.ops.debug: PrintSampleOp (per-sample summary probe) + - sampleflux.ops.debug: PrintSampleOp (per-record summary probe) + +The sequential composer ``Pipeline`` lives in :mod:`sampleflux.transform` (package-root +export) — one list mixing native ops with bare albumentations / torchvision-v2 transforms. """ from sampleflux.ops.configure import ConfigureOp @@ -31,7 +33,7 @@ from sampleflux.ops.parallel import Parallel from sampleflux.ops.random_apply import RandomApply from sampleflux.ops.sink import SampleSinkOp -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields from sampleflux.ops.target import ( CocoToTorchVisionDetection, DecodeTarget, @@ -40,7 +42,6 @@ MetadataToTarget, ) from sampleflux.ops.torch import ToTensor -from sampleflux.ops.transform_chain import TransformChain __all__ = [ "Apply", @@ -66,9 +67,7 @@ "Save", "SampleSinkOp", "SelectFields", - "SetRole", "Threshold", "ToTensor", - "TransformChain", "Use", ] diff --git a/sampleflux/ops/_augment_bridge.py b/sampleflux/ops/_augment_bridge.py deleted file mode 100644 index 6cc0a5e..0000000 --- a/sampleflux/ops/_augment_bridge.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Shared generator for the per-transform augmentation op families (``Alb*`` / ``Tv*``). - -Mirrors the waivefront-helios transform auto-bridge: walk a library's public transform -classes and generate ONE ``@configurable`` SampleFlux op per transform — a subclass of the -library's adapter op (:class:`~sampleflux.ops.albumentations.AlbumentationsOp` / -:class:`~sampleflux.ops.torchvision.TorchvisionTransformOp`) whose constructor mirrors the -transform's own parameters (plus the adapter's ``target`` / ``seed`` knobs). Each -generated op: - -* is a normal sample-scoped op (``__call__(sample)``) — it chains in a ``Flux`` ops list, - a ``TransformChain`` / ``RandomApply``, a Confluid YAML (``!class:AlbHorizontalFlip``), - or a visual canvas exactly like any hand-written op; -* exposes ``raw_transform`` — the configured library transform instance — so an adapter - op's ``transforms`` list unwraps a wired generated op back to the library object; -* carries a synthesized ``__signature__`` / ``__annotations__`` / ``Args:`` docstring so - static introspection (``to_pydantic`` → form-specs / MCP schemas, ``parse_param_docs`` - → widget tooltips) sees the transform's real parameters. - -The mandatory name prefix (``Alb`` / ``Tv``) keeps confluid's flat, name-keyed registry -collision-free — albumentations and torchvision share many bare names (``ColorJitter``, -``Normalize``, ``Resize``, …). -""" - -import inspect -# Callable/Dict/Literal/Sequence/Union are referenced only inside the wrapped transforms' -# STRING annotations, which get_type_hints() evaluates against THIS module's globals when -# to_pydantic introspects a synthesized __init__ — so they must be importable here (flake8 -# can't see string-annotation usage). -from typing import Any, Iterable, List, Optional, Tuple, Type, get_type_hints - -from confluid import configurable -from loggair import get_logger - -from sampleflux.ops.albumentations import TargetMode - -logger = get_logger(__name__) - -#: Adapter-owned constructor names — a library transform whose ctor collides is skipped. -_RESERVED = frozenset({"target", "seed", "transform", "transforms"}) - -_TARGET_DOC = " target: Joint-augmentation mode — ``none`` (input-only, default), ``mask``, or ``boxes``." -_SEED_DOC = " seed: Compose seed for deterministic draws. ``None`` = non-deterministic (default)." - - -def _param_specs(transform_cls: type) -> List[inspect.Parameter]: - """The transform constructor's named parameters (``self`` and variadics dropped).""" - sig = inspect.signature(transform_cls.__init__) # type: ignore[misc] - return [ - p - for n, p in sig.parameters.items() - if n != "self" and p.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) - ] - - -def _synth_doc(name: str, transform_cls: type, seed_param: bool) -> str: - """The generated op's docstring: the library docstring with the adapter params spliced in. - - ``parse_param_docs`` reads the (first) Google-style ``Args:`` block, so the adapter's - ``target`` / ``seed`` lines are inserted right after the library's own ``Args:`` - heading — or a fresh block is appended when the library docstring has none. - """ - extra = [_TARGET_DOC] + ([_SEED_DOC] if seed_param else []) - lib_doc = inspect.getdoc(transform_cls) or f"{name} transform (see the library documentation)." - lines = lib_doc.splitlines() - if any(ln.strip() == "Args:" for ln in lines): - out: List[str] = [] - for ln in lines: - out.append(ln) - if ln.strip() == "Args:": - out.extend(extra) - return "\n".join(out) - return lib_doc + "\n\nArgs:\n" + "\n".join(extra) - - -def _make_op( - name: str, - transform_cls: type, - *, - base: type, - prefix: str, - group: str, - module_name: str, - seed_param: bool, -) -> type: - """One generated op class wrapping ``transform_cls`` (see the module docstring).""" - specs = _param_specs(transform_cls) - pnames = [p.name for p in specs] - clash = _RESERVED & set(pnames) - if clash: - raise ValueError(f"constructor params clash with adapter params: {sorted(clash)}") - defaults = {p.name: (None if p.default is inspect.Parameter.empty else p.default) for p in specs} - required = {p.name for p in specs if p.default is inspect.Parameter.empty} - op_name = f"{prefix}{name}" - allowed = set(pnames) | {"target"} | ({"seed"} if seed_param else set()) - - def __init__(self: Any, **kwargs: Any) -> None: - # Lazy / zero-arg: store config only (required transform params default to None and - # surface lazily via the library's own missing-argument error on first call). - unknown = set(kwargs) - allowed - if unknown: - raise TypeError(f"{op_name}: unexpected parameters {sorted(unknown)}") - if seed_param: - base.__init__(self, target=kwargs.get("target", "none"), seed=kwargs.get("seed")) # type: ignore[misc] - else: - base.__init__(self, target=kwargs.get("target", "none")) # type: ignore[misc] - for pname in pnames: - setattr(self, pname, kwargs.get(pname, defaults[pname])) - self._params_key = None - - # Synthesized signature/annotations: static introspection (to_pydantic / FluxStudio - # widgets / parse_param_docs) sees the transform's real parameters, keyword-only, with - # the adapter's target/seed appended LAST. Required params are defaulted to None so - # zero-arg construction always works (the workspace lazy-init mandate). - # Resolve annotations to REAL objects against the transform's own module. A library's - # param annotations are often strings (PEP 563) referencing names local to that module - # (cv2, Literal, the library's own aliases); left as strings they'd blow up later when - # to_pydantic's get_type_hints evals them against THIS module. Anything that still won't - # resolve degrades to Any so introspection never chokes on a stray name. - try: - _hints = get_type_hints(transform_cls.__init__) # type: ignore[misc] - except Exception: - _hints = {} - - def _resolve(p: inspect.Parameter) -> Any: - ann = _hints.get(p.name, p.annotation) - return Any if isinstance(ann, str) else ann - - sig_params = [inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD)] - for p in specs: - sig_params.append( - p.replace(kind=inspect.Parameter.KEYWORD_ONLY, default=defaults[p.name], annotation=_resolve(p)) - ) - sig_params.append( - inspect.Parameter("target", inspect.Parameter.KEYWORD_ONLY, default="none", annotation=TargetMode) - ) - if seed_param: - sig_params.append( - inspect.Parameter("seed", inspect.Parameter.KEYWORD_ONLY, default=None, annotation=Optional[int]) - ) - __init__.__signature__ = inspect.Signature(sig_params) # type: ignore[attr-defined] - annotations = {p.name: _resolve(p) for p in specs if p.annotation is not inspect.Parameter.empty} - annotations["target"] = TargetMode - if seed_param: - annotations["seed"] = Optional[int] - __init__.__annotations__ = annotations - - def raw_transform(self: Any) -> Any: - kwargs = {} - for pname in pnames: - value = getattr(self, pname, None) - if value is None and pname in required: - continue # omitted → the library raises its own clear missing-argument error - kwargs[pname] = value - return transform_cls(**kwargs) - - base_pipeline = base.pipeline.fget # type: ignore[attr-defined] - - def pipeline(self: Any) -> Any: - # Rebuild the wrapped transform when any mirrored param changed (post-construction - # configuration), then reuse the adapter's compose/cache machinery verbatim. - key: tuple = tuple(repr(getattr(self, pname, None)) for pname in pnames) - key += (self.target, getattr(self, "seed", None)) - if key != getattr(self, "_params_key", None): - self._params_key = key - self.transform = self.raw_transform - return base_pipeline(self) - - namespace = { - "__init__": __init__, - # The base __call__ re-stated in the class dict: canvas op-classification checks - # vars(cls) for __call__, and inherited-only methods are invisible to it. - "__call__": base.__call__, - "__doc__": _synth_doc(name, transform_cls, seed_param), - "__module__": module_name, - "raw_transform": property( - raw_transform, doc="The configured library transform instance (built fresh per access)." - ), - "pipeline": property(pipeline, doc="The live library pipeline for the current parameter values."), - "LIBRARY_CLS": transform_cls, - } - cls = type(op_name, (base,), namespace) - return configurable(category="op", group=group, random=True)(cls) - - -def generate_transform_ops( - *, - classes: Iterable[Tuple[str, type]], - base: Type[Any], - prefix: str, - group: str, - module_globals: dict, - seed_param: bool, -) -> List[str]: - """Generate one op per ``(name, transform_cls)`` into ``module_globals``; returns the sorted names. - - Per-class failures (uninspectable constructor, adapter-param clash) skip that - transform with a DEBUG note and never break the module import — the helios-bridge - warn-and-continue contract. - """ - module_name = module_globals.get("__name__", base.__module__) - names: List[str] = [] - for name, transform_cls in classes: - try: - op_cls = _make_op( - name, - transform_cls, - base=base, - prefix=prefix, - group=group, - module_name=module_name, - seed_param=seed_param, - ) - except Exception as exc: - logger.debug(f"augment bridge: skipping {name}: {exc}") - continue - module_globals[op_cls.__name__] = op_cls - names.append(op_cls.__name__) - logger.debug(f"augment bridge: generated {len(names)} {prefix}* ops in group {group!r}") - return sorted(names) - - -__all__ = ["generate_transform_ops"] diff --git a/sampleflux/ops/albumentations.py b/sampleflux/ops/albumentations.py deleted file mode 100644 index 2971c5d..0000000 --- a/sampleflux/ops/albumentations.py +++ /dev/null @@ -1,182 +0,0 @@ -"""``AlbumentationsOp`` — run `albumentations `_ transforms as a SampleFlux op. - -One random draw is applied jointly to the primary input item and (per the ``target`` mode) -its segmentation mask / detection boxes, so a geometric augmentation moves image AND target -consistently; the aux fields pass through untouched. - -Transforms are authored **Confluid-natively** — nested ``!class:`` nodes, never -albumentations' own ``to_dict`` format:: - - - !class:sampleflux.ops.albumentations.AlbumentationsOp - target: mask - seed: 0 - transforms: - - !class:albumentations.HorizontalFlip - p: 0.5 - - !class:albumentations.Affine - translate_percent: 0.1 - -For per-transform graph nodes (one op per albumentations transform, e.g. -``AlbHorizontalFlip``) see :mod:`sampleflux.ops.albumentations_transforms`. - -Layout contract: albumentations operates on **numpy HWC** images (PIL inputs are converted -via ``np.asarray``) and the output stays numpy HWC — tensorize downstream with -:class:`~sampleflux.ops.torch.ToTensorOp`. Contrast with -:class:`~sampleflux.ops.torchvision.TorchvisionTransformOp`, which emits CHW torch tensors. -""" - -from typing import Any, List, Literal, Optional - -import numpy as np -from confluid import configurable -from loggair import get_logger - -from sampleflux.bag.items import Mask, item_data, with_data -from sampleflux.bag.sample import Sample, primary - -logger = get_logger(__name__) - -#: Which part of the ``Sample`` rides through the library jointly with the input. Closed -#: set so a typo fails at the call site and UIs / form-specs enumerate the choices. -TargetMode = Literal["none", "mask", "boxes"] - - -def _as_array(value: Any) -> np.ndarray: - """``value`` as a numpy array (PIL images and array-likes alike).""" - return np.asarray(value) - - -def _resolve_transform(entry: Any) -> Any: - """A raw library transform from a wired entry. - - Confluid ``!class:`` / ``!lazy:`` markers are flowed lazily (the RandomApply - paradigm), and a generated per-transform op (``AlbHorizontalFlip`` …) wired on a - visual canvas unwraps to its inner library transform via ``raw_transform``. - """ - from confluid import flow - from confluid.fluid import Fluid - - if isinstance(entry, Fluid): - entry = flow(entry) - return getattr(entry, "raw_transform", entry) - - -@configurable(category="op", group="augment", random=True) -class AlbumentationsOp: - """Apply albumentations transforms to the primary input item (and optionally the target). - - Pass EITHER ``transform`` (one transform, or a prebuilt ``A.Compose``) OR - ``transforms`` (a list composed into an ``A.Compose`` lazily) — never both. Entries - may be live albumentations objects, Confluid ``!class:`` markers, or generated - per-transform ops (:mod:`sampleflux.ops.albumentations_transforms`), which unwrap to - their inner library transform. - - Target modes (the ``target`` knob): - - * ``"none"`` — input-only augmentation (color jitter, noise, blur); the sample's - target passes through untouched. - * ``"mask"`` — the target-role item is a segmentation mask (2-D array or PIL ``L`` - image); image and mask receive the SAME spatial transform. - * ``"boxes"`` — the target-role item is the torchvision detection dict - ``{"boxes": [N,4] xyxy-pixel, "labels": [N]}`` (what - :class:`~sampleflux.ops.target.CocoToTorchVisionDetectionOp` / - :class:`~sampleflux.ops.target.MasksToDetectionBoxesOp` emit). When the op builds - the Compose itself the required ``bbox_params`` are added automatically - (``pascal_voc`` = absolute-pixel xyxy); a prebuilt Compose must carry its own. - - Stochasticity lives in the library: ``seed`` maps onto ``A.Compose(seed=...)``; gate - per sample via :class:`~sampleflux.ops.random_apply.RandomApply` (each albumentations - transform also carries its own ``p``). - - YAML: - - .. code-block:: yaml - - - !class:sampleflux.ops.albumentations.AlbumentationsOp - target: mask - seed: 0 - transforms: - - !class:albumentations.HorizontalFlip - p: 0.5 - - Args: - transform: ONE albumentations transform or a prebuilt ``A.Compose``. Validated lazily on first call. - transforms: List of albumentations transforms composed lazily into an ``A.Compose``. - target: Joint-augmentation mode — ``none`` (input-only, default), ``mask``, or ``boxes``. - seed: ``A.Compose`` seed for deterministic draws. ``None`` = non-deterministic (default). - """ - - def __init__( - self, - transform: Optional[object] = None, - transforms: Optional[List[Any]] = None, - target: TargetMode = "none", - seed: Optional[int] = None, - ) -> None: - # Lazy / zero-arg: store config only; transforms are resolved/validated on first call. - self.transform = transform - self.transforms: List[Any] = list(transforms) if transforms else [] - self.target = target - self.seed = seed - self._pipeline: Optional[object] = None - self._pipeline_key: Optional[tuple] = None - - def _entries(self) -> List[Any]: - """The configured raw transforms (markers flowed, generated ops unwrapped).""" - if self.transform is not None and self.transforms: - raise ValueError("AlbumentationsOp: pass either 'transform' or 'transforms', not both.") - entries = [self.transform] if self.transform is not None else list(self.transforms) - if not entries: - raise ValueError( - "AlbumentationsOp requires 'transform' (one transform / A.Compose) or " - "'transforms' (a list) to be set before calling." - ) - return [_resolve_transform(entry) for entry in entries] - - @property - def pipeline(self) -> Any: - """The live ``A.Compose`` — built lazily, cached until the configuration changes.""" - key = (id(self.transform), tuple(id(t) for t in self.transforms), self.target, self.seed) - if self._pipeline is None or self._pipeline_key != key: - # Albumentations is a hard dependency but slow to import — keep it lazy so - # module import (entry-point discovery) stays light (the target.py precedent). - import albumentations as A - from albumentations.core.composition import BaseCompose - - entries = self._entries() - if len(entries) == 1 and isinstance(entries[0], BaseCompose): - if self.seed is not None: - raise ValueError( - "AlbumentationsOp: 'seed' only applies when the op builds the Compose itself; " - "put the seed on your prebuilt A.Compose(..., seed=...) instead." - ) - self._pipeline = entries[0] - else: - bbox_params = ( - A.BboxParams(format="pascal_voc", label_fields=["labels"]) if self.target == "boxes" else None - ) - self._pipeline = A.Compose(entries, seed=self.seed, bbox_params=bbox_params) - self._pipeline_key = key - return self._pipeline - - def __call__(self, sample: Sample) -> Sample: - key, item = primary(sample, "input") - image = _as_array(item_data(item)) - if self.target == "mask": - mask_field = next(iter(sample.items_of_type(Mask)), None) - if mask_field is None: - raise ValueError("AlbumentationsOp(target='mask'): no Mask field in the sample to transform jointly.") - mkey, mitem = mask_field - out = self.pipeline(image=image, mask=_as_array(item_data(mitem))) - result = sample.replace_field(key, with_data(item, out["image"])) - return result.replace_field(mkey, with_data(mitem, out["mask"])) - if self.target == "boxes": - raise NotImplementedError( - "AlbumentationsOp(target='boxes') is not yet ported to the typed-bag Regions target " - "(migration follow-up); use target='none' or 'mask'." - ) - out = self.pipeline(image=image) - return sample.replace_field(key, with_data(item, out["image"])) - - -__all__ = ["AlbumentationsOp", "TargetMode"] diff --git a/sampleflux/ops/albumentations_transforms.py b/sampleflux/ops/albumentations_transforms.py deleted file mode 100644 index 54f0451..0000000 --- a/sampleflux/ops/albumentations_transforms.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Auto-generated ops: every public albumentations transform as its own SampleFlux op. - -Generated at import time by :mod:`sampleflux.ops._augment_bridge` from albumentations' -public namespace — one ``Alb`` op per concrete transform (``AlbHorizontalFlip``, -``AlbAffine``, ``AlbRandomBrightnessContrast``, …), each a subclass of -:class:`~sampleflux.ops.albumentations.AlbumentationsOp` mirroring the transform's own -constructor parameters plus the adapter's ``target`` / ``seed`` knobs. - -YAML (the registered short name resolves via the confluid registry): - -.. code-block:: yaml - - - !class:AlbHorizontalFlip - p: 0.5 - target: mask - -The ``Alb`` prefix is MANDATORY — albumentations and torchvision share many bare class -names (``ColorJitter``, ``Normalize``, ``Resize``, …) and confluid's registry is flat and -name-keyed, so unprefixed names would silently clobber each other. - -Composition transforms (``Compose`` / ``OneOf`` / ``SomeOf`` …) are deliberately NOT -generated — chaining ops is native SampleFlux (``ops:`` lists, ``TransformChain``, -``RandomApply``), and an ``A.Compose`` still wires verbatim into -``AlbumentationsOp(transform=...)``. -""" - -from typing import Any, List, Tuple - -from loggair import get_logger - -from sampleflux.ops._augment_bridge import generate_transform_ops -from sampleflux.ops.albumentations import AlbumentationsOp - -logger = get_logger(__name__) - - -def __getattr__(name: str) -> Any: - # Generated names live in module globals; this fallback only fires for genuinely - # missing ones — and tells mypy the dynamic attributes exist (module-__getattr__ rule). - raise AttributeError( - f"module {__name__!r} has no generated op {name!r} — " - "the albumentations transform may not exist in the installed version." - ) - - -def _library_classes() -> List[Tuple[str, type]]: - """The concrete, generatable albumentations transform classes, sorted by name.""" - import albumentations as A - from albumentations.core.composition import BaseCompose - from albumentations.core.transforms_interface import BasicTransform - - out: List[Tuple[str, type]] = [] - for name in sorted(vars(A)): - obj = getattr(A, name) - if not (isinstance(obj, type) and issubclass(obj, BasicTransform)): - continue - if issubclass(obj, BaseCompose) or obj.__name__ != name or name.startswith("_"): - continue - if obj.__module__.startswith("albumentations.core"): - continue # the abstract bases (BasicTransform / DualTransform / ImageOnlyTransform) - if name == "Lambda": - continue # callable-valued params — not representable as config - out.append((name, obj)) - return out - - -__all__ = generate_transform_ops( - classes=_library_classes(), - base=AlbumentationsOp, - prefix="Alb", - group="augment/albumentations", - module_globals=globals(), - seed_param=True, -) diff --git a/sampleflux/ops/configure.py b/sampleflux/ops/configure.py index af30942..dea06dc 100644 --- a/sampleflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -1,15 +1,13 @@ -"""``ConfigureOp`` — per-sample parameter injection (the helios ``Configure`` pattern). - -Some op parameters are only known per sample (a threshold derived from the sample's own -max, a crop length derived from its duration). ``ConfigureOp`` is the taidal port of the -legacy helios ``Configure`` transform (``Split`` → ``ToMetadata`` → ``Config``): a -``compute`` op-chain derives the value FROM the sample, the value is written to -``metadata[key]`` (traceability) and injected as a constructor attribute of the ``target`` -op (post-construction configuration — the confluid paradigm), then ``target`` is applied -to the ORIGINAL sample. - -Modality-neutral — it threads any ``Sample`` through any ops — so it lives in core -sampleflux (compose group, alongside ``TransformChain`` / ``Enable`` / ``RandomApply``). +"""``ConfigureOp`` — per-record parameter injection. + +Some op parameters are only known per record (a threshold derived from the record's own +max, a crop length derived from its duration). ``ConfigureOp`` runs a ``compute`` op-chain +that derives the value FROM the record, injects it as a constructor attribute of the +``target`` op (post-construction configuration — the confluid paradigm), then applies +``target`` to the ORIGINAL record. + +Modality-neutral — it threads any record through any ops — so it lives in core +sampleflux (compose group, alongside ``Pipeline`` / ``Enable`` / ``RandomApply``). """ from typing import Any, List, Optional, cast @@ -17,40 +15,39 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.bag.items import item_data -from sampleflux.bag.sample import Sample, primary +from sampleflux.items import Record, item_data @configurable(category="op", group="compose") class ConfigureOp: - """Compute a value from the sample and inject it as a parameter of a target op. + """Compute a value from the record and inject it as a parameter of a target op. - The ``ops`` chain runs on the incoming sample as a SIDE branch — its input/target - transformations are discarded (the original sample continues), while metadata writes - survive (the shared metadata-bus convention). The chain's final primary input item - becomes the VALUE: it is written to ``metadata[key]`` and set as the ``param`` - attribute of ``target``, then ``target`` is applied to the original sample. + The ``ops`` chain runs on the incoming record as a SIDE branch — its transformations + are discarded (the original record continues). The ``source``-keyed entry of the + chain's final record becomes the VALUE (payload-unwrapped via ``item_data``): it is + set as the ``param`` attribute of ``target``, then ``target`` is applied to the + original record. Confluid ``!class:`` / ``!lazy:`` markers in ``ops`` / ``target`` are flowed lazily at - first call (like ``TransformChain``), so a ``ConfigureOp()`` built from YAML costs nothing. + first call (like ``Pipeline``), so a ``ConfigureOp()`` built from YAML costs nothing. - YAML — a per-sample threshold (the helios ``Configure(TimeInSamples, CropToSize)`` - shape, here deriving ``ThresholdOp.low_level`` from the sample's own statistics): + YAML — a per-record threshold derived from the record's own statistics: .. code-block:: yaml - !class:sampleflux.ops.configure.ConfigureOp ops: - - !class:sampleflux.ops.numpy.MaxOp {} - target: !class:sampleflux.ops.numpy.ThresholdOp + - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "a.max() * 0.5"} + source: image + target: !class:sampleflux.ops.numpy.Threshold low_op: ">=" param: low_level Args: - ops: Value-computing op-chain; the chain's final primary input item is injected. Empty = the incoming input. + ops: Value-computing op-chain run on a side-branch copy of the record. Empty = the incoming record. target: The op to configure and apply; required at call time, validated lazily. param: Target attribute name to set with the computed value (e.g. ``low_level``). - key: Metadata key the value is also written to. Blank (default) = ``param``. + source: Record key of the side-branch result holding the computed value; required at call time. """ def __init__( @@ -58,26 +55,28 @@ def __init__( ops: Optional[List[Any]] = None, target: Optional[object] = None, param: str = "", - key: str = "", + source: str = "", ) -> None: - # Lazy / zero-arg: store config only; target/param are validated at first call. + # Lazy / zero-arg: store config only; target/param/source are validated at first call. self.ops = list(ops) if ops else [] self.target = target self.param = str(param) - self.key = str(key) + self.source = str(source) - def __call__(self, sample: Sample) -> Optional[Sample]: + def __call__(self, record: Record) -> Optional[Record]: if self.target is None: raise ValueError("ConfigureOp: a 'target' op is required") if not self.param: raise ValueError("ConfigureOp: 'param' (the target attribute to set) is required") + if not self.source: + raise ValueError("ConfigureOp: 'source' (the record key holding the computed value) is required") if isinstance(self.target, Fluid): self.target = flow(self.target) - # _apply_op = the engine's contract-aware chokepoint, so field-scoped ops - # (e.g. a pair-scoped op from the kinds grid) work in the compute chain and as target. + # _apply_op = the engine's op-family dispatch, so bare library transforms + # work in the compute chain and as target exactly as in a bare ops list. from sampleflux.core import _apply_op - current: Sample = sample + current: Record = record for i, op in enumerate(self.ops): if isinstance(op, Fluid): op = flow(op) @@ -86,13 +85,14 @@ def __call__(self, sample: Sample) -> Optional[Sample]: continue result = _apply_op(current, op) if result is None: - return None # the compute chain filtered the sample (FilterOp semantics) + return None # the compute chain filtered the record (FilterOp semantics) current = result - # The computed value is the primary input item's payload of the side-branch result. - value = item_data(primary(current)[1]) + if self.source not in current: + raise KeyError(f"ConfigureOp: side-branch result has no key {self.source!r} (keys: {list(current)})") + value = item_data(current[self.source]) target = cast(Any, self.target) setattr(target, self.param, value) - return _apply_op(sample, target) + return _apply_op(record, target) def close(self) -> None: """Propagate close() to inner ops that own resources.""" diff --git a/sampleflux/ops/context.py b/sampleflux/ops/context.py index 3c59d78..d3eebc3 100644 --- a/sampleflux/ops/context.py +++ b/sampleflux/ops/context.py @@ -1,17 +1,16 @@ -"""Context ops — move data between the per-sample :class:`~sampleflux.context.Context` and the stream. +"""Context ops — move data between the per-record :class:`~sampleflux.context.Context` and the stream. The six flat-list building blocks of graph-shaped pipelines: ``Save`` (fork snapshot), -``Use`` (branch start), ``Drop`` (cell hygiene), ``Apply`` (per-sample parameter from a -cell), ``Capture`` (an op's ``@output`` into a cell), and ``Mix`` (fan-in). A branchy -canvas graph or ``flow:`` document lowers to a plain sequential op list containing these -(``sampleflux.flow.to_ops``), executable by the ordinary ``Flux`` engine — and lifts back -(``from_ops``). - -Unlike the stash family these NEVER touch ``sample.metadata``: graph wiring lives on the -engine-created Context data plane, so the metadata bus stays byte-identical to a linear -run. Cells are stored by reference (ops are copy-on-write by convention); ``Use`` copies -on read unless it drops the cell — the same copy-on-read / move-on-drop idiom as -``Apply(source=cell)``. +``Use`` (branch start), ``Drop`` (cell hygiene), ``Apply`` (per-record parameter from a +cell), ``Capture`` (an op's ``@output`` into a cell), and ``MergeFields`` (fan-in). A +branchy canvas graph or ``flow:`` document lowers to a plain sequential op list containing +these (``sampleflux.flow.to_ops``), executable by the ordinary ``Flux`` engine — and lifts +back (``from_ops``). + +Graph wiring lives on the engine-created Context data plane, so the record itself stays +byte-identical to a linear run. Cells are stored by reference (ops are copy-on-write by +convention); ``Use`` copies on read unless it drops the cell — the same copy-on-read / +move-on-drop idiom as ``Apply(source=cell)``. """ from copy import deepcopy @@ -20,8 +19,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.bag.sample import Sample, primary from sampleflux.context import require +from sampleflux.items import Record _MISSING = object() @@ -47,48 +46,47 @@ def _read_output(op: Any, name: str) -> Any: return _MISSING -def _cell_field(value: Any, field: str = "input", key: str = "") -> Any: +def _cell_field(value: Any, key: str = "") -> Any: """A cell's contribution to a value slot. - A :class:`~sampleflux.bag.sample.Sample` cell contributes the ``key``-named item when - ``key`` is given, else its PRIMARY input-role item (:func:`primary` — the sanctioned - "the input" accessor); a raw cell value is used verbatim. + A record (dict) cell contributes its ``key``-named entry when ``key`` is given, else + the whole record verbatim; a raw cell value is used verbatim. """ - if isinstance(value, Sample): - return value[key] if key else primary(value)[1] + if isinstance(value, dict) and key: + return value[key] return value @configurable(category="op", group="structure") class Save: - """Snapshot the stream sample into a Context cell (pass-through). + """Snapshot the stream record into a Context cell (pass-through). - The sample continues down the linear stream unchanged AND becomes readable by later - ``Use`` / ``Apply`` / ``Mix`` steps — the fork point of a fan-out. Stored by + The record continues down the linear stream unchanged AND becomes readable by later + ``Use`` / ``Apply`` / ``MergeFields`` steps — the fork point of a fan-out. Stored by reference (readers copy); ops are copy-on-write by convention, so the snapshot stays - intact as the stream continues (insert ``CopySampleOp`` before an in-place op). + intact as the stream continues. Args: - name: Context cell to store the sample under; required at call time, validated lazily. + name: Context cell to store the record under; required at call time, validated lazily. """ def __init__(self, name: str = "") -> None: # Lazy / zero-arg: store config only; the cell name is validated at first call. self.name = str(name) - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.name: raise ValueError("Save: 'name' (the context cell to write) is required") - require("Save").put(self.name, sample) - return sample + require("Save").put(self.name, record) + return record @configurable(category="op", group="structure") class Use: - """Replace the stream sample with a Context cell's value (a branch start). + """Replace the stream record with a Context cell's value (a branch start). - The incoming sample is discarded; the cell's value becomes the stream sample - (a raw, non-``Sample`` cell value is used verbatim). Reads a DEEP COPY so two branches + The incoming record is discarded; the cell's value becomes the stream record + (a raw cell value is used verbatim). Reads a DEEP COPY so two branches reading one fork stay independent — unless ``drop`` frees the cell, which skips the copy (move semantics, the right choice for a cell's LAST reader). @@ -129,23 +127,24 @@ def __init__(self, names: Optional[List[str]] = None) -> None: # Lazy / zero-arg: store config only. self.names = list(names) if names else [] - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if self.names: ctx = require("Drop") for name in self.names: ctx.delete(name) - return sample + return record @configurable(category="op", group="structure") class Apply: """Set a wrapped op's parameter from a Context cell, then apply the op. - The declarative per-sample-parameter step (``ConfigureOp`` with the value coming from + The declarative per-record-parameter step (``ConfigureOp`` with the value coming from a cell instead of an inline compute chain): the cell holds a prior branch's result — - a Sample cell contributes its ``input``, a raw cell value (e.g. a ``Capture``\\ d - ``@output``) is used as-is. The value is ``setattr``'d as ``param`` on ``op`` - post-construction (the confluid paradigm), then ``op`` runs on the incoming sample. + a record cell contributes its ``key``-named entry (or the whole record when ``key`` is + blank), a raw cell value (e.g. a ``Capture``\\ d ``@output``) is used as-is. The value + is ``setattr``'d as ``param`` on ``op`` post-construction (the confluid paradigm), + then ``op`` runs on the incoming record. Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call (like ``ConfigureOp``), so an ``Apply()`` built from YAML costs nothing. @@ -154,7 +153,7 @@ class Apply: op: The op to configure and apply; required at call time, validated lazily. param: Attribute name on ``op`` to set with the cell value; required at call time. source: Context cell holding the value; required at call time, validated lazily. - key: For a Sample cell — the named field to contribute. Blank (default) = the primary input field. + key: For a record cell — the named entry to contribute. Blank (default) = the whole cell value. drop: When True, free the source cell after reading it. """ @@ -185,11 +184,11 @@ def __call__(self, sample: Any) -> Optional[Any]: value = ctx.get(self.source) if self.drop: ctx.delete(self.source) - value = _cell_field(value, "input", key=self.key) + value = _cell_field(value, key=self.key) op = cast(Any, self.op) setattr(op, self.param, value) - # _apply_op = the engine's contract-aware chokepoint, so a field-scoped wrapped op - # (e.g. a pair-scoped op from the kinds grid) applies exactly as in a bare ops list. + # _apply_op = the engine's op-family dispatch, so a bare library transform wired as + # the wrapped op applies exactly as in a bare ops list. from sampleflux.core import _apply_op return _apply_op(sample, op) @@ -240,7 +239,7 @@ def _items(self) -> Dict[str, str]: items.setdefault(self.output, self.name or self.output) return items - def __call__(self, sample: Sample) -> Optional[Sample]: + def __call__(self, record: Record) -> Optional[Record]: if self.op is None: raise ValueError("Capture: an 'op' to apply is required") items = self._items() @@ -249,18 +248,18 @@ def __call__(self, sample: Sample) -> Optional[Sample]: self.op = _flow_if_fluid(self.op) ctx = require("Capture") op = cast(Any, self.op) - # _apply_op = the engine's contract-aware chokepoint (field-scoped ops capture too). + # _apply_op = the engine's op-family dispatch (bare library transforms capture too). from sampleflux.core import _apply_op - result = _apply_op(sample, op) + result = _apply_op(record, op) if result is None: - return None # the wrapped op filtered the sample (FilterOp semantics) + return None # the wrapped op filtered the record (FilterOp semantics) for attr, cell in items.items(): value = _read_output(op, attr) if value is _MISSING: raise AttributeError(f"Capture: {type(op).__name__!r} has no @output attribute {attr!r} to capture") ctx.put(cell, value) - return cast(Optional[Sample], result) + return cast(Optional[Record], result) def close(self) -> None: """Propagate close() to the wrapped op if it owns resources.""" @@ -271,18 +270,17 @@ def close(self) -> None: @configurable(category="op", group="structure") class MergeFields: - """Typed fan-in: UNION the named cells' fields into the incoming :class:`Sample`. + """Fan-in: UNION the named cells' entries into the incoming record. - The typed replacement for :class:`Mix`'s metadata dict-merge: each source cell (a - ``Sample`` saved by an earlier branch) contributes its FIELDS and ROLES, united in - listed order with last-write-wins on a key collision (the deterministic slot-order rule; - avoid a deliberate collision by renaming on the producing branch — + Each source cell (a record saved by an earlier branch) contributes its ENTRIES, + united in listed order with last-write-wins on a key collision (the deterministic + slot-order rule; avoid a deliberate collision by renaming on the producing branch — ``sampleflux.ops.structure.RenameField``). ``keys`` selects a subset of a source's - fields before the union. + entries before the union. Args: - sources: Context cells (earlier branch results) to union into the incoming sample, in order. - keys: Restrict the union to these field keys across all sources. Empty (default) = every field. + sources: Context cells (earlier branch results) to union into the incoming record, in order. + keys: Restrict the union to these keys across all sources. Empty (default) = every entry. drop: Context cells to free after merging (defaults to none). """ @@ -297,24 +295,20 @@ def __init__( self.keys = list(keys) if keys else [] self.drop = list(drop) if drop else [] - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.sources: raise ValueError("MergeFields: 'sources' (the context cells to union) is required") - if not isinstance(sample, Sample): - raise TypeError( - f"MergeFields: the incoming carrier is {type(sample).__name__}, expected Sample — " - "typed fan-in unions named fields (legacy Sample fan-in is Mix)." - ) + if not isinstance(record, dict): + raise TypeError(f"MergeFields: the incoming carrier is {type(record).__name__}, expected a record dict.") ctx = require("MergeFields") - merged = sample + merged = dict(record) for cell_name in self.sources: value = ctx.get(cell_name) - if not isinstance(value, Sample): - raise TypeError(f"MergeFields: cell {cell_name!r} holds {type(value).__name__}, expected a Sample") + if not isinstance(value, dict): + raise TypeError(f"MergeFields: cell {cell_name!r} holds {type(value).__name__}, expected a record") if self.keys: - keep = [k for k in self.keys if k in value] - value = Sample({k: value[k] for k in keep}, {k: value.role_of(k) for k in keep}) - merged = Sample.merge(merged, value) + value = {k: value[k] for k in self.keys if k in value} + merged.update(value) for cell_name in self.drop: ctx.delete(cell_name) return merged diff --git a/sampleflux/ops/debug.py b/sampleflux/ops/debug.py index 287da4a..723b71d 100644 --- a/sampleflux/ops/debug.py +++ b/sampleflux/ops/debug.py @@ -1,12 +1,11 @@ -"""Sample inspection / debug ops.""" +"""Record inspection / debug ops.""" from typing import Any, Literal, Optional from confluid import configurable from loggair import get_logger -from sampleflux.bag.items import item_data -from sampleflux.bag.sample import Sample +from sampleflux.items import Record, is_item, item_data logger = get_logger(__name__) @@ -57,24 +56,24 @@ def _summarize_metadata(metadata: Any) -> str: @configurable(category="op", group="debug") class PrintSampleOp: - """Log / print a summary of each sample passing through (a pass-through ``Sample -> Sample`` op). + """Log / print a summary of each record passing through (a pass-through op). - A pipeline probe: emits a compact description of the sample — ``input`` / ``target`` shape+dtype - plus a length-capped value preview (large arrays elided), and the ``metadata`` (values - summarised the same way) — to the Loggair logger (the LOG file + console) and, by default, to stdout via - ``print`` (so it shows in a terminal / a GUI node output panel regardless of log - level). The sample is returned UNCHANGED. + A pipeline probe: emits a compact description of the record — each typed value's + shape+dtype plus a length-capped value preview (large arrays elided), and the plain + entries (scalars etc., summarised the same way) — to the Loggair logger (the LOG file + + console) and, by default, to stdout via ``print`` (so it shows in a terminal / a GUI + node output panel regardless of log level). The record is returned UNCHANGED. Args: label: A prefix identifying this probe in the output (e.g. "after-impairments"). - level: Loggair level for the logged line — "trace" or "debug" (per-sample output is + level: Loggair level for the logged line — "trace" or "debug" (per-record output is diagnostic, so info/warning are deliberately not offered; use ``to_console`` to see it). - include_data: Include an ``input`` / ``target`` shape+dtype + value preview. - include_metadata: Include the sample's metadata (values summarised). + include_data: Include the typed item values (shape+dtype + value preview). + include_metadata: Include the plain (non-item) entries — scalars, strings, side values. to_console: Also ``print`` the line to stdout — guaranteed console / node-panel visibility, independent of the log level. Set False to log only. - limit: Stop emitting after this many samples (None = every sample) — avoids flooding on a - large dataset; the op still passes EVERY sample through unchanged. + limit: Stop emitting after this many records (None = every record) — avoids flooding on a + large dataset; the op still passes EVERY record through unchanged. """ def __init__( @@ -94,22 +93,23 @@ def __init__( self.limit = limit self._count = 0 # runtime probe counter — NOT config (per-instance, per-process) - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if self.limit is None or self._count < self.limit: - message = self._format(sample) + message = self._format(record) getattr(logger, self.level)(message) # level is a closed Literal, so this method exists if self.to_console: print(message) self._count += 1 - return sample + return record - def _format(self, sample: Sample) -> str: + def _format(self, record: Record) -> str: parts = [f"[{self.label} #{self._count}]"] - for key in sample.keys(): - role = sample.role_of(key) - if role == "aux" and not self.include_metadata: + for key, value in record.items(): + typed = is_item(value) + if typed and not self.include_data: continue - if role != "aux" and not self.include_data: + if not typed and not self.include_metadata: continue - parts.append(f"{key}[{role}]={_summarize(item_data(sample[key]))}") + tag = type(value).__name__ if typed else "plain" + parts.append(f"{key}[{tag}]={_summarize(item_data(value))}") return " ".join(parts) diff --git a/sampleflux/ops/enable.py b/sampleflux/ops/enable.py index 8ea5eb3..70761e9 100644 --- a/sampleflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -1,8 +1,8 @@ """``Enable`` — toggle one or more ops on/off via a single named CLI flag. -A compose-group op (alongside ``TransformChain`` / ``Parallel``): wrap an inner op-list +A compose-group op (alongside ``Pipeline`` / ``Parallel``): wrap an inner op-list so the whole chain can be switched on or off from one boolean attribute whose -name becomes the CLI flag. Modality-neutral — it threads any ``Sample`` +name becomes the CLI flag. Modality-neutral — it threads any record through any ops — so it lives in core sampleflux, not a domain package. """ @@ -11,7 +11,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag.sample import Sample +from sampleflux.items import Record logger = get_logger(__name__) @@ -21,10 +21,9 @@ class Enable: """Wrap one or more ops so they can be toggled on/off via a single named CLI flag. ``ops`` is a list; even a single-op guard uses ``ops: [op]``. The wrapper - threads each sample through every op in sequence — same semantics as - listing them inline in ``Flux.ops`` — so visualization chains like - ``ConvertToImageOp`` → ``SaveImageOp`` share one toggle instead - of needing a wrapper per op. + threads each record through every op in sequence — same semantics as + listing them inline in ``Flux.ops`` — so a whole visualization chain + shares one toggle instead of needing a wrapper per op. The toggle flag is supplied in YAML as an *extra* kwarg whose name becomes the CLI hook — Confluid's post-construction setattr promotes it to an @@ -39,8 +38,8 @@ class Enable: - !class:sampleflux.ops.enable.Enable visualize: false # ← any boolean attribute name works; this name IS the CLI flag ops: - - !class:sampleflux.ops.image.ConvertToImageOp {} - - !class:waivefront.visualizers.SaveImageOp + - !class:sampleflux.ops.image.ConvertToImage {} + - !class:waivefront.visualizers.SaveImage output_dir: ./segments_png CLI: @@ -96,7 +95,7 @@ class Enable: present. Args: - ops: Non-empty list of callables ``Sample -> Sample`` gated by the toggle. + ops: Non-empty list of ops (native or bare library transforms) gated by the toggle. """ def __init__(self, ops: Optional[List] = None) -> None: @@ -126,19 +125,19 @@ def flag_name(self) -> str: name, _ = self._toggle() return name - def __call__(self, sample: Sample) -> Optional[Sample]: + def __call__(self, record: Record) -> Optional[Record]: if not self.ops: raise ValueError("Enable requires a non-empty 'ops' list.") if not self.enabled: - return sample + return record from confluid import flow from confluid.fluid import Fluid - # _apply_op = the engine's contract-aware chokepoint, so field-scoped ops - # (e.g. a pair-scoped op from the kinds grid) run under the toggle unchanged. + # _apply_op = the engine's op-family dispatch, so bare library transforms + # run under the toggle exactly as in a bare ops list. from sampleflux.core import _apply_op - current: Optional[Sample] = sample + current: Optional[Record] = record for i, op in enumerate(self.ops): if current is None: return None diff --git a/sampleflux/ops/formula.py b/sampleflux/ops/formula.py index cd06573..0f39bcb 100644 --- a/sampleflux/ops/formula.py +++ b/sampleflux/ops/formula.py @@ -1,12 +1,12 @@ -"""``FormulaOp`` — evaluate a math formula over the primary input item. +"""``FormulaOp`` — evaluate a math formula over one record entry. The op-form of a visual canvas *Math* node: a restricted Python expression over one -named variable bound to the incoming primary input item (plus the stdlib ``math`` namespace -and the scalar helpers ``abs``/``min``/``max``/``round``/``pow`` — no builtins, so -``__import__``/``open``/``exec`` are unavailable). Its main consumer is the ops-export's +named variable bound to the ``field``-keyed record value (plus the stdlib ``math`` +namespace and the scalar helpers ``abs``/``min``/``max``/``round``/``pow`` — no builtins, +so ``__import__``/``open``/``exec`` are unavailable). Its main consumer is an ops-export's value-chain compilation: an on-canvas ``… → Extract → Math → widget`` wire becomes -``ConfigureOp(ops=[…, FormulaOp(formula)], target=…, param=…)``, so the per-sample value -survives serialization. +``ConfigureOp(ops=[…, FormulaOp(field, formula)], target=…, param=…)``, so the per-record +value survives serialization. """ import math as _math @@ -14,8 +14,7 @@ from confluid import configurable -from sampleflux.bag.items import item_data, with_data -from sampleflux.bag.sample import Sample, primary +from sampleflux.items import Record, item_data, with_data # Every public ``math`` symbol + the scalar built-in helpers, mirroring the canvas Math # node's namespace. The bound variable shadows same-named constants (e.g. ``e``). @@ -25,25 +24,35 @@ @configurable(category="op", group="compose") class FormulaOp: - """Replace the primary input item with ``formula`` evaluated over it. + """Replace the ``field``-keyed record value with ``formula`` evaluated over it. Args: formula: Expression over ``var`` (e.g. ``"a * 0.2"``); ``math.*`` + ``abs``/``min``/``max``/``round`` allowed. - var: Variable name the incoming primary input item binds to. Defaults to ``a``. + field: Record key whose value the formula reads and replaces; required at call time. + var: Variable name the incoming value binds to. Defaults to ``a``. """ - def __init__(self, formula: str = "a", var: str = "a") -> None: - # Lazy / zero-arg: store config only; the formula is validated at first call. + def __init__(self, formula: str = "a", field: str = "", var: str = "a") -> None: + # Lazy / zero-arg: store config only; formula and field are validated at first call. self.formula = str(formula) + self.field = str(field) self.var = str(var) - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.formula.strip(): raise ValueError("FormulaOp: 'formula' must be a non-empty expression") - key, item = primary(sample, "input") + if not self.field: + raise ValueError("FormulaOp: 'field' (the record key to evaluate over) is required") + if self.field not in record: + raise KeyError(f"FormulaOp: record has no key {self.field!r} (keys: {list(record)})") + item = record[self.field] namespace = {**_FORMULA_NAMESPACE, self.var: item_data(item)} try: value = eval(self.formula, {"__builtins__": {}}, namespace) # noqa: S307 - restricted namespace except Exception as exc: raise ValueError(f"FormulaOp: formula {self.formula!r} failed: {exc}") from exc - return sample.replace_field(key, with_data(item, value)) + try: + new_value = with_data(item, value) + except TypeError: + new_value = value # a plain (non-item) value is replaced verbatim + return {**record, self.field: new_value} diff --git a/sampleflux/ops/image.py b/sampleflux/ops/image.py index b96ea45..eb81297 100644 --- a/sampleflux/ops/image.py +++ b/sampleflux/ops/image.py @@ -1,7 +1,7 @@ """Generic, modality-agnostic image conversion for SampleFlux pipelines. This is the single home for "turn an arbitrary value into an image": the -:class:`ConvertToImageOp` op plus the library functions +:class:`ConvertToImage` op plus the library functions (:func:`value_to_image` / :func:`sample_to_image`) that back it and the GUI sample preview. It lives in sampleflux (not waivefront) because the conversion is fully generic — a 2-D map, a CHW tensor, a PIL image, a boolean mask all render @@ -26,10 +26,9 @@ from loggair import get_logger from PIL import Image, ImageDraw -from sampleflux.bag.items import Image as ImageItem -from sampleflux.bag.items import NDArrayItem, item_data -from sampleflux.bag.sample import Sample, primary -from sampleflux.bag.transform import Transform +from sampleflux.items import Image as ImageItem +from sampleflux.items import NDArrayItem, Record, item_data +from sampleflux.transform import Transform logger = get_logger("sampleflux.ops.image") @@ -58,7 +57,7 @@ def normalize_to_uint8( # Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob -# across the workspace (``value_to_image`` / ``sample_to_image`` / ``ConvertToImageOp`` and, via +# across the workspace (``value_to_image`` / ``sample_to_image`` / ``ConvertToImage`` and, via # re-export, waivefront's renderers) AND for GUI colormap dropdowns (which read ``COLORMAPS``). # A closed ``Literal`` (never a bare ``str``) makes the choice self-documenting and machine- # introspectable: visual-editor palettes, navigaitor's form-spec, and MCP tool schemas enumerate the @@ -115,7 +114,7 @@ def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: """Render an arbitrary value to an ``(H, W, 3)`` uint8 RGB image WITHOUT resizing. The core of :func:`value_to_image` factored out so callers that need their - own resize policy (e.g. :class:`ConvertToImageOp`'s exact ``width``/``height``) + own resize policy (e.g. :class:`ConvertToImage`'s exact ``width``/``height``) don't pay a double resize. Handles PIL images, torch tensors, numpy arrays (2-D maps → ``colormap``; 3-D → image with channel coercion; bool → 0/255); anything else falls back to a text rendering of its ``repr``. @@ -169,7 +168,7 @@ def _bound_longest_side(rgb: np.ndarray, max_size: int) -> np.ndarray: def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: - """Render an arbitrary value (a Sample's ``input`` OR ``target``) to an ``(H, W, 3)`` uint8 RGB image. + """Render an arbitrary value (any record entry) to an ``(H, W, 3)`` uint8 RGB image. A generic, modality-agnostic preview usable from any SampleFlux pipeline (and by a GUI sample extractor, which renders the selected field). Handles: @@ -194,19 +193,23 @@ def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 5 return _bound_longest_side(_render_rgb(value, colormap), max_size) -def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: - """Render a sample's primary input to an ``(H, W, 3)`` uint8 RGB image for display. +def sample_to_image(record: Record, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: + """Render a record's first array-bearing value to an ``(H, W, 3)`` uint8 RGB image for display. Thin wrapper over :func:`value_to_image` (which does the modality-agnostic rendering) - applied to the payload of the sample's primary ``input``-role field. Use - :func:`value_to_image` directly to render an arbitrary field payload. + applied to the payload of the record's first array-bearing (2-D / 3-D) value. Use + :func:`value_to_image` directly to render an arbitrary value. Args: - sample: The Sample to preview; its primary ``input`` field is rendered. + record: The record to preview; its first array-bearing (2-D / 3-D) value is rendered. colormap: Colormap applied to 2-D maps — one of the supported names (see ``Colormap``; ``"gray"`` = greyscale). max_size: Maximum length in pixels of the longest image side; larger renders are downscaled. """ - return value_to_image(item_data(primary(sample, "input")[1]), colormap=colormap, max_size=max_size) + for value in record.values(): + arr = _coerce_to_ndarray(item_data(value)) + if arr is not None and arr.ndim in (2, 3): + return value_to_image(item_data(value), colormap=colormap, max_size=max_size) + raise ValueError(f"sample_to_image: no array-bearing value in record (keys: {list(record)})") # --------------------------------------------------------------------------- # @@ -216,7 +219,7 @@ def sample_to_image(sample: Sample, colormap: Colormap = "viridis", max_size: in # from any pipeline / notebook): a generic, modality-agnostic way to look at the # RAW numeric values of an array/tensor — pick a channel, render it, and bin its # values. Pure functions (NOT @configurable ops): they measure/derive, they don't -# transform a Sample, so they're library helpers like value_to_image — not canvas +# transform a record, so they're library helpers like value_to_image — not canvas # nodes. They live here (not in the GUI node) so the computation is reusable # and unit-tested, per the workspace "rendering/analysis lives in sampleflux" mandate. # --------------------------------------------------------------------------- # @@ -596,12 +599,10 @@ def draw_text( @configurable(category="op", group="image") class ConvertToImage(Transform): - """Typed twin of :class:`ConvertToImageOp` — an array-bearing field → an ``Image`` item. + """An array-bearing field → an ``Image`` item. - The typed-bag counterpart of :class:`ConvertToImageOp`: instead of rendering - the primary input item into a PIL image in place, it reads an array-bearing field from a - :class:`~sampleflux.Sample` and writes a fresh :class:`~sampleflux.Image` item - (HWC ``uint8`` RGB) under ``output``, tagged with the ``input`` role (it is the + Reads an array-bearing field from the record and writes a fresh + :class:`~sampleflux.Image` item (HWC ``uint8`` RGB) under ``output`` (it is the pipeline's working image). Any other field passes through untouched. Rendering is byte-identical to the legacy op — it reuses the SAME @@ -615,7 +616,7 @@ class ConvertToImage(Transform): Unlike the legacy op it does NOT publish ``image_width_px`` / ``image_height_px`` — the ``Image`` item's array SHAPE carries the pixel dimensions, so a downstream consumer (e.g. a back-projection) reads them straight off the payload; there is no shared - metadata dict to publish into in the typed model. + metadata dict to publish into in the record model. Args: colormap: Colormap applied to 2-D maps — a supported ``Colormap`` name (``"gray"`` = greyscale). @@ -623,8 +624,8 @@ class ConvertToImage(Transform): height: Exact output height in pixels; resize to ``(width, height)`` when both width and height are > 0. max_size: When ``width``/``height`` aren't both set, bound the longest side to this many pixels (aspect kept). flip_vertical: Mirror the image top-to-bottom (e.g. spectrogram row 0 = f_min → display f_max at the top). - field: Name of the source field to render; blank (default) picks the first array-bearing item in the bag. - output: Name of the field the ``Image`` item is written to (added if new); its role is set to ``input``. + field: Name of the source field to render; blank (default) picks the first array-bearing item in the record. + output: Name of the key the ``Image`` item is written to (added if new). """ handles = (NDArrayItem,) @@ -650,20 +651,20 @@ def __init__( self.field = field self.output = output - def _find_source(self, sample: Sample) -> Any: + def _find_source(self, record: Record) -> Any: """Resolve the payload to render (``self.field`` or the first array-bearing item).""" if self.field: - if self.field not in sample.keys(): - raise ValueError(f"ConvertToImage: field {self.field!r} not in sample (fields: {list(sample.keys())})") - return item_data(sample[self.field]) - for _key, item in sample.items(): + if self.field not in record: + raise ValueError(f"ConvertToImage: field {self.field!r} not in record (keys: {list(record)})") + return item_data(record[self.field]) + for _key, item in record.items(): arr = _coerce_to_ndarray(item_data(item)) if arr is not None and arr.ndim in (2, 3): return item_data(item) - raise ValueError(f"ConvertToImage: no array-bearing field in sample (fields: {list(sample.keys())})") + raise ValueError(f"ConvertToImage: no array-bearing field in record (keys: {list(record)})") - def __call__(self, sample: Sample) -> Sample: - rgb = _render_rgb(self._find_source(sample), self.colormap) + def __call__(self, record: Record) -> Record: + rgb = _render_rgb(self._find_source(record), self.colormap) if self.flip_vertical: rgb = rgb[::-1, :, :] if self.width > 0 and self.height > 0: @@ -672,8 +673,7 @@ def __call__(self, sample: Sample) -> Sample: ) else: out_arr = _bound_longest_side(rgb, self.max_size) - out = sample.replace_field(self.output, ImageItem(out_arr, layout="HWC")) - return out.set_role(self.output, "input") + return {**record, self.output: ImageItem(out_arr, layout="HWC")} __all__ = [ diff --git a/sampleflux/ops/numpy.py b/sampleflux/ops/numpy.py index 91dddb1..fc5c51d 100644 --- a/sampleflux/ops/numpy.py +++ b/sampleflux/ops/numpy.py @@ -7,9 +7,8 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag.items import Mask, NDArrayItem, Regions, item_data -from sampleflux.bag.sample import Sample -from sampleflux.bag.transform import Transform +from sampleflux.items import Mask, NDArrayItem, Record, Regions, item_data +from sampleflux.transform import Transform logger = get_logger(__name__) @@ -21,8 +20,8 @@ def resolve_expression(value: str, meta: Optional[Dict[str, Any]] = None) -> str """Substitute ``{key}`` from ``meta`` and ``$NAME`` from ``os.environ``. Returns the substituted string verbatim — the caller is responsible for any further - casting (e.g. ``float(...)`` for a numeric expression). In the typed-bag model an item - owns its own metadata (there is no shared sample dict), so ``meta`` is usually empty and + casting (e.g. ``float(...)`` for a numeric expression). In the record model an item + owns its own metadata (there is no shared metadata dict), so ``meta`` is usually empty and only literals / ``$ENV`` expressions resolve; a ``{key}`` bound then raises ``KeyError``. Args: @@ -125,15 +124,15 @@ def threshold_array( class Threshold(Transform): """An array-bearing field → a boolean ``Mask`` item. - Reads the array at ``field`` (blank = the first array-bearing item in the bag) and thresholds + Reads the array at ``field`` (blank = the first array-bearing item in the record) and thresholds it into a boolean mask with the bound / comparison / expression math (:func:`threshold_array`), - writing a :class:`~sampleflux.Mask` item under ``output`` tagged ``aux`` (a threshold mask is an - intermediate that a later op — e.g. :class:`ConnectedComponents` — consumes). Any other field + writing a :class:`~sampleflux.Mask` item under ``output`` (a threshold mask is an + intermediate that a later op — e.g. :class:`ConnectedComponents` — consumes). Any other key passes through untouched. Each bound is a numeric literal or a ``resolve_expression`` string — ``5.5`` / ``"5.5"`` (literal) or ``"$REF_SNR"`` (environment variable). NOTE: ``{meta_key}`` expressions have no - typed metadata source in the bag model, so only literals and ``$ENV`` resolve here. + metadata source in the record model, so only literals and ``$ENV`` resolve here. Args: low_level: Lower bound (numeric literal or ``$ENV`` expression) compared with ``low_op`` when set; @@ -143,7 +142,7 @@ class Threshold(Transform): low_op: Lower-bound comparison — ``">"`` (strict, default) or ``">="`` (inclusive). high_op: Upper-bound comparison — ``"<"`` (strict, default) or ``"<="`` (inclusive). field: Name of the array field to threshold; blank (default) picks the first array-bearing item. - output: Name of the field the boolean ``Mask`` item is written to (added if new; role ``aux``). + output: Name of the key the boolean ``Mask`` item is written to (added if new). """ handles = (NDArrayItem,) @@ -167,26 +166,25 @@ def __init__( self.field = field self.output = output - def _find_array(self, sample: Sample) -> np.ndarray: + def _find_array(self, record: Record) -> np.ndarray: """Resolve the array to threshold (``self.field`` or the first array-bearing item).""" if self.field: - if self.field not in sample.keys(): - raise ValueError(f"Threshold: field {self.field!r} not in sample (fields: {list(sample.keys())})") - data = item_data(sample[self.field]) + if self.field not in record: + raise ValueError(f"Threshold: field {self.field!r} not in record (keys: {list(record)})") + data = item_data(record[self.field]) if not isinstance(data, np.ndarray): raise TypeError(f"Threshold: field {self.field!r} payload is {type(data).__name__}, expected an array") return data - for _key, item in sample.items(): + for _key, item in record.items(): data = item_data(item) if isinstance(data, np.ndarray): return data - raise ValueError(f"Threshold: no array-bearing field in sample (fields: {list(sample.keys())})") + raise ValueError(f"Threshold: no array-bearing field in record (keys: {list(record)})") - def __call__(self, sample: Sample) -> Sample: - arr = self._find_array(sample) + def __call__(self, record: Record) -> Record: + arr = self._find_array(record) mask = threshold_array(arr, self.low_level, self.high_level, self.low_op, self.high_op) - out = sample.replace_field(self.output, Mask(mask)) - return out.set_role(self.output, "aux") + return {**record, self.output: Mask(mask)} def connected_component_bboxes( @@ -237,11 +235,11 @@ def connected_component_bboxes( class ConnectedComponents(Transform): """A boolean ``Mask`` → a ``Regions`` item. - Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, else the + Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the first array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into ``(row_min, row_max, col_min, col_max)`` inclusive bin-box tuples via :func:`connected_component_bboxes`, writing them as a :class:`~sampleflux.Regions` item under - ``output`` tagged ``aux`` (RAW detections, not model predictions). Any other field passes through. + ``output`` (RAW detections, not model predictions). Any other key passes through. Components smaller than ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or 8-neighborhood. Requires ``scipy`` (``pip install sampleflux[vision]``). @@ -250,7 +248,7 @@ class ConnectedComponents(Transform): min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). connectivity: Pixel neighborhood — ``4`` (orthogonal only) or ``8`` (orthogonal + diagonal). field: Name of the ``Mask`` field to label; blank (default) picks the first ``Mask`` (else first array). - output: Name of the field the ``Regions`` item is written to (added if new; role ``aux``). + output: Name of the key the ``Regions`` item is written to (added if new). """ handles = (Mask,) @@ -270,29 +268,27 @@ def __init__( self.field = field self.output = output - def _find_mask(self, sample: Sample) -> np.ndarray: + def _find_mask(self, record: Record) -> np.ndarray: """Resolve the mask to label (``self.field``, else the first ``Mask``, else the first array).""" if self.field: - if self.field not in sample.keys(): - raise ValueError( - f"ConnectedComponents: field {self.field!r} not in sample (fields: {list(sample.keys())})" - ) - data = item_data(sample[self.field]) + if self.field not in record: + raise ValueError(f"ConnectedComponents: field {self.field!r} not in record (keys: {list(record)})") + data = item_data(record[self.field]) else: data = None - for _key, item in sample.items(): + for _key, item in record.items(): if isinstance(item, Mask): data = item_data(item) break if data is None: - for _key, item in sample.items(): + for _key, item in record.items(): payload = item_data(item) if isinstance(payload, np.ndarray): data = payload break if data is None: raise ValueError( - f"ConnectedComponents: no Mask or array-bearing field in sample (fields: {list(sample.keys())})" + f"ConnectedComponents: no Mask or array-bearing field in record (keys: {list(record)})" ) if not isinstance(data, np.ndarray): raise TypeError(f"ConnectedComponents expects an np.ndarray mask, got {type(data).__name__}") @@ -300,11 +296,10 @@ def _find_mask(self, sample: Sample) -> np.ndarray: raise ValueError(f"ConnectedComponents expects a 2-D mask; got shape {data.shape}") return data - def __call__(self, sample: Sample) -> Sample: - mask = self._find_mask(sample) + def __call__(self, record: Record) -> Record: + mask = self._find_mask(record) bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) - out = sample.replace_field(self.output, Regions(boxes=list(bboxes))) - return out.set_role(self.output, "aux") + return {**record, self.output: Regions(boxes=list(bboxes))} __all__ = [ diff --git a/sampleflux/ops/parallel.py b/sampleflux/ops/parallel.py index 6904ebf..1a11ec7 100644 --- a/sampleflux/ops/parallel.py +++ b/sampleflux/ops/parallel.py @@ -11,8 +11,8 @@ Note: Do not nest a ``Parallel`` op inside another ``Parallel.ops`` — workers - must not themselves spawn workers. ``TransformChain``, ``Enable``, and any - pickle-safe per-sample op are fine inside. + must not themselves spawn workers. ``Pipeline``, ``Enable``, and any + pickle-safe per-record op are fine inside. """ from __future__ import annotations @@ -25,8 +25,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.bag.sample import Sample from sampleflux.core import _worker_task +from sampleflux.items import Record @configurable(category="op", group="compose") @@ -34,7 +34,7 @@ class Parallel: """Run an inner op sub-pipeline in a worker pool with bounded prefetch. Args: - ops: Sequential sub-pipeline applied to each sample inside a worker. + ops: Sequential sub-pipeline applied to each record inside a worker. workers: Number of worker processes (spawn context). Must be >= 1. """ @@ -45,26 +45,26 @@ def __init__(self, ops: Optional[List[Any]] = None, workers: int = 4) -> None: def _materialize_ops(self) -> None: # Confluid post-construction paradigm leaves nested ops as Fluid - # markers; resolve them in-place on first use, mirroring TransformChain. + # markers; resolve them in-place on first use, mirroring Pipeline. for i, op in enumerate(self.ops): if isinstance(op, Fluid): self.ops[i] = flow(op) - def __call__(self, sample: Sample) -> Optional[Sample]: + def __call__(self, record: Record) -> Optional[Record]: # Inline fallback for non-streaming callers (e.g. Flux.__getitem__). Routed through - # _apply_op — the same contract-aware chokepoint the streamed route's _worker_task - # uses — so field-scoped ops (e.g. a pair-scoped op from the kinds grid) behave identically. + # _apply_op — the same op-family dispatch the streamed route's _worker_task uses — + # so bare library transforms behave identically. from sampleflux.core import _apply_op self._materialize_ops() - current: Optional[Sample] = sample + current: Optional[Record] = record for op in self.ops: if current is None: return None current = _apply_op(current, op) return current - def stream(self, samples: Iterable[Optional[Sample]]) -> Iterator[Optional[Sample]]: + def stream(self, samples: Iterable[Optional[Record]]) -> Iterator[Optional[Record]]: """Stream-level dispatch with bounded prefetch (in-order yield).""" if self.workers < 1: raise ValueError(f"Parallel(workers={self.workers!r}): must be >= 1") @@ -73,7 +73,7 @@ def stream(self, samples: Iterable[Optional[Sample]]) -> Iterator[Optional[Sampl limit = max(2 * self.workers, self.workers + 1) with concurrent.futures.ProcessPoolExecutor(max_workers=self.workers, mp_context=ctx) as executor: - pending: "deque[concurrent.futures.Future[Optional[Sample]]]" = deque() + pending: "deque[concurrent.futures.Future[Optional[Record]]]" = deque() for s in samples: if s is None: continue diff --git a/sampleflux/ops/random_apply.py b/sampleflux/ops/random_apply.py index 2e25058..325f5fc 100644 --- a/sampleflux/ops/random_apply.py +++ b/sampleflux/ops/random_apply.py @@ -1,10 +1,10 @@ """``RandomApply`` — apply an op with a given probability. -A compose-group op (alongside ``Enable`` / ``TransformChain`` / ``Parallel``): -wrap any single ``Sample → Sample`` op so it fires only *p* fraction of -the time. Samples that are skipped pass through unchanged. +A compose-group op (alongside ``Enable`` / ``Pipeline`` / ``Parallel``): +wrap any single op (native or a bare library transform) so it fires only *p* +fraction of the time. Records that are skipped pass through unchanged. -Modality-neutral — it threads any ``Sample`` through any op — so it lives +Modality-neutral — it threads any record through any op — so it lives in core sampleflux, not a domain package. """ @@ -14,7 +14,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag.sample import Sample +from sampleflux.items import Record logger = get_logger(__name__) @@ -24,7 +24,7 @@ class RandomApply: """Gate any op behind a Bernoulli coin flip. On each call, a uniform ``U ~ [0, 1)`` is drawn; if ``U < probability`` - the inner ``op`` is applied, otherwise the sample passes through unchanged. + the inner ``op`` is applied, otherwise the record passes through unchanged. ``op`` is flowed lazily on first use (Confluid ``!class:`` / ``!lazy:`` markers are resolved at call-time, not at construction), so building a @@ -36,12 +36,10 @@ class RandomApply: - !class:sampleflux.ops.random_apply.RandomApply probability: 0.5 - op: !class:sampleflux.ops.numpy.RescaleOp - in_min: -1.0 - in_max: 1.0 + op: !class:albumentations.HorizontalFlip {p: 1.0} Args: - op: Inner ``Sample → Sample`` callable to gate. Defaults to ``None`` + op: Inner op to gate (native op or bare library transform). Defaults to ``None`` (identity); validated lazily on first call. probability: Gate probability in ``[0, 1]``. ``0.0`` = never apply; ``1.0`` = always apply. Defaults to ``0.5``. @@ -59,24 +57,24 @@ def __init__( self.random_state = random_state self._gate_rng: Optional[random.Random] = None - def __call__(self, sample: Sample) -> Optional[Sample]: + def __call__(self, record: Record) -> Optional[Record]: if self.op is None: raise ValueError("RandomApply requires 'op' to be set before calling.") if self._gate_rng is None: self._gate_rng = random.Random(self.random_state) if self._gate_rng.random() >= self.probability: - return sample + return record from confluid import flow from confluid.fluid import Fluid - # _apply_op is the engine's single contract-aware chokepoint — routing through it - # (instead of op(sample)) lets a field-scoped op (e.g. a pair-scoped op from the - # kinds grid) nest inside the gate exactly as it would sit in a bare ops list. + # _apply_op is the engine's op-family dispatch — routing through it (instead of + # op(record)) lets a bare albumentations / torchvision-v2 transform nest inside + # the gate exactly as it would sit in a bare ops list. from sampleflux.core import _apply_op op = flow(self.op) if isinstance(self.op, Fluid) else self.op self.op = op # cache the flowed op so we only flow once - return _apply_op(sample, op) + return _apply_op(record, op) __all__ = ["RandomApply"] diff --git a/sampleflux/ops/sink.py b/sampleflux/ops/sink.py index 408dab2..17e71e8 100644 --- a/sampleflux/ops/sink.py +++ b/sampleflux/ops/sink.py @@ -1,8 +1,8 @@ """``SampleSinkOp`` — adapt a :class:`~sampleflux.storage.base.DataSink` as a pass-through op. -Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, the waivefront JSON -sinks …) slot into a ``Sample``-based op chain: on first call it opens the -sink, every call writes the sample and returns it unchanged, and ``close()`` +Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, a domain package's JSON +sinks …) slot into a record-based op chain: on first call it opens the +sink, every call writes the record and returns it unchanged, and ``close()`` flushes + closes. Modality-neutral (duck-typed ``open``/``write``/``close``), so it lives in core sampleflux. """ @@ -12,7 +12,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag.sample import Sample +from sampleflux.items import Record logger = get_logger(__name__) @@ -21,30 +21,25 @@ class SampleSinkOp: """Adapter: wrap a :class:`sampleflux.storage.base.DataSink` as a pass-through op. - Sinks (``JsonPerWindowSink``, ``JsonSink``, ``HDF5Sink`` …) implement the - ``open()`` / ``write(sample)`` / ``close()`` protocol and are normally - attached to a :class:`sampleflux.processing.DatasetProcessor` as the - flux's terminal sink. This adapter lets the same sinks slot into any - Sample-based op chain — notably the ``ops`` list of - :class:`waivefront.sinks.SigMFPredictionsSink`, where the model's - predictions arrive as a Sample whose metadata carries the new - ``predicted_regions`` and need to be persisted to disk just like a - segment-pipeline output. + Sinks implement the ``open()`` / ``write(record)`` / ``close()`` protocol and + are normally attached to a :class:`sampleflux.processing.DatasetProcessor` as + the flux's terminal sink. This adapter lets the same sinks slot into any + record-based op chain (e.g. persisting a prediction pipeline's outputs + mid-chain). On the first call the adapter calls ``sink.open()`` (when present); each - subsequent call forwards the Sample to ``sink.write(sample)`` and returns - the Sample unchanged. ``close()`` flushes (when present) and closes the - underlying sink — propagated by :class:`sampleflux.ops.enable.Enable` and - :class:`waivefront.sinks.SigMFPredictionsSink` at end-of-run. + subsequent call forwards the record to ``sink.write(record)`` and returns + the record unchanged. ``close()`` flushes (when present) and closes the + underlying sink — propagated by the composing ops at end-of-run. YAML:: - !class:sampleflux.ops.sink.SampleSinkOp - sink: !class:waivefront.sinks.JsonPerWindowSink - output_dir: ./predictions_per_window + sink: !class:sampleflux.storage.hdf5.HDF5Sink + path: ./records.h5 Args: - sink: A DataSink-like object exposing ``write(sample)`` (and optionally ``open``/``flush``/``close``). + sink: A DataSink-like object exposing ``write(record)`` (and optionally ``open``/``flush``/``close``). """ def __init__(self, sink: Any = None) -> None: @@ -52,7 +47,7 @@ def __init__(self, sink: Any = None) -> None: self.sink = sink self._opened = False - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if self.sink is None: raise ValueError("SampleSinkOp requires a non-None 'sink'.") if not self._opened: @@ -60,8 +55,8 @@ def __call__(self, sample: Sample) -> Sample: if callable(opener): opener() self._opened = True - self.sink.write(sample) - return sample + self.sink.write(record) + return record def close(self) -> None: flush = getattr(self.sink, "flush", None) diff --git a/sampleflux/ops/structure.py b/sampleflux/ops/structure.py index a7ca899..2cc8f48 100644 --- a/sampleflux/ops/structure.py +++ b/sampleflux/ops/structure.py @@ -1,9 +1,7 @@ -"""Structure ops for the typed bag — reshape a :class:`~sampleflux.bag.sample.Sample`'s fields. +"""Structure ops — reshape a record dict's entries. -The typed analogue of the classic triple-slot plumbing (``MetadataToTargetOp``, the stash/swap -family): where the old model moved values between the fixed ``input``/``target`` slots and the -shared metadata dict, the bag model just RENAMES, RETAGS, COPIES, or DROPS named fields. Each op -is a thin copy-on-write wrapper over a ``Sample`` mutator — no payload is touched. +Plumbing over the plain-dict carrier: RENAME, COPY, DROP, or SELECT named entries. Each op +is a thin copy-on-write dict expression — no payload is touched. All ops are lazy / zero-arg constructible (config validated in ``__call__``) and ``@configurable(category="op", group="structure")`` so they surface as canvas nodes. @@ -12,121 +10,96 @@ from typing import List, Optional from confluid import configurable -from typing_extensions import get_args -from sampleflux.bag.sample import ROLES, Role, Sample +from sampleflux.items import Record -__all__ = ["SetRole", "RenameField", "DropField", "CopyField", "SelectFields"] - - -@configurable(category="op", group="structure") -class SetRole: - """Retag a field's role (``input`` / ``target`` / ``aux`` / ``pred``) without moving it. - - The typed replacement for the classic "metadata value becomes the target" op — in the bag - model a field's role is a tag, so promotion is a retag, not a move. - - Args: - key: The field to retag. - role: The new role — one of ``input`` / ``target`` / ``aux`` / ``pred``. - """ - - def __init__(self, key: str = "", role: Role = "input") -> None: - self.key = key - self.role = role - - def __call__(self, sample: Sample) -> Sample: - if not self.key: - raise ValueError("SetRole: 'key' (the field to retag) is required") - if self.role not in get_args(Role): - raise ValueError(f"SetRole: invalid role {self.role!r} (allowed: {list(ROLES)})") - return sample.set_role(self.key, self.role) +__all__ = ["RenameField", "DropField", "CopyField", "SelectFields"] @configurable(category="op", group="structure") class RenameField: - """Rename a field (role travels with it). Renaming onto an existing key replaces it. + """Rename a record entry. Renaming onto an existing key replaces it. The sanctioned way to avoid a deliberate fan-in collision: rename on the producing branch - BEFORE the merge, instead of a merge-policy knob. + BEFORE the merge, instead of a merge-policy knob. Also the way to route a value into an + albumentations op's vocabulary (``image`` / ``mask`` / ``bboxes``). Args: - src: The field to rename. - dst: The new field name. + src: The entry to rename. + dst: The new key. """ def __init__(self, src: str = "", dst: str = "") -> None: self.src = src self.dst = dst - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.src or not self.dst: raise ValueError("RenameField: both 'src' and 'dst' are required") - return sample.rename(self.src, self.dst) + if self.src not in record: + raise KeyError(f"RenameField: unknown key {self.src!r} (keys: {list(record)})") + return {(self.dst if k == self.src else k): v for k, v in record.items()} @configurable(category="op", group="structure") class DropField: - """Remove a field from the bag (e.g. free a heavy Signal after its Spectrogram is derived). + """Remove an entry from the record (e.g. free a heavy signal after its spectrogram is derived). Args: - key: The field to remove. Missing keys raise unless ``missing_ok``. - missing_ok: Silently pass through when the field is absent (default False). + key: The entry to remove. Missing keys raise unless ``missing_ok``. + missing_ok: Silently pass through when the entry is absent (default False). """ def __init__(self, key: str = "", missing_ok: bool = False) -> None: self.key = key self.missing_ok = missing_ok - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.key: - raise ValueError("DropField: 'key' (the field to remove) is required") - if self.key not in sample: + raise ValueError("DropField: 'key' (the entry to remove) is required") + if self.key not in record: if self.missing_ok: - return sample - raise KeyError(f"DropField: unknown field {self.key!r} (fields: {list(sample.keys())})") - return sample.drop(self.key) + return record + raise KeyError(f"DropField: unknown key {self.key!r} (keys: {list(record)})") + return {k: v for k, v in record.items() if k != self.key} @configurable(category="op", group="structure") class CopyField: - """Duplicate a field under a new name (same item object; items are treated as immutable). + """Duplicate an entry under a new key (same value object; values are treated as immutable). Args: - src: The field to copy. - dst: The name of the copy. An existing ``dst`` is replaced. - role: Optional role for the copy; ``None`` keeps the source field's role. + src: The entry to copy. + dst: The key of the copy. An existing ``dst`` is replaced. """ - def __init__(self, src: str = "", dst: str = "", role: Optional[Role] = None) -> None: + def __init__(self, src: str = "", dst: str = "") -> None: self.src = src self.dst = dst - self.role = role - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.src or not self.dst: raise ValueError("CopyField: both 'src' and 'dst' are required") - if self.src not in sample: - raise KeyError(f"CopyField: unknown field {self.src!r} (fields: {list(sample.keys())})") - out = sample.replace_field(self.dst, sample[self.src]) - return out.set_role(self.dst, self.role if self.role is not None else sample.role_of(self.src)) + if self.src not in record: + raise KeyError(f"CopyField: unknown key {self.src!r} (keys: {list(record)})") + return {**record, self.dst: record[self.src]} @configurable(category="op", group="structure") class SelectFields: - """Keep ONLY the named fields (order = the given order); everything else is dropped. + """Keep ONLY the named entries (order = the given order); everything else is dropped. Args: - keys: The fields to keep. Unknown keys raise (a silent miss hides a typo). + keys: The entries to keep. Unknown keys raise (a silent miss hides a typo). """ def __init__(self, keys: Optional[List[str]] = None) -> None: self.keys = list(keys) if keys else [] - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.keys: - raise ValueError("SelectFields: 'keys' (the fields to keep) is required") - missing = [k for k in self.keys if k not in sample] + raise ValueError("SelectFields: 'keys' (the entries to keep) is required") + missing = [k for k in self.keys if k not in record] if missing: - raise KeyError(f"SelectFields: unknown fields {missing} (fields: {list(sample.keys())})") - return Sample({k: sample[k] for k in self.keys}, {k: sample.role_of(k) for k in self.keys}) + raise KeyError(f"SelectFields: unknown keys {missing} (keys: {list(record)})") + return {k: record[k] for k in self.keys} diff --git a/sampleflux/ops/target.py b/sampleflux/ops/target.py index 609a559..b4c30cd 100644 --- a/sampleflux/ops/target.py +++ b/sampleflux/ops/target.py @@ -1,4 +1,4 @@ -"""Typed-bag target-shaping transforms. +"""Target-shaping transforms over plain-dict records. * :class:`MetadataToTarget` promotes a field / attr value into a target ``Label``. * :class:`EncodeTarget` / :class:`DecodeTarget` map a class-name ``Label`` to a class-id @@ -20,9 +20,8 @@ import numpy as np from confluid import configurable -from sampleflux.bag.items import Label, Mask, Regions, item_data -from sampleflux.bag.sample import Sample -from sampleflux.bag.transform import Transform +from sampleflux.items import Label, Mask, Record, Regions, item_data +from sampleflux.transform import Transform #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo #: fails at the call site and UIs / form-specs enumerate the choices. @@ -143,16 +142,16 @@ class MetadataToTarget(Transform): Reads a value from a SOURCE field (``field``; blank picks the first ``Label``, else the first field) — either the field's natural value (a ``Label``'s ``.value``, otherwise the item's array payload) or, when ``key`` is set, the named ATTRIBUTE of the source item — - and writes a fresh :class:`~sampleflux.Label` under ``output`` tagged ``target``. + and writes a fresh :class:`~sampleflux.Label` under ``output``. - In a typical typed classification pipeline the source emits the label directly as a - ``Label`` field already tagged ``target``, so this op is usually a NO-OP-ish re-home; it + In a typical classification pipeline the source emits the label directly as a + ``Label`` field, so this op is usually a NO-OP-ish re-home; it exists for the case where a label rode as another item's attribute (``key=``). Args: field: Source field to read; blank (default) picks the first ``Label`` field, else the first field. key: Optional attribute name to read off the source item; blank (default) reads the item's natural value. - output: Field the target ``Label`` is written to (added if new); its role is set to ``target``. + output: Key the target ``Label`` is written to (added if new). """ handles = (Label,) @@ -165,23 +164,21 @@ def __init__(self, field: str = "", key: str = "", output: str = "target") -> No self.key = str(key) self.output = str(output) - def _find_source(self, sample: Sample) -> str: + def _find_source(self, record: Record) -> str: """Resolve the KEY of the source field (``self.field``, else first ``Label``, else first field).""" if self.field: - if self.field not in sample.keys(): - raise ValueError( - f"MetadataToTarget: field {self.field!r} not in sample (fields: {list(sample.keys())})" - ) + if self.field not in record: + raise ValueError(f"MetadataToTarget: field {self.field!r} not in record (keys: {list(record)})") return self.field - for key, _item in sample.items_of_type(Label): + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): return key - for key in sample.keys(): + for key in record: return key - raise ValueError("MetadataToTarget: sample is empty — no source field to read") + raise ValueError("MetadataToTarget: record is empty — no source field to read") - def __call__(self, sample: Sample) -> Sample: - key = self._find_source(sample) - item = sample[key] + def __call__(self, record: Record) -> Record: + key = self._find_source(record) + item = record[key] if self.key: if not hasattr(item, self.key): raise AttributeError( @@ -192,19 +189,18 @@ def __call__(self, sample: Sample) -> Sample: value = item.value else: value = item_data(item) - out = sample.replace_field(self.output, Label(value)) - return out.set_role(self.output, "target") + return {**record, self.output: Label(value)} @configurable(category="op", group="structure") class EncodeTarget(Transform): - """A class-NAME ``Label`` → a class-ID ``Label`` (role ``target``). + """A class-NAME ``Label`` → a class-ID ``Label``. Reads a :class:`~sampleflux.Label` field (``field``; blank picks the first ``Label``) whose ``.value`` is a raw class name and maps it to its class id through the config-pinned ``mapping`` — the declarative ``LabelEncoder`` analogue. The result is a new :class:`~sampleflux.Label` (carrying the source label's ``classes`` vocabulary) written - under ``output`` — blank (default) replaces the source field in place — tagged ``target``. + under ``output`` — blank (default) replaces the source field in place. Args: mapping: Lookup from raw label name → class id, e.g. ``{"DJI AVATA2": 2, ...}``. Must be non-empty. @@ -212,8 +208,7 @@ class EncodeTarget(Transform): ``True``, substitute ``default``. default: Value written for an unknown label when ``ignore_unknown=True`` (default ``0``). field: ``Label`` field to encode; blank (default) picks the first ``Label`` field. - output: Field the encoded ``Label`` is written to; blank (default) replaces the source field - in place. Its role is set to ``target``. + output: Key the encoded ``Label`` is written to; blank (default) replaces the source field in place. """ handles = (Label,) @@ -236,28 +231,27 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_label(self, sample: Sample) -> str: + def _find_label(self, record: Record) -> str: """Resolve the KEY of the ``Label`` field to encode (``self.field`` or the first ``Label``).""" if self.field: - if self.field not in sample.keys(): - raise ValueError(f"EncodeTarget: field {self.field!r} not in sample (fields: {list(sample.keys())})") - item = sample[self.field] + if self.field not in record: + raise ValueError(f"EncodeTarget: field {self.field!r} not in record (keys: {list(record)})") + item = record[self.field] if not isinstance(item, Label): raise TypeError(f"EncodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label") return self.field - for key, _item in sample.items_of_type(Label): + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): return key - raise ValueError(f"EncodeTarget: no Label field in sample (fields: {list(sample.keys())})") + raise ValueError(f"EncodeTarget: no Label field in record (keys: {list(record)})") - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.mapping: raise ValueError("EncodeTarget: mapping must contain at least one entry.") - key = self._find_label(sample) - label = sample[key] + key = self._find_label(record) + label = record[key] encoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") out_key = self.output or key - out = sample.replace_field(out_key, Label(encoded, classes=label.classes)) - return out.set_role(out_key, "target") + return {**record, out_key: Label(encoded, classes=label.classes)} @configurable(category="op", group="structure") @@ -267,7 +261,7 @@ class DecodeTarget(Transform): Reads a :class:`~sampleflux.Label` field (``field``; blank picks the first ``Label``) whose ``.value`` is an encoded class id and maps it back to its label name through ``mapping`` — the readback half used in prediction / reporting. The result is a new - :class:`~sampleflux.Label` written under ``output`` (blank replaces in place) tagged ``target``. + :class:`~sampleflux.Label` written under ``output`` (blank replaces in place). Args: mapping: Lookup from class id → label name, e.g. ``{2: "DJI AVATA2", ...}``. Must be non-empty. @@ -275,8 +269,7 @@ class DecodeTarget(Transform): ``True``, substitute ``default``. default: Value written for an unknown id when ``ignore_unknown=True`` (default ``None``). field: ``Label`` field to decode; blank (default) picks the first ``Label`` field. - output: Field the decoded ``Label`` is written to; blank (default) replaces the source field - in place. Its role is set to ``target``. + output: Key the decoded ``Label`` is written to; blank (default) replaces the source field in place. """ handles = (Label,) @@ -299,28 +292,27 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_label(self, sample: Sample) -> str: + def _find_label(self, record: Record) -> str: """Resolve the KEY of the ``Label`` field to decode (``self.field`` or the first ``Label``).""" if self.field: - if self.field not in sample.keys(): - raise ValueError(f"DecodeTarget: field {self.field!r} not in sample (fields: {list(sample.keys())})") - item = sample[self.field] + if self.field not in record: + raise ValueError(f"DecodeTarget: field {self.field!r} not in record (keys: {list(record)})") + item = record[self.field] if not isinstance(item, Label): raise TypeError(f"DecodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label") return self.field - for key, _item in sample.items_of_type(Label): + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): return key - raise ValueError(f"DecodeTarget: no Label field in sample (fields: {list(sample.keys())})") + raise ValueError(f"DecodeTarget: no Label field in record (keys: {list(record)})") - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: if not self.mapping: raise ValueError("DecodeTarget: mapping must contain at least one entry.") - key = self._find_label(sample) - label = sample[key] + key = self._find_label(record) + label = record[key] decoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") out_key = self.output or key - out = sample.replace_field(out_key, Label(decoded, classes=label.classes)) - return out.set_role(out_key, "target") + return {**record, out_key: Label(decoded, classes=label.classes)} @configurable(category="op", group="structure") @@ -331,7 +323,7 @@ class CocoToTorchVisionDetection(Transform): first field) carrying a HuggingFace / COCO ``objects`` mapping and rewrites it to the torchvision detection target, riding as a :class:`~sampleflux.Regions` item under ``output`` (``boxes`` = the ``[N, 4]`` float32 xyxy tensor, ``labels`` = the ``[N]`` int64 - class-id tensor) tagged ``target``. An empty annotation yields empty ``[0,4]`` / ``[0]`` + class-id tensor). An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract). Args: @@ -340,7 +332,7 @@ class CocoToTorchVisionDetection(Transform): bbox_format: Box layout in pixels — ``xywh`` (COCO, default), ``xyxy``, or ``cxcywh``; output is xyxy. label_offset: Added to each class id (default ``0``). Set ``1`` to reserve class ``0`` for background. field: Source field with the objects mapping; blank (default) picks the first ``Label``, else the first field. - output: Field the target ``Regions`` is written to (added if new); its role is set to ``target``. + output: Key the target ``Regions`` is written to (added if new). """ handles = (Label,) @@ -364,37 +356,36 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_source(self, sample: Sample) -> str: + def _find_source(self, record: Record) -> str: """Resolve the KEY of the source field (``self.field``, else the first ``Label``, else the first field).""" if self.field: - if self.field not in sample.keys(): + if self.field not in record: raise ValueError( - f"CocoToTorchVisionDetection: field {self.field!r} not in sample (fields: {list(sample.keys())})" + f"CocoToTorchVisionDetection: field {self.field!r} not in record (keys: {list(record)})" ) return self.field - for key, _item in sample.items_of_type(Label): + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): return key - for key in sample.keys(): + for key in record: return key - raise ValueError("CocoToTorchVisionDetection: sample is empty — no source field to read") + raise ValueError("CocoToTorchVisionDetection: record is empty — no source field to read") - def __call__(self, sample: Sample) -> Sample: - key = self._find_source(sample) - item = sample[key] + def __call__(self, record: Record) -> Record: + key = self._find_source(record) + item = record[key] objects = item.value if isinstance(item, Label) else item_data(item) target = coco_to_detection(objects, self.bbox_key, self.category_key, self.bbox_format, self.label_offset) - out = sample.replace_field(self.output, Regions(boxes=target["boxes"], labels=target["labels"])) - return out.set_role(self.output, "target") + return {**record, self.output: Regions(boxes=target["boxes"], labels=target["labels"])} @configurable(category="op", group="structure") class MasksToDetectionBoxes(Transform): """A segmentation ``Mask`` → a target ``Regions``. - Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the bag, + Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the first array-bearing item) as a 2-D integer mask and derives one tight ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~sampleflux.Regions` item - under ``output`` tagged ``target``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. + under ``output``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. Args: label: Foreground class id assigned to every derived box (default ``1``; class 0 = background). @@ -402,7 +393,7 @@ class MasksToDetectionBoxes(Transform): min_area: Drop objects whose mask area (in pixels) is below this (default ``1``). connectivity: Connected-components neighborhood when ``connected=True`` — ``4`` or ``8`` (default ``4``). field: Name of the ``Mask`` field to read; blank (default) picks the first ``Mask`` (else the first array). - output: Field the target ``Regions`` is written to (added if new); its role is set to ``target``. + output: Key the target ``Regions`` is written to (added if new). """ handles = (Mask,) @@ -426,38 +417,35 @@ def __init__( self.field = str(field) self.output = str(output) - def _find_mask(self, sample: Sample) -> np.ndarray: + def _find_mask(self, record: Record) -> np.ndarray: """Resolve the mask array (``self.field``, else the first ``Mask``, else the first array-bearing item).""" if self.field: - if self.field not in sample.keys(): - raise ValueError( - f"MasksToDetectionBoxes: field {self.field!r} not in sample (fields: {list(sample.keys())})" - ) - data = item_data(sample[self.field]) + if self.field not in record: + raise ValueError(f"MasksToDetectionBoxes: field {self.field!r} not in record (keys: {list(record)})") + data = item_data(record[self.field]) else: data = None - for _key, item in sample.items_of_type(Mask): + for _key, item in ((k, v) for k, v in record.items() if isinstance(v, Mask)): data = item_data(item) break if data is None: - for _key, item in sample.items(): + for _key, item in record.items(): payload = item_data(item) if isinstance(payload, np.ndarray): data = payload break if data is None: raise ValueError( - f"MasksToDetectionBoxes: no Mask or array-bearing field in sample (fields: {list(sample.keys())})" + f"MasksToDetectionBoxes: no Mask or array-bearing field in record (keys: {list(record)})" ) if not isinstance(data, np.ndarray): raise TypeError(f"MasksToDetectionBoxes: expected an np.ndarray mask, got {type(data).__name__}") return data - def __call__(self, sample: Sample) -> Sample: - mask = self._find_mask(sample) + def __call__(self, record: Record) -> Record: + mask = self._find_mask(record) target = masks_to_detection(mask, self.label, self.connected, self.min_area, self.connectivity) - out = sample.replace_field(self.output, Regions(boxes=target["boxes"], labels=target["labels"])) - return out.set_role(self.output, "target") + return {**record, self.output: Regions(boxes=target["boxes"], labels=target["labels"])} __all__ = [ diff --git a/sampleflux/ops/torch.py b/sampleflux/ops/torch.py index ab54834..472efb2 100644 --- a/sampleflux/ops/torch.py +++ b/sampleflux/ops/torch.py @@ -4,10 +4,8 @@ import torch from confluid import configurable -from sampleflux.bag.items import Image as ImageItem -from sampleflux.bag.items import NDArrayItem, item_data -from sampleflux.bag.sample import Sample -from sampleflux.bag.transform import Transform +from sampleflux.items import NDArrayItem, Record, item_data +from sampleflux.transform import Transform def to_tensor(img: Any, normalize: bool = True, mode: Optional[str] = None) -> torch.Tensor: @@ -40,31 +38,31 @@ def to_tensor(img: Any, normalize: bool = True, mode: Optional[str] = None) -> t @configurable(category="op", group="torch") class ToTensor(Transform): - """An array-bearing field → a CHW-float ``Image`` item. + """An array-bearing field → a LIVE CHW-float ``torch.Tensor`` record value. Reads the payload of an array-bearing field (blank ``field`` picks the first array/PIL-bearing item — typically the :class:`~sampleflux.Image` a :class:`~sampleflux.ops.image.ConvertToImage` produced), runs the HWC→CHW transpose + ``normalize`` conversion (:func:`to_tensor`), and writes - a CHW-layout :class:`~sampleflux.Image` back. By default it REPLACES the resolved field in place - (``output`` blank), so the field's role is preserved; set ``output`` to write a NEW field - (tagged ``input``) instead. Any other field passes through untouched. + the resulting ``torch.Tensor`` back AS-IS. By default it REPLACES the resolved field in place + (``output`` blank); set ``output`` to write a NEW key instead. Any other key passes + through untouched. - IMPORTANT — payload dtype. A :class:`~sampleflux.NDArrayItem` (which ``Image`` is) coerces its - payload through ``np.asarray`` on construction, so it CANNOT hold a live ``torch.Tensor``: the - stored payload is a CHW ``float32`` **numpy** array. The typed collate stacks these payloads with - ``np.stack``; the numpy→``torch.Tensor`` conversion happens at the collate / model boundary. + The output is a PLAIN record value (a record holds arbitrary values — the ``"plain"`` codec + tag covers storage): ``collate_records`` stacks torch tensors natively (``torch.stack``), a + torchvision ``transforms.v2`` op downstream transforms it as-is, and array sinks convert via + ``to_numpy`` on write. It is deliberately NOT wrapped in an :class:`~sampleflux.Image` — an + ``NDArrayItem`` coerces through ``np.asarray`` and cannot hold a live tensor. Args: normalize: When ``True`` (default), scale integer pixel inputs into the ``[0, 1]`` float range. mode: Optional PIL mode to convert a PIL payload to (e.g. ``"RGB"`` forces 3 channels); ``None`` = as-is. field: Name of the source field to tensorize; blank (default) picks the first array/PIL-bearing item. - output: Field the CHW ``Image`` is written to; blank (default) replaces the source field in place - (role preserved). A non-blank name writes a new field tagged ``input``. + output: Key the tensor is written to; blank (default) replaces the source field in place. """ handles = (NDArrayItem,) consumes = (NDArrayItem,) - produces = (ImageItem,) + produces = (torch.Tensor,) def __init__( self, @@ -79,28 +77,24 @@ def __init__( self.field = field self.output = output - def _find_field(self, sample: Sample) -> str: + def _find_field(self, record: Record) -> str: """Resolve the KEY of the field to tensorize (``self.field`` or the first array/PIL item).""" if self.field: - if self.field not in sample.keys(): - raise ValueError(f"ToTensor: field {self.field!r} not in sample (fields: {list(sample.keys())})") + if self.field not in record: + raise ValueError(f"ToTensor: field {self.field!r} not in record (keys: {list(record)})") return self.field - for key, item in sample.items(): + for key, item in record.items(): data = item_data(item) if isinstance(data, np.ndarray) or hasattr(data, "convert"): return key - raise ValueError(f"ToTensor: no array-bearing field in sample (fields: {list(sample.keys())})") + raise ValueError(f"ToTensor: no array-bearing field in record (keys: {list(record)})") - def __call__(self, sample: Sample) -> Sample: - key = self._find_field(sample) - data = item_data(sample[key]) + def __call__(self, record: Record) -> Record: + key = self._find_field(record) + data = item_data(record[key]) tensor = to_tensor(data, self.normalize, self.mode) - arr = tensor.detach().cpu().numpy() out_key = self.output or key - out = sample.replace_field(out_key, ImageItem(arr, layout="CHW")) - if self.output: - out = out.set_role(out_key, "input") - return out + return {**record, out_key: tensor} __all__ = ["ToTensor", "to_tensor"] diff --git a/sampleflux/ops/torchvision.py b/sampleflux/ops/torchvision.py deleted file mode 100644 index f88675d..0000000 --- a/sampleflux/ops/torchvision.py +++ /dev/null @@ -1,198 +0,0 @@ -"""``TorchvisionTransformOp`` — run torchvision ``transforms.v2`` transforms as a SampleFlux op. - -v2 transforms draw their random parameters ONCE per call and apply them to every -``tv_tensors`` carrier passed in, so a geometric augmentation moves the primary input item -AND (per the ``target`` mode) its segmentation mask / detection boxes consistently; the -aux fields pass through untouched. - -Transforms are authored **Confluid-natively** as nested ``!class:`` nodes:: - - - !class:sampleflux.ops.torchvision.TorchvisionTransformOp - target: mask - transforms: - - !class:torchvision.transforms.v2.RandomHorizontalFlip - p: 0.5 - -For per-transform graph nodes (one op per v2 transform, e.g. ``TvRandomHorizontalFlip``) -see :mod:`sampleflux.ops.torchvision_transforms`. - -Layout contract: torchvision operates on **CHW torch tensors** (PIL images pass through -natively; numpy HWC arrays are converted on entry) and the output is CHW tensors — no -:class:`~sampleflux.ops.torch.ToTensorOp` needed downstream. Contrast with -:class:`~sampleflux.ops.albumentations.AlbumentationsOp`, which stays numpy HWC. - -This module imports WITHOUT torchvision installed (it is entry-pointed for discovery); -torchvision is lazy-imported on first call and a missing install raises a clear error -pointing at the ``sampleflux[vision]`` extra. -""" - -from typing import Any, List, Optional, Tuple - -import numpy as np -from confluid import configurable -from loggair import get_logger - -from sampleflux.bag.items import Mask, item_data, with_data -from sampleflux.bag.sample import Sample, primary -from sampleflux.ops.albumentations import TargetMode, _resolve_transform - -logger = get_logger(__name__) - - -def _import_v2() -> Any: - """The ``torchvision.transforms.v2`` module, or a clear error naming the extra.""" - try: - from torchvision.transforms import v2 - except ImportError as exc: # pragma: no cover - exercised only without torchvision - raise ImportError( - "TorchvisionTransformOp requires torchvision (transforms.v2 / tv_tensors). " - 'Install it via `pip install "sampleflux[vision]"`.' - ) from exc - return v2 - - -@configurable(category="op", group="augment", random=True) -class TorchvisionTransformOp: - """Apply torchvision ``transforms.v2`` transforms to the primary input item (and optionally the target). - - Pass EITHER ``transform`` (one v2 transform, or a prebuilt ``v2.Compose``) OR - ``transforms`` (a list composed into a ``v2.Compose`` lazily) — never both. Entries - may be live v2 objects, Confluid ``!class:`` markers, or generated per-transform ops - (:mod:`sampleflux.ops.torchvision_transforms`), which unwrap to their inner library - transform. - - Target modes (the ``target`` knob): - - * ``"none"`` — input-only augmentation; the sample's target passes through untouched. - * ``"mask"`` — the target-role item is a segmentation mask (2-D array / tensor or PIL - ``L`` image), wrapped as ``tv_tensors.Mask`` so image and mask receive the SAME - spatial transform. - * ``"boxes"`` — the target-role item is the torchvision detection dict - ``{"boxes": [N,4] xyxy-pixel, "labels": [N]}`` (what - :class:`~sampleflux.ops.target.CocoToTorchVisionDetectionOp` / - :class:`~sampleflux.ops.target.MasksToDetectionBoxesOp` emit); boxes are wrapped as - ``tv_tensors.BoundingBoxes(format="XYXY", canvas_size=(H, W))`` and come back as - plain float32 / int64 tensors. - - Stochasticity lives in the library: v2 transforms draw from torch's global RNG - (``torch.manual_seed(N)`` pins it); gate per sample via - :class:`~sampleflux.ops.random_apply.RandomApply`. - - YAML: - - .. code-block:: yaml - - - !class:sampleflux.ops.torchvision.TorchvisionTransformOp - target: mask - transforms: - - !class:torchvision.transforms.v2.RandomHorizontalFlip - p: 0.5 - - Args: - transform: ONE ``transforms.v2`` transform or a prebuilt ``v2.Compose``. Validated lazily on first call. - transforms: List of v2 transforms composed lazily into a ``v2.Compose``. - target: Joint-augmentation mode — ``none`` (input-only, default), ``mask``, or ``boxes``. - """ - - def __init__( - self, - transform: Optional[object] = None, - transforms: Optional[List[Any]] = None, - target: TargetMode = "none", - ) -> None: - # Lazy / zero-arg: store config only; transforms are resolved/validated on first call. - self.transform = transform - self.transforms: List[Any] = list(transforms) if transforms else [] - self.target = target - self._pipeline: Optional[object] = None - self._pipeline_key: Optional[tuple] = None - - @property - def pipeline(self) -> Any: - """The live v2 transform — built lazily, cached until the configuration changes.""" - if self.transform is not None and self.transforms: - raise ValueError("TorchvisionTransformOp: pass either 'transform' or 'transforms', not both.") - key = (id(self.transform), tuple(id(t) for t in self.transforms)) - if self._pipeline is None or self._pipeline_key != key: - if self.transform is not None: - self._pipeline = _resolve_transform(self.transform) - elif self.transforms: - v2 = _import_v2() - self._pipeline = v2.Compose([_resolve_transform(entry) for entry in self.transforms]) - else: - raise ValueError( - "TorchvisionTransformOp requires 'transform' (one v2 transform / v2.Compose) or " - "'transforms' (a list) to be set before calling." - ) - self._pipeline_key = key - return self._pipeline - - def __call__(self, sample: Sample) -> Sample: - import torch - - _import_v2() # raise the actionable extra hint before any torchvision use - from torchvision import tv_tensors - - pipeline = self.pipeline - key, item = primary(sample, "input") - image = self._wrap_image(item_data(item), tv_tensors, torch) - - if self.target == "mask": - mask_field = next(iter(sample.items_of_type(Mask)), None) - if mask_field is None: - raise ValueError( - "TorchvisionTransformOp(target='mask'): no Mask field in the sample to transform jointly." - ) - mkey, mitem = mask_field - mask = self._wrap_mask(item_data(mitem), tv_tensors, torch) - out_image, out_mask = pipeline(image, mask) - result = sample.replace_field(key, with_data(item, self._unwrap(out_image, torch).detach().cpu().numpy())) - return result.replace_field(mkey, with_data(mitem, self._unwrap(out_mask, torch).detach().cpu().numpy())) - if self.target == "boxes": - raise NotImplementedError( - "TorchvisionTransformOp(target='boxes') is not yet ported to the typed-bag Regions target " - "(migration follow-up); use target='none' or 'mask'." - ) - out_image = self._unwrap(pipeline(image), torch) - return sample.replace_field(key, with_data(item, out_image.detach().cpu().numpy())) - - @staticmethod - def _wrap_image(value: Any, tv_tensors: Any, torch: Any) -> Any: - """``value`` as a v2 carrier: PIL passes through, tensors/arrays become CHW ``tv_tensors.Image``.""" - if hasattr(value, "convert"): # PIL image — v2 transforms handle it natively - return value - if isinstance(value, torch.Tensor): - tensor = value - else: - array = np.asarray(value) - tensor = torch.as_tensor(np.ascontiguousarray(array)) - if tensor.ndim == 3: # numpy convention is HWC; torchvision wants CHW - tensor = tensor.permute(2, 0, 1) - if tensor.ndim == 2: - tensor = tensor.unsqueeze(0) - return tv_tensors.Image(tensor) - - @staticmethod - def _wrap_mask(value: Any, tv_tensors: Any, torch: Any) -> Any: - """``value`` as a ``tv_tensors.Mask`` (PIL ``L`` images and 2-D arrays alike).""" - if hasattr(value, "convert"): - value = np.asarray(value) - tensor = value if isinstance(value, torch.Tensor) else torch.as_tensor(np.ascontiguousarray(value)) - return tv_tensors.Mask(tensor) - - @staticmethod - def _canvas_size(image: Any) -> Tuple[int, int]: - """``(H, W)`` of the wrapped input — the reference frame for bounding boxes.""" - if hasattr(image, "convert"): # PIL - return int(image.height), int(image.width) - return int(image.shape[-2]), int(image.shape[-1]) - - @staticmethod - def _unwrap(value: Any, torch: Any) -> Any: - """Strip the ``tv_tensors`` subclass so plain tensors flow downstream.""" - if isinstance(value, torch.Tensor): - return value.as_subclass(torch.Tensor) - return value - - -__all__ = ["TorchvisionTransformOp"] diff --git a/sampleflux/ops/torchvision_transforms.py b/sampleflux/ops/torchvision_transforms.py deleted file mode 100644 index 2e1edca..0000000 --- a/sampleflux/ops/torchvision_transforms.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Auto-generated ops: every torchvision ``transforms.v2`` transform as its own SampleFlux op. - -Generated at import time by :mod:`sampleflux.ops._augment_bridge` from the -``torchvision.transforms.v2`` namespace — one ``Tv`` op per concrete transform -(``TvRandomHorizontalFlip``, ``TvColorJitter``, …), each a subclass of -:class:`~sampleflux.ops.torchvision.TorchvisionTransformOp` mirroring the transform's own -constructor parameters plus the adapter's ``target`` knob. - -YAML (the registered short name resolves via the confluid registry): - -.. code-block:: yaml - - - !class:TvRandomHorizontalFlip - p: 0.5 - target: mask - -The ``Tv`` prefix is MANDATORY — torchvision and albumentations share many bare class -names (``ColorJitter``, ``Normalize``, ``Resize``, …) and confluid's registry is flat and -name-keyed, so unprefixed names would silently clobber each other. - -The module imports WITHOUT torchvision installed (entry-point discovery stays safe): it -then generates zero ops and logs a debug note pointing at the ``sampleflux[vision]`` -extra. Container transforms (``Compose`` / ``RandomApply`` / ``RandomChoice`` / -``RandomOrder``) are deliberately NOT generated — chaining ops is native SampleFlux. -""" - -from typing import Any, List, Tuple - -from loggair import get_logger - -from sampleflux.ops._augment_bridge import generate_transform_ops -from sampleflux.ops.torchvision import TorchvisionTransformOp - -logger = get_logger(__name__) - - -def __getattr__(name: str) -> Any: - # Generated names live in module globals; this fallback only fires for genuinely - # missing ones — and tells mypy the dynamic attributes exist (module-__getattr__ rule). - raise AttributeError( - f"module {__name__!r} has no generated op {name!r} — torchvision may be missing " - '(install via `pip install "sampleflux[vision]"`) or the transform may not exist ' - "in the installed version." - ) - - -#: v2 names not generated: containers (native chaining) + the deprecated v1 shim ToTensor. -_EXCLUDED = frozenset({"Compose", "RandomApply", "RandomChoice", "RandomOrder", "ToTensor", "Transform"}) - - -def _library_classes() -> List[Tuple[str, type]]: - """The concrete, generatable v2 transform classes, sorted by name (empty without torchvision).""" - try: - from torchvision.transforms import v2 - except ImportError: - logger.debug( - "torchvision not installed — no Tv* transform ops generated " - '(install via `pip install "sampleflux[vision]"`).' - ) - return [] - - out: List[Tuple[str, type]] = [] - for name in sorted(vars(v2)): - obj = getattr(v2, name) - if not (isinstance(obj, type) and issubclass(obj, v2.Transform)): - continue - if name in _EXCLUDED or obj.__name__ != name or name.startswith("_"): - continue - out.append((name, obj)) - return out - - -__all__ = generate_transform_ops( - classes=_library_classes(), - base=TorchvisionTransformOp, - prefix="Tv", - group="augment/torchvision", - module_globals=globals(), - seed_param=False, -) diff --git a/sampleflux/ops/transform_chain.py b/sampleflux/ops/transform_chain.py deleted file mode 100644 index c3b328c..0000000 --- a/sampleflux/ops/transform_chain.py +++ /dev/null @@ -1,88 +0,0 @@ -"""``TransformChain`` — group a sequence of ops into a single named unit. - -A compose-group op (alongside ``Enable`` / ``Parallel``): -wrap an ordered list of ``Sample → Sample`` callables so they appear as -one node on a visual canvas (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` -inputs instead of N wired ``SAMPLEFLUX_SAMPLE`` connections) and one named -block in a Confluid YAML. - -Unlike ``Enable`` there is no boolean gate — the chain always fires. -Unlike ``Parallel`` there is no worker pool — ops run sequentially in the -calling thread. If any op returns ``None`` the chain stops early and -propagates ``None`` (consistent with ``FilterOp`` semantics). -""" - -from typing import List, Optional - -from confluid import configurable -from loggair import get_logger - -from sampleflux.bag.sample import Sample - -logger = get_logger(__name__) - - -@configurable(category="op", group="compose") -class TransformChain: - """Apply a fixed sequence of ops to every sample, always. - - Wrap a list of ops into one named unit so they appear as a single node - on a visual canvas (dynamic ``op_0``, ``op_1``, … ``SAMPLEFLUX_OP`` inputs) - and one block in Confluid YAML instead of N separate connections. - - If any op in the chain returns ``None`` the remaining ops are skipped - and ``None`` is propagated (consistent with ``FilterOp`` semantics — - the sample is dropped). - - Inner ops keep full autonomy over their own randomness; ``TransformChain`` - itself is deterministic. Nest a - :class:`~sampleflux.ops.random_apply.RandomApply` inside the chain to - gate individual ops stochastically. - - YAML example:: - - - !class:sampleflux.ops.transform_chain.TransformChain - ops: - - !class:sampleflux.ops.random_apply.RandomApply - op: !class:waivefront.torchsig.processing.AWGNOp {} - probability: 0.8 - - !class:sampleflux.ops.torch.ToTensorOp {} - - Args: - ops: Ordered list of callables ``Sample -> Optional[Sample]`` applied - in sequence. Defaults to ``[]`` (identity — the chain passes - every sample through unchanged). - """ - - def __init__(self, ops: Optional[List] = None) -> None: - self.ops: List = list(ops) if ops else [] - - def __call__(self, sample: Sample) -> Optional[Sample]: - from confluid import flow - from confluid.fluid import Fluid - - # _apply_op = the engine's contract-aware chokepoint, so field-scoped ops - # (e.g. a pair-scoped op from the kinds grid) chain exactly as in a bare ops list. - from sampleflux.core import _apply_op - - current: Optional[Sample] = sample - for i, op in enumerate(self.ops): - if current is None: - return None - if isinstance(op, Fluid): - op = flow(op) - self.ops[i] = op - if op is None: - continue - current = _apply_op(current, op) - return current - - def close(self) -> None: - """Propagate close() to inner ops that own resources (e.g. SampleSinkOp).""" - for op in self.ops: - close_fn = getattr(op, "close", None) - if callable(close_fn): - close_fn() - - -__all__ = ["TransformChain"] diff --git a/sampleflux/projection.py b/sampleflux/projection.py index c4250ef..b9d54b0 100644 --- a/sampleflux/projection.py +++ b/sampleflux/projection.py @@ -1,14 +1,13 @@ -"""Field projection for SampleFlux sources — read only the input or only the target. +"""Key projection for SampleFlux sources — read only the record keys you need. -Walking a source for a single field (the canonical case: counting classes from -*targets*) should not pay for constructing the fields you don't need — e.g. -decoding image inputs you are about to throw away. This module adds an **opt-in** -projection protocol plus walk helpers that any consumer can use against any -source, with a correct (if unoptimized) fallback for sources that don't -implement the protocol. +Walking a source for a single key (the canonical case: counting classes from the label +key) should not pay for constructing the values you don't need — e.g. decoding image +inputs you are about to throw away. This module adds an **opt-in** projection protocol +plus walk helpers that any consumer can use against any source, with a correct (if +unoptimized) fallback for sources that don't implement the protocol. -The primitive is deliberately general (``input`` / ``target`` / ``metadata`` -selection); :func:`num_classes` is one helper built on top of it. +The primitive is deliberately general (any subset of record KEYS); :func:`num_classes` +is one helper built on top of it. Design notes ------------ @@ -22,83 +21,53 @@ make every ``Flux`` look classification-capable to duck-typed consumers. """ -from typing import Any, Collection, Dict, Iterator, Literal, Protocol, Tuple, get_args, runtime_checkable +from typing import Any, Collection, Iterator, Protocol, runtime_checkable -from sampleflux.bag.items import Label, item_data -from sampleflux.bag.sample import Role, Sample, primary - -#: The projectable :class:`~sampleflux.bag.sample.Sample` roles, as a *closed* -#: ``Literal`` rather than a bare ``str``. Typing the field set this way lets UIs, -#: form-spec builders, and MCP tool schemas enumerate the allowed values straight -#: from the annotation (``typing.get_args(ProjectionField)``) and lets a type -#: checker reject a typo at the call site. ``metadata`` maps onto the bag's ``aux`` role. -ProjectionField = Literal["input", "target", "metadata"] - -INPUT: ProjectionField = "input" -TARGET: ProjectionField = "target" -METADATA: ProjectionField = "metadata" -_FIELDS: Tuple[ProjectionField, ...] = get_args(ProjectionField) -_FIELD_ROLES: Dict[str, str] = {"input": "input", "target": "target", "metadata": "aux"} +from sampleflux.items import Label, Record, is_item, item_data @runtime_checkable class SupportsProjection(Protocol): - """A source that can yield partial :class:`~sampleflux.bag.sample.Sample` records. + """A source that can yield partial records restricted to the requested keys. - Implementers SHOULD avoid building unrequested fields — e.g. skip decoding the - input image when only ``target`` is asked for; that efficiency is the whole - point of the protocol. ``fields`` is a subset of ``{"input", "target", "metadata"}``. + Implementers SHOULD avoid building unrequested values — e.g. skip decoding the + input image when only the label key is asked for; that efficiency is the whole + point of the protocol. ``keys`` is a subset of the source's record keys. """ - def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: ... - + def project(self, keys: Collection[str]) -> Iterator[Record]: ... -def _carrier_field(sample: Sample, field: ProjectionField) -> Any: - """Read one field's VALUE from a typed :class:`Sample`. - - The value of the ``input`` / ``target`` role is the FIRST field of that role — a - ``Label``'s ``.value`` (the class id / scalar), else the item's raw payload - (:func:`item_data`). A missing role yields ``None``, so a target-only walk feeds - :func:`num_classes`. - """ - role: Role = "input" if field == INPUT else "target" - try: - _key, item = primary(sample, role) - except KeyError: - return None - return item.value if isinstance(item, Label) else item_data(item) - -def project(source: Any, fields: Collection[ProjectionField]) -> Iterator[Sample]: - """Yield partial records from ``source`` carrying only ``fields``. +def project(source: Any, keys: Collection[str]) -> Iterator[Record]: + """Yield partial records from ``source`` carrying only ``keys``. Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the - efficient path that skips building unrequested fields); otherwise falls back to a full - iteration that keeps only the fields whose role matches the request. Lazy: a generator. + efficient path that skips building unrequested values); otherwise falls back to a full + iteration that keeps only the requested keys. Lazy: a generator. """ - want = frozenset(fields) - unknown = want - frozenset(_FIELDS) - if unknown: - raise ValueError(f"Unknown projection field(s): {sorted(unknown)}; valid fields are {list(_FIELDS)}.") + want = frozenset(keys) if isinstance(source, SupportsProjection): yield from source.project(want) return - want_roles = {_FIELD_ROLES[f] for f in want} - for sample in source: - keep = [k for k in sample.keys() if sample.role_of(k) in want_roles] - yield Sample({k: sample[k] for k in keep}, {k: sample.role_of(k) for k in keep}) - + for record in source: + yield {k: v for k, v in record.items() if k in want} -def iter_inputs(source: Any) -> Iterator[Any]: - """Lazily yield each sample's ``input`` value (skipping target construction when supported).""" - for s in project(source, (INPUT,)): - yield _carrier_field(s, INPUT) +def iter_key(source: Any, key: str) -> Iterator[Any]: + """Lazily yield each record's ``key`` VALUE (skipping other-key construction when supported). -def iter_targets(source: Any) -> Iterator[Any]: - """Lazily yield each sample's ``target`` value (skipping input construction when supported).""" - for s in project(source, (TARGET,)): - yield _carrier_field(s, TARGET) + A :class:`~sampleflux.items.Label` unwraps to its ``.value`` (the class id / name); any + other registered item unwraps to its payload via :func:`~sampleflux.items.item_data`; a + plain value passes through verbatim. A record without ``key`` yields ``None``. + """ + for record in project(source, (key,)): + value = record.get(key) + if isinstance(value, Label): + yield value.value + elif is_item(value): + yield item_data(value) + else: + yield value def _to_int(value: Any) -> int: @@ -129,34 +98,36 @@ def _to_int(value: Any) -> int: raise TypeError(f"target {value!r} of type {type(value).__name__} is not a scalar class id") -def num_classes(source: Any) -> int: - """Derive the number of classes by walking **every** target in ``source``. +def num_classes(source: Any, key: str = "class") -> int: + """Derive the number of classes by walking **every** ``key`` value in ``source``. - Always walks the full target stream (target-only, so inputs are never + Always walks the full label stream (key-restricted, so other values are never constructed when the source supports projection) and returns ``max(class_id) + 1`` — the classifier-head size needed to cover the largest label, robust to a class id that happens not to appear in this split. Raises - ``ValueError`` if the source yields no targets (or a ``None`` target). + ``ValueError`` if the source yields no values (or a ``None`` value) under ``key``. This is the engine behind a dataset's lazy ``num_classes()`` method. + + Args: + source: The source to walk (any iterable of records; projection-aware when supported). + key: The record key holding the class label. Defaults to ``"label"``. """ highest = -1 - for target in iter_targets(source): + for target in iter_key(source, key): if target is None: - raise ValueError("num_classes: encountered a sample with no target — cannot derive a class count.") + raise ValueError(f"num_classes: encountered a record with no {key!r} value — cannot derive a class count.") cid = _to_int(target) if cid > highest: highest = cid if highest < 0: - raise ValueError("num_classes: source yielded no targets — cannot derive a class count.") + raise ValueError(f"num_classes: source yielded no {key!r} values — cannot derive a class count.") return highest + 1 __all__ = [ - "ProjectionField", "SupportsProjection", "project", - "iter_inputs", - "iter_targets", + "iter_key", "num_classes", ] diff --git a/sampleflux/sources.py b/sampleflux/sources.py index d281952..4b411f1 100644 --- a/sampleflux/sources.py +++ b/sampleflux/sources.py @@ -5,8 +5,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag import Image, Label, Sample -from sampleflux.projection import ProjectionField +from sampleflux.items import Image, Label, Record logger = get_logger(__name__) @@ -14,9 +13,9 @@ def _pass_through(item: Any) -> Any: """Pass a wrapped source's item through verbatim. - Every carrier is a typed-bag :class:`~sampleflux.Sample`; the view sources + Every carrier is a plain record dict; the view sources (:class:`DatasetSplit` / :class:`RangeSource` / :class:`ConcatSource`) only slice/index, - they never inspect payloads, so a source's samples flow through them unchanged. + they never inspect payloads, so a source's records flow through them unchanged. """ return item @@ -64,16 +63,16 @@ def _resolve_metadata_features( @configurable(category="source") class HuggingFaceSource: """ - SampleFlux Source for Hugging Face Datasets, yielding typed-bag :class:`~sampleflux.Sample`\\ s. + SampleFlux Source for Hugging Face Datasets, yielding plain record dicts. - Field mapping (the typed-bag layout that replaces the ``Sample(input, target, metadata)`` triple): + Key mapping (the record layout): - * the ``input_feature`` value (image / array) -> an :class:`~sampleflux.Image` field named - ``"image"`` (role ``input``); - * the ``target_feature`` value (label) -> a :class:`~sampleflux.Label` field named ``"class"`` - (role ``target``); - * each ``metadata_features`` column -> its own :class:`~sampleflux.Label` field keyed by the column - name (role ``aux``), plus the source-provenance ``hf_path`` / ``hf_split`` aux Labels. + * the ``input_feature`` value (image / array) -> an :class:`~sampleflux.Image` under the + record key ``"image"``; + * the ``target_feature`` value (label) -> a :class:`~sampleflux.Label` under the record key + ``"class"``; + * each ``metadata_features`` column -> its own :class:`~sampleflux.Label` keyed by the column + name, plus the source-provenance ``hf_path`` / ``hf_split`` Labels. Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and @@ -84,9 +83,9 @@ class HuggingFaceSource: Args: path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. split: HF split name (``train`` / ``validation`` / ``test`` / etc.). - input_feature: Dataset feature column mapped onto the ``"image"`` input field (an ``Image`` item). - target_feature: Dataset feature column mapped onto the ``"class"`` target field (a ``Label`` item). - metadata_features: Columns -> per-column aux ``Label`` fields; ``None``=none, ``"*"``=all-but-i/o, else a list. + input_feature: Dataset feature column mapped onto the ``"image"`` record key (an ``Image`` item). + target_feature: Dataset feature column mapped onto the ``"class"`` record key (a ``Label`` item). + metadata_features: Columns -> per-column ``Label`` entries; ``None``=none, ``"*"``=all-but-i/o, else a list. count: Optional cap on the number of samples yielded (useful for fast smoke runs). name: Optional HF subset/config name (e.g. for multi-config datasets). """ @@ -150,44 +149,36 @@ def resolved_metadata_features(self) -> List[str]: self.metadata_features, getattr(self.dataset, "column_names", None), self.input_feature, self.target_feature ) - def _to_typed_sample( + def _to_record( self, item: Any, metadata_features: List[str], - *, - want_input: bool = True, - want_target: bool = True, - want_meta: bool = True, - ) -> Sample: - """Assemble one :class:`~sampleflux.Sample` from a raw HF row dict (see the class docstring - for the field mapping). - - ``want_input`` / ``want_target`` / ``want_meta`` gate which roles are built — the projection - path (:meth:`project`) passes only the requested ones, so an unwanted image is never decoded. + keys: Optional[Collection[str]] = None, + ) -> Record: + """Assemble one record dict from a raw HF row dict (see the class docstring for the key mapping). + + ``keys`` gates which record entries are built (``None`` = all) — the projection path + (:meth:`project`) passes only the requested ones, so an unwanted image is never decoded. + ``metadata_features`` arrives pre-filtered on the projection path. """ - fields: Dict[str, Any] = {} - roles: Dict[str, Any] = {} - if want_input: + record: Record = {} + if keys is None or "image" in keys: # The input value (image/array) becomes an ``Image`` item; a PIL image / list is coerced # to an ndarray by ``Image.__new__`` (np.asarray), preserving the default HWC layout. - fields["image"] = Image(item.get(self.input_feature)) - roles["image"] = "input" - if want_target: - fields["class"] = Label(item.get(self.target_feature)) - roles["class"] = "target" - if want_meta: - # Each requested metadata column rides its OWN aux Label field (typed-bag: metadata belongs - # to the item it describes), keyed by the column name. Source provenance follows the same shape. - for feature in metadata_features: - fields[feature] = Label(item.get(feature)) - roles[feature] = "aux" - fields["hf_path"] = Label(self.path) - fields["hf_split"] = Label(self.split) - roles["hf_path"] = "aux" - roles["hf_split"] = "aux" - return Sample(fields, roles) - - def __iter__(self) -> Iterator[Sample]: + record["image"] = Image(item.get(self.input_feature)) + if keys is None or "class" in keys: + record["class"] = Label(item.get(self.target_feature)) + # Each requested metadata column rides its OWN Label entry keyed by the column name (the + # metadata a value needs travels WITH it). Source provenance follows the same shape. + for feature in metadata_features: + record[feature] = Label(item.get(feature)) + if keys is None or "hf_path" in keys: + record["hf_path"] = Label(self.path) + if keys is None or "hf_split" in keys: + record["hf_split"] = Label(self.split) + return record + + def __iter__(self) -> Iterator[Record]: dataset = self.dataset metadata_features = self.resolved_metadata_features limit = self.count or len(dataset) @@ -195,33 +186,29 @@ def __iter__(self) -> Iterator[Sample]: for counter, item in enumerate(dataset): if counter >= limit: break - yield self._to_typed_sample(item, metadata_features) + yield self._to_record(item, metadata_features) - def __getitem__(self, index: int) -> Sample: - return self._to_typed_sample(self.dataset[index], self.resolved_metadata_features) + def __getitem__(self, index: int) -> Record: + return self._to_record(self.dataset[index], self.resolved_metadata_features) - def project(self, fields: Collection[ProjectionField]) -> Iterator[Sample]: - """Yield role-restricted ``Sample``\\ s — the ``SupportsProjection`` efficient path. + def project(self, keys: Collection[str]) -> Iterator[Record]: + """Yield key-restricted records — the ``SupportsProjection`` efficient path. - Only the requested roles are built, so a target-only walk (e.g. :func:`~sampleflux.num_classes`) - skips decoding the image entirely: ``"input"`` -> the ``"image"`` field, ``"target"`` -> the - ``"class"`` Label, ``"metadata"`` -> the aux metadata-feature / provenance Labels. + Only the requested keys are built, so a label-only walk (e.g. :func:`~sampleflux.num_classes`) + skips decoding the image entirely: ``"image"`` -> the input feature, ``"class"`` -> the target + Label, plus any requested metadata-column / provenance keys. """ - want = frozenset(fields) + want = frozenset(keys) dataset = self.dataset - want_meta = "metadata" in want - metadata_features = self.resolved_metadata_features if want_meta else [] + # Resolve (and pre-filter) the metadata columns only when a key beyond the fixed image/class + # pair is requested — the "*" expansion needs the loaded dataset's columns. + meta_requested = bool(want - {"image", "class"}) + metadata_features = [f for f in self.resolved_metadata_features if f in want] if meta_requested else [] limit = self.count or len(dataset) for counter, item in enumerate(dataset): if counter >= limit: break - yield self._to_typed_sample( - item, - metadata_features, - want_input="input" in want, - want_target="target" in want, - want_meta=want_meta, - ) + yield self._to_record(item, metadata_features, keys=want) def __len__(self) -> int: # A ``count`` of 0 (or None) means "all samples", matching __iter__'s @@ -237,7 +224,7 @@ class DatasetSplit: """ Splits an indexable source into reproducible ``train`` / ``val`` / ``test`` views. - A ``source`` (it yields ``Sample``s and is wired into a trainer's ``source:`` slot), + A ``source`` (it yields records and is wired into a trainer's ``source:`` slot), not an engine — it applies no ops, it just exposes a reproducible partition of another source. (For a contiguous index slice use :class:`RangeSource`; to concatenate several sources use :class:`ConcatSource`.) @@ -370,7 +357,7 @@ def test(self) -> "_SplitView": """Cached test-split view (≈ ``test_fraction`` of the source).""" return self._view("test") - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: return iter(self._view(self.split or "train")) def __getitem__(self, index: int) -> Any: @@ -393,7 +380,7 @@ def __init__(self, source: Any, indices: List[int]) -> None: self.source = source self.indices = indices - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: for idx in self.indices: yield _pass_through(self.source[idx]) @@ -406,10 +393,10 @@ def __len__(self) -> int: @configurable(category="source") class RangeSource: - """A contiguous index slice ``[start:end)`` over an indexable source. + """A contiguous index slice ``[start:stop)`` over an indexable source. The plain-slice counterpart to :class:`DatasetSplit` (which shuffles + partitions) — - extracted from DatasetSplit's old "range mode". Negative ``start`` / ``end`` count from + extracted from DatasetSplit's old "range mode". Negative ``start`` / ``stop`` count from the end; both are clamped to ``[0, len(source)]``. Lazy: only index arithmetic happens up front; samples are produced on demand. @@ -418,20 +405,20 @@ class RangeSource: Args: source: The underlying indexable source (defaults to ``None``; validated lazily on first use). start: Inclusive start index (``None`` ⇒ 0; a negative value counts from the end). - end: Exclusive end index (``None`` ⇒ len(source); a negative value counts from the end). + stop: Exclusive stop index (``None`` ⇒ len(source); a negative value counts from the end). """ - def __init__(self, source: Any = None, start: Optional[int] = None, end: Optional[int] = None) -> None: + def __init__(self, source: Any = None, start: Optional[int] = None, stop: Optional[int] = None) -> None: # Lazy / zero-arg: store config only; the index arithmetic (and source validation) is deferred # to the ``indices`` property so the source can be configured post-construction. self.source = source self.start = start - self.end = end + self.stop = stop self._indices: Optional[List[int]] = None @property def indices(self) -> List[int]: - """The contiguous ``[start:end)`` source indices, computed lazily on first access and cached.""" + """The contiguous ``[start:stop)`` source indices, computed lazily on first access and cached.""" if self._indices is None: source = self.source if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): @@ -440,7 +427,7 @@ def indices(self) -> List[int]: ) n = len(source) s = 0 if self.start is None else self.start - e = n if self.end is None else self.end + e = n if self.stop is None else self.stop if s < 0: s = max(0, n + s) if e < 0: @@ -451,7 +438,7 @@ def indices(self) -> List[int]: logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) return self._indices - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: for idx in self.indices: yield _pass_through(self.source[idx]) @@ -518,7 +505,7 @@ def __getitem__(self, index: int) -> Any: start = self.offsets[j - 1] if j > 0 else 0 return _pass_through(self.sources[j][index - start]) - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: for src in self.sources: for item in src: yield _pass_through(item) diff --git a/sampleflux/storage/base.py b/sampleflux/storage/base.py index 974bd8a..91e2d21 100644 --- a/sampleflux/storage/base.py +++ b/sampleflux/storage/base.py @@ -4,15 +4,35 @@ import numpy as np import torch -from sampleflux.bag.sample import Sample +from sampleflux.items import Record -#: Root-attribute format tag stamped on stores written in the typed field-group layout. -TYPED_FORMAT = "typedsample-v1" +#: Root-attribute format tag stamped on stores written in the record key-group layout. +TYPED_FORMAT = "typedrecord-v1" + +#: Attr/entry name a ``"plain"``-tagged SCALAR record value is stored under (the array +#: payload of a plain value rides the regular ``data`` slot instead). +PLAIN_VALUE = "value" #: Prefix marking a JSON-encoded structured attr value (list/tuple/dict/None) in plain attrs. _JSON_MARK = "__json__:" +def require_record_format(found: Any, where: str) -> None: + """Raise unless ``found`` is the record-layout tag ``"typedrecord-v1"``. + + There is deliberately NO backward compatibility with pre-record layouts: a store whose + ``sampleflux_format`` tag is missing or different was written before the plain-dict + record model and must be re-generated with a current sink. + """ + if found == TYPED_FORMAT: + return + detail = "no sampleflux_format tag" if found is None else f"sampleflux_format={found!r}" + raise ValueError( + f"{where}: {detail} — expected {TYPED_FORMAT!r}. Pre-record-model datasets are not " + "readable/appendable; re-generate them with a current sink." + ) + + def to_numpy(data: Any) -> Any: """Convert a torch tensor to a numpy array for array-storage backends (HDF5 / Zarr). @@ -26,49 +46,23 @@ def to_numpy(data: Any) -> Any: @runtime_checkable class DataSource(Protocol): - """Minimum contract for a SampleFlux data source (LEGACY carrier — dies with the purge stage).""" + """Minimum contract for a SampleFlux data source.""" - def __iter__(self) -> Iterator[Sample]: - """Iterate over samples in the source.""" + def __iter__(self) -> Iterator[Record]: + """Iterate over records in the source.""" ... def __len__(self) -> int: - """Total number of samples available.""" + """Total number of records available.""" ... @runtime_checkable class DataSink(Protocol): - """Minimum contract for a SampleFlux data sink (LEGACY carrier — dies with the purge stage).""" - - def write(self, sample: Sample) -> None: - """Write a single sample to the sink.""" - ... - - def flush(self) -> None: - """Ensure all pending writes are committed to storage.""" - ... - - -@runtime_checkable -class TypedDataSource(Protocol): - """Minimum contract for a typed-bag data source.""" - - def __iter__(self) -> Iterator[Sample]: - """Iterate over typed samples in the source.""" - ... - - def __len__(self) -> int: - """Total number of samples available.""" - ... - - -@runtime_checkable -class TypedDataSink(Protocol): - """Minimum contract for a typed-bag data sink.""" + """Minimum contract for a SampleFlux data sink.""" - def write(self, sample: Sample) -> None: - """Write a single typed sample to the sink.""" + def write(self, record: Record) -> None: + """Write a single record to the sink.""" ... def flush(self) -> None: @@ -77,13 +71,13 @@ def flush(self) -> None: # -------------------------------------------------------------------------------------- -# The shared attr wire-format for the typed field-group layout (HDF5 attrs / Zarr .zattrs +# The shared attr wire-format for the record key-group layout (HDF5 attrs / Zarr .zattrs # / directory JSON all speak it): scalars stay native (queryable), array values become # separate datasets, and structured values (list/tuple/dict/None) ride a JSON string with # TUPLE TAGGING so a round-trip preserves tuple-ness (Regions.canvas == (H, W), not [H, W]). # -------------------------------------------------------------------------------------- def split_attrs(attrs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: - """Split an item's attrs into ``(plain, arrays)`` for storage. + """Split an encoded value's attrs into ``(plain, arrays)`` for storage. ``plain`` holds natively-storable scalars/strings plus JSON-marked structured values; ``arrays`` holds ndarray/Tensor attr values (stored as their own datasets). @@ -103,7 +97,7 @@ def split_attrs(attrs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: def restore_attrs(plain: Dict[str, Any], arrays: Dict[str, Any]) -> Dict[str, Any]: - """Rebuild an item's attrs dict from :func:`split_attrs`' two halves.""" + """Rebuild an encoded value's attrs dict from :func:`split_attrs`' two halves.""" attrs: Dict[str, Any] = {} for key, value in plain.items(): if isinstance(value, np.generic): diff --git a/sampleflux/storage/directory.py b/sampleflux/storage/directory.py index 352226d..c74d674 100644 --- a/sampleflux/storage/directory.py +++ b/sampleflux/storage/directory.py @@ -5,11 +5,20 @@ import confluid import numpy as np -from sampleflux.bag.io import EncodedItem, decode_item, encode_item -from sampleflux.bag.sample import Sample -from sampleflux.storage.base import DataSink, Storage, restore_attrs, split_attrs, to_numpy - -#: Typed-layout filenames inside each per-sample directory. +from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from sampleflux.items import Record +from sampleflux.storage.base import ( + PLAIN_VALUE, + TYPED_FORMAT, + DataSink, + Storage, + require_record_format, + restore_attrs, + split_attrs, + to_numpy, +) + +#: Record-layout filenames inside each per-sample directory. _FIELDS_JSON = "fields.json" _FIELDS_NPZ = "fields.npz" @@ -18,7 +27,7 @@ @confluid.configurable(category="sink") class DirectorySink(Storage, DataSink): """ - High-concurrency sink that stores each Sample in its own directory. + High-concurrency sink that stores each record in its own directory. Perfect for irregular data lengths and massive parallel writing. """ @@ -36,35 +45,50 @@ def open(self) -> "DirectorySink": self.path.mkdir(parents=True, exist_ok=True) return self - def write(self, sample: Any) -> None: - """Write a sample to its own subdirectory.""" - if not isinstance(sample, Sample): - raise TypeError(f"DirectorySink: expected a Sample bag, got {type(sample).__name__}") + def write(self, record: Any) -> None: + """Write a record to its own subdirectory.""" + if not isinstance(record, dict): + raise TypeError(f"DirectorySink: expected a record dict, got {type(record).__name__}") self.open() - self._write_typed(sample) + self._write_record(record) - def _write_typed(self, sample: Sample) -> None: - """One sample in the typed field-group layout: ``fields.json`` + ``fields.npz``. + def _write_record(self, record: Record) -> None: + """One record in the key-group layout: ``fields.json`` + ``fields.npz``. - ``fields.json`` describes every field (order, item type, role, plain attrs); - ``fields.npz`` carries the array halves — payloads keyed by field name, array-valued - attrs keyed ``.``. Every item serializes through the - :mod:`sampleflux.bag.io` codec, so externally-registered item types round-trip with - no storage edits. + ``fields.json`` describes every entry (order, item type, plain attrs — a ``"plain"`` + value's non-array payload rides its ``attrs`` under ``"value"``, JSON-marked when + structured); ``fields.npz`` carries the array halves — payloads keyed by record key, + array-valued attrs keyed ``.``. Every value serializes through the + :mod:`sampleflux.io` codec, so externally-registered item types round-trip with no + storage edits. """ sample_dir = self.path / f"{self._counter:06d}" sample_dir.mkdir(parents=True, exist_ok=True) - spec: Dict[str, Any] = {"sampleflux_format": "typedsample-v1", "fields": []} + spec: Dict[str, Any] = {"sampleflux_format": TYPED_FORMAT, "fields": []} payloads: Dict[str, Any] = {} - for key, item in sample.items(): - encoded = encode_item(item) + for key, value in record.items(): + encoded = encode_item(value) + if encoded.type_name == PLAIN_TYPE: + plain, arrays = split_attrs({PLAIN_VALUE: encoded.payload}) + has_payload = bool(arrays) + spec["fields"].append( + { + "key": key, + "type": encoded.type_name, + "attrs": plain, + "array_attrs": [], + "has_payload": has_payload, + } + ) + if has_payload: + payloads[key] = np.asarray(arrays[PLAIN_VALUE]) + continue plain, arrays = split_attrs(encoded.attrs) spec["fields"].append( { "key": key, "type": encoded.type_name, - "role": sample.role_of(key), "attrs": plain, "array_attrs": sorted(arrays), "has_payload": encoded.payload is not None, @@ -72,8 +96,8 @@ def _write_typed(self, sample: Sample) -> None: ) if encoded.payload is not None: payloads[key] = np.asarray(to_numpy(encoded.payload)) - for name, value in arrays.items(): - payloads[f"{key}.{name}"] = np.asarray(value) + for name, attr_value in arrays.items(): + payloads[f"{key}.{name}"] = np.asarray(attr_value) (sample_dir / _FIELDS_JSON).write_text(json.dumps(spec, indent=2)) if payloads: @@ -86,9 +110,9 @@ def flush(self) -> None: @confluid.configurable class DirectorySource(Storage): - """Read typed samples written by :class:`DirectorySink` (one ``fields.json`` + ``fields.npz`` per sample). + """Read records written by :class:`DirectorySink` (one ``fields.json`` + ``fields.npz`` per record). - The matching source of the sink's TYPED layout (one directory per sample, sorted by the + The matching source of the sink's record layout (one directory per record, sorted by the zero-padded name, so read order matches write order). Args: @@ -104,7 +128,7 @@ def _sample_dirs(self) -> list: raise FileNotFoundError(f"DirectorySource: {self.path} does not exist") return sorted(p for p in self.path.iterdir() if p.is_dir() and (p / _FIELDS_JSON).exists()) - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: for sample_dir in self._sample_dirs(): yield self._read(sample_dir) @@ -112,17 +136,26 @@ def __len__(self) -> int: return len(self._sample_dirs()) @staticmethod - def _read(sample_dir: Path) -> Sample: + def _read(sample_dir: Path) -> Record: spec = json.loads((sample_dir / _FIELDS_JSON).read_text()) + require_record_format(spec.get("sampleflux_format"), "DirectorySource") npz_path = sample_dir / _FIELDS_NPZ payloads = dict(np.load(npz_path, allow_pickle=False)) if npz_path.exists() else {} - fields: Dict[str, Any] = {} - roles: Dict[str, Any] = {} + record: Record = {} + payload: Any for entry in spec["fields"]: key = entry["key"] + if entry["type"] == PLAIN_TYPE: + # A plain value: array payload in the npz, non-array payload restored from the + # ``value`` attr (see DirectorySink._write_record). + if entry["has_payload"]: + payload = payloads[key] + else: + payload = restore_attrs(dict(entry["attrs"]), {}).get(PLAIN_VALUE) + record[key] = decode_item(EncodedItem(type_name=PLAIN_TYPE, payload=payload, attrs={})) + continue arrays = {name: payloads[f"{key}.{name}"] for name in entry["array_attrs"]} attrs = restore_attrs(dict(entry["attrs"]), arrays) payload = payloads[key] if entry["has_payload"] else None - fields[key] = decode_item(EncodedItem(type_name=entry["type"], payload=payload, attrs=attrs)) - roles[key] = entry["role"] - return Sample(fields, roles) + record[key] = decode_item(EncodedItem(type_name=entry["type"], payload=payload, attrs=attrs)) + return record diff --git a/sampleflux/storage/hdf5.py b/sampleflux/storage/hdf5.py index 1fbd830..61a3704 100644 --- a/sampleflux/storage/hdf5.py +++ b/sampleflux/storage/hdf5.py @@ -7,26 +7,45 @@ from confluid import configurable from loggair import get_logger -from sampleflux.bag.io import EncodedItem, decode_item, encode_item -from sampleflux.bag.sample import Sample -from sampleflux.storage.base import TYPED_FORMAT, DataSink, DataSource, Storage, restore_attrs, split_attrs, to_numpy +from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from sampleflux.items import Record +from sampleflux.storage.base import ( + PLAIN_VALUE, + TYPED_FORMAT, + DataSink, + DataSource, + Storage, + require_record_format, + restore_attrs, + split_attrs, + to_numpy, +) logger = get_logger("sampleflux.storage.hdf5") -#: Reserved field-group attr names in the typed layout (never item attrs). +#: Reserved key-group attr names in the record layout (never item attrs). _TYPE_ATTR = "__item_type__" -_ROLE_ATTR = "__role__" _ORDER_ATTR = "__field_order__" -def _read_typed_sample(group: h5py.Group) -> Sample: - """Decode one ``sNNNNNN`` sample group of the typed field-group layout.""" +def _read_record(group: h5py.Group) -> Record: + """Decode one ``sNNNNNN`` sample group of the record key-group layout.""" order = json.loads(group.attrs[_ORDER_ATTR]) - fields: Dict[str, Any] = {} - roles: Dict[str, Any] = {} + record: Record = {} + payload: Any for name in order: fgrp = group[name] - plain = {k: v for k, v in fgrp.attrs.items() if k not in (_TYPE_ATTR, _ROLE_ATTR)} + type_name = str(fgrp.attrs[_TYPE_ATTR]) + if type_name == PLAIN_TYPE: + # A plain value: array payload as the ``data`` dataset, scalar payload as the + # ``value`` attr (JSON-marked when structured) — see HDF5Sink._write_record. + if "data" in fgrp: + payload = fgrp["data"][()] + else: + payload = restore_attrs({PLAIN_VALUE: fgrp.attrs[PLAIN_VALUE]}, {})[PLAIN_VALUE] + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs={})) + continue + plain = {k: v for k, v in fgrp.attrs.items() if k != _TYPE_ATTR} arrays: Dict[str, Any] = {} agrp = fgrp.get("attrs") if isinstance(agrp, h5py.Group): @@ -34,31 +53,32 @@ def _read_typed_sample(group: h5py.Group) -> Sample: arrays[key] = dset[()] payload = fgrp["data"][()] if "data" in fgrp else None attrs = restore_attrs(dict(plain), arrays) - fields[name] = decode_item(EncodedItem(type_name=str(fgrp.attrs[_TYPE_ATTR]), payload=payload, attrs=attrs)) - roles[name] = str(fgrp.attrs[_ROLE_ATTR]) - return Sample(fields, roles) + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=attrs)) + return record @configurable class HDF5Source(Storage, DataSource): - """Clean, high-performance HDF5 data source.""" + """Read records written by :class:`HDF5Sink` (the record key-group layout). - def __init__( - self, - path: Union[str, Path] = "", - sample_key: str = "data", - target_key: Optional[str] = "target", - ) -> None: + Args: + path: Path to the HDF5 file written by HDF5Sink. + """ + + def __init__(self, path: Union[str, Path] = "") -> None: # Lazy / zero-arg: store config only; the file is opened lazily in open() (an unset path # surfaces there, not in __init__). self.path = Path(path) - self.sample_key = sample_key - self.target_key = target_key self._file: Optional[h5py.File] = None def open(self) -> "HDF5Source": if self._file is None: - self._file = h5py.File(self.path, "r") + handle = h5py.File(self.path, "r") + found = handle.attrs.get("sampleflux_format") + if found != TYPED_FORMAT: + handle.close() + require_record_format(found, "HDF5Source") + self._file = handle return self def close(self) -> None: @@ -66,18 +86,12 @@ def close(self) -> None: self._file.close() self._file = None - @property - def is_typed(self) -> bool: - """True when the file carries the typed field-group layout (``sampleflux_format`` root attr).""" - self.open() - return self._file is not None and self._file.attrs.get("sampleflux_format") == TYPED_FORMAT - - def __iter__(self) -> Iterator[Any]: + def __iter__(self) -> Iterator[Record]: self.open() if self._file is None: return for name in sorted(k for k in self._file.keys() if k.startswith("s")): - yield _read_typed_sample(self._file[name]) + yield _read_record(self._file[name]) def __len__(self) -> int: self.open() @@ -86,7 +100,7 @@ def __len__(self) -> int: return len([k for k in self._file.keys() if k.startswith("s")]) def iter_metadata(self) -> "Iterator[tuple[str, dict]]": - """(prefix, metadata) per sample WITHOUT loading data arrays (SupportsMetadataScan). + """(prefix, metadata) per record WITHOUT loading data arrays (SupportsMetadataScan). Array-valued metadata appears as shape/dtype stub strings — see :func:`sampleflux.storage.query.scan_hdf5_metadata`. @@ -99,7 +113,7 @@ def iter_metadata(self) -> "Iterator[tuple[str, dict]]": # category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @configurable(category="sink") class HDF5Sink(Storage, DataSink): - """High-performance HDF5 data sink focused on typed-bag ``Sample``s.""" + """High-performance HDF5 data sink for plain record dicts.""" def __init__( self, @@ -127,54 +141,57 @@ def close(self) -> None: self._file.close() self._file = None - def write(self, sample: Any) -> None: + def write(self, record: Any) -> None: self.open() if self._file is None: return - if not isinstance(sample, Sample): - raise TypeError(f"HDF5Sink: expected a Sample bag, got {type(sample).__name__}") - self._write_typed(sample) + if not isinstance(record, dict): + raise TypeError(f"HDF5Sink: expected a record dict, got {type(record).__name__}") + self._write_record(record) - def _write_typed(self, sample: Sample) -> None: - """One sample in the typed field-group layout — see ``docs/typed-model.md`` (storage). + def _write_record(self, record: Record) -> None: + """One record in the key-group layout. - Layout: root attr ``sampleflux_format = "typedsample-v1"``; per sample a group + Layout: root attr ``sampleflux_format = "typedrecord-v1"``; per record a group ``sNNNNNN`` (attr ``__field_order__`` preserves insertion order) holding one subgroup - per FIELD with attrs ``__item_type__``/``__role__`` + the item's plain attrs, the - payload as ``data``, and array-valued attrs as datasets under ``attrs/``. Every item - serializes through the :mod:`sampleflux.bag.io` codec, so externally-registered item - types round-trip with no storage edits. + per KEY with the ``__item_type__`` attr + the item's plain attrs, the payload as + ``data``, and array-valued attrs as datasets under ``attrs/``. A ``"plain"`` value + stores an array payload as ``data`` and any other payload as the ``value`` attr + (JSON-marked when structured). Every value serializes through the + :mod:`sampleflux.io` codec, so externally-registered item types round-trip with no + storage edits. """ assert self._file is not None - if self._counter == 0 and "sampleflux_format" not in self._file.attrs: - if any(k.endswith("_data") for k in self._file.keys()): - raise TypeError( - "HDF5Sink: this file carries the legacy Sample layout — cannot append a " - "Sample to it (one carrier per file)." - ) + existing = self._file.attrs.get("sampleflux_format") + if existing is None and len(self._file) == 0: self._file.attrs["sampleflux_format"] = TYPED_FORMAT - elif self._file.attrs.get("sampleflux_format") != TYPED_FORMAT: - raise TypeError( - "HDF5Sink: this file carries the legacy Sample layout — cannot append a " - "Sample to it (one carrier per file)." - ) + elif existing != TYPED_FORMAT: + require_record_format(existing, "HDF5Sink") group = self._file.create_group(f"s{self._counter:06d}") - group.attrs[_ORDER_ATTR] = json.dumps(list(sample.keys())) - for key, item in sample.items(): - encoded = encode_item(item) + group.attrs[_ORDER_ATTR] = json.dumps(list(record.keys())) + for key, value in record.items(): + encoded = encode_item(value) fgrp = group.create_group(key) fgrp.attrs[_TYPE_ATTR] = encoded.type_name - fgrp.attrs[_ROLE_ATTR] = sample.role_of(key) + if encoded.type_name == PLAIN_TYPE: + plain, arrays = split_attrs({PLAIN_VALUE: encoded.payload}) + if arrays: + arr = np.asarray(arrays[PLAIN_VALUE]) + kwargs = {"compression": self.compression} if self.compression and arr.ndim > 0 else {} + fgrp.create_dataset("data", data=arr, **kwargs) + else: + fgrp.attrs[PLAIN_VALUE] = plain[PLAIN_VALUE] + continue plain, arrays = split_attrs(encoded.attrs) - for name, value in plain.items(): - fgrp.attrs[name] = value + for name, attr_value in plain.items(): + fgrp.attrs[name] = attr_value if encoded.payload is not None: payload = np.asarray(to_numpy(encoded.payload)) kwargs = {"compression": self.compression} if self.compression and payload.ndim > 0 else {} fgrp.create_dataset("data", data=payload, **kwargs) - for name, value in arrays.items(): - arr = np.asarray(value) + for name, attr_value in arrays.items(): + arr = np.asarray(attr_value) kwargs = {"compression": self.compression} if self.compression and arr.ndim > 0 else {} fgrp.create_dataset(f"attrs/{name}", data=arr, **kwargs) self._counter += 1 diff --git a/sampleflux/storage/query.py b/sampleflux/storage/query.py index 2eab8bf..bb24c26 100644 --- a/sampleflux/storage/query.py +++ b/sampleflux/storage/query.py @@ -1,4 +1,4 @@ -"""Queryable metadata — filter stored samples by metadata predicates WITHOUT loading arrays. +"""Queryable metadata — filter stored records by metadata predicates WITHOUT loading arrays. Two pieces (mirroring the ``sampleflux.projection`` protocol-plus-fallback design): @@ -9,42 +9,43 @@ ``scan_zarr_metadata``); any external storage source can implement the protocol directly (it is structural — no import of this module required). - :class:`MetadataFilterSource` — a view source (``category="source"``) yielding only - the samples whose metadata passes a predicate: the YAML-friendly ``where`` expression + the records whose metadata passes a predicate: the YAML-friendly ``where`` expression (the same restricted-eval namespace as ``FormulaOp`` — metadata keys become variables) and/or a programmatic ``predicate`` callable. The matching index set is computed lazily from the metadata scan (cached), so arrays load only for matches; a source without the protocol falls back to a full iteration filter. -Existing HDF5/Zarr files are queryable with NO rewrite — their metadata already lives in -attrs/``.zattrs``. (A ``.metaindex`` sidecar accelerator is a TASKS.md follow-up if -scans ever become hot.) +Record-layout HDF5/Zarr stores are queryable with NO extra index — their metadata already +lives in attrs/``.zattrs``. (A ``.metaindex`` sidecar accelerator is a TASKS.md follow-up +if scans ever become hot.) """ import json from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, cast, runtime_checkable import h5py +import numpy as np from confluid import configurable from loggair import get_logger -from sampleflux.bag.io import encode_item -from sampleflux.bag.sample import Sample +from sampleflux.io import PLAIN_TYPE, encode_item +from sampleflux.items import Record from sampleflux.ops.formula import _FORMULA_NAMESPACE -from sampleflux.storage.base import TYPED_FORMAT, restore_attrs +from sampleflux.storage.base import PLAIN_VALUE, require_record_format, restore_attrs logger = get_logger("sampleflux.storage.query") __all__ = [ "MetadataFilterSource", "SupportsMetadataScan", + "record_metadata", "scan_hdf5_metadata", "scan_zarr_metadata", - "typed_sample_metadata", ] class _AttrView(dict): - """A metadata sub-dict that ALSO answers attribute access — so a typed scan's per-field + """A metadata sub-dict that ALSO answers attribute access — so a record scan's per-key attrs evaluate naturally in a ``where`` expression (``signal.samplerate > 1e6``) while staying a plain dict for programmatic predicates.""" @@ -56,85 +57,84 @@ def __getattr__(self, name: str) -> Any: def _viewed(metadata: Dict[str, Any]) -> Dict[str, Any]: - """Wrap dict-valued entries in :class:`_AttrView` (one level — the typed field/attr shape).""" + """Wrap dict-valued entries in :class:`_AttrView` (one level — the record key/attr shape).""" return {k: _AttrView(v) if isinstance(v, dict) else v for k, v in metadata.items()} -def typed_sample_metadata(sample: Sample) -> Dict[str, Dict[str, Any]]: - """A live sample's queryable metadata: ``{field: {attr: value}}`` (attrs via the io codec, - payloads untouched) — the same nested shape the typed storage scans yield.""" - return {key: dict(encode_item(item).attrs) for key, item in sample.items()} +def record_metadata(record: Record) -> Dict[str, Dict[str, Any]]: + """A live record's queryable metadata: ``{key: {attr: value}}`` (attrs via the io codec, + payloads untouched) — the same nested shape the record storage scans yield. A ``"plain"`` + value contributes ``{"value": }`` when its payload is a scalar, else ``{}``.""" + out: Dict[str, Dict[str, Any]] = {} + for key, value in record.items(): + encoded = encode_item(value) + if encoded.type_name == PLAIN_TYPE: + payload = encoded.payload + if isinstance(payload, np.generic): + payload = payload.item() + out[key] = {PLAIN_VALUE: payload} if isinstance(payload, (bool, int, float, str)) else {} + else: + out[key] = dict(encoded.attrs) + return out @runtime_checkable class SupportsMetadataScan(Protocol): - """A source that can enumerate per-sample metadata WITHOUT loading data arrays.""" + """A source that can enumerate per-record metadata WITHOUT loading data arrays.""" def iter_metadata(self) -> Iterator[Tuple[str, Dict[str, Any]]]: - """Yield ``(sample key, metadata dict)`` pairs, array payloads untouched.""" + """Yield ``(record key, metadata dict)`` pairs, array payloads untouched.""" ... # pragma: no cover - protocol def scan_hdf5_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: """Scan an ``HDF5Sink`` file's metadata WITHOUT loading payload arrays. - Legacy layout: dataset attrs + array-metadata SHAPE/DTYPE stubs (a stub string - ``""`` — queries can test presence/shape without an array - read). Typed field-group layout: per sample the NESTED shape ``{field: {attr: value}}`` - (plain attrs decoded; array-valued attrs as stubs) — a ``where`` expression addresses it - as ``"."`` (e.g. ``"signal.samplerate > 1e6"``). + Per record the NESTED shape ``{key: {attr: value}}`` (plain attrs decoded; array-valued + attrs as SHAPE/DTYPE stubs ``""`` — queries can test + presence/shape without an array read; a ``"plain"`` value's scalar payload appears under + its ``value`` attr) — a ``where`` expression addresses it as ``"."`` + (e.g. ``"signal.samplerate > 1e6"``). """ with h5py.File(str(path), "r") as handle: - if handle.attrs.get("sampleflux_format") == TYPED_FORMAT: - for name in sorted(k for k in handle.keys() if k.startswith("s")): - group = handle[name] - nested: Dict[str, Any] = {} - for field in json.loads(group.attrs["__field_order__"]): - fgrp = group[field] - plain = {k: v for k, v in fgrp.attrs.items() if k not in ("__item_type__", "__role__")} - attrs = restore_attrs(dict(plain), {}) - agrp = fgrp.get("attrs") - if isinstance(agrp, h5py.Group): - for key, dset in agrp.items(): - attrs[key] = f"" - nested[field] = attrs - yield name, nested - return - prefixes = sorted(k.split("_data")[0] for k in handle.keys() if k.endswith("_data")) - for prefix in prefixes: - metadata: Dict[str, Any] = dict(handle[f"{prefix}_data"].attrs) - meta_grp = handle.get(f"{prefix}_meta") - if isinstance(meta_grp, h5py.Group): - for key, dset in meta_grp.items(): - metadata[key] = f"" - yield prefix, metadata + require_record_format(handle.attrs.get("sampleflux_format"), "scan_hdf5_metadata") + for name in sorted(k for k in handle.keys() if k.startswith("s")): + group = handle[name] + nested: Dict[str, Any] = {} + for field in json.loads(group.attrs["__field_order__"]): + fgrp = group[field] + plain = {k: v for k, v in fgrp.attrs.items() if k != "__item_type__"} + attrs = restore_attrs(dict(plain), {}) + agrp = fgrp.get("attrs") + if isinstance(agrp, h5py.Group): + for key, dset in agrp.items(): + attrs[key] = f"" + nested[field] = attrs + yield name, nested def scan_zarr_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: """Scan a ``ZarrGroupSink`` store's metadata: ``.zattrs`` only, no payload arrays. - Typed field-group stores yield the same NESTED ``{field: {attr: value}}`` shape as the - HDF5 scan (array-valued attrs as name stubs). + Yields the same NESTED ``{key: {attr: value}}`` shape as the HDF5 scan (array-valued + attrs as name stubs). """ import zarr root = zarr.open_group(str(path), mode="r") - if root.attrs.get("sampleflux_format") == TYPED_FORMAT: - for name in sorted(root.group_keys()): - group = cast(Any, root[name]) - nested: Dict[str, Any] = {} - for field in json.loads(group.attrs["__field_order__"]): - fgrp = group[field] - plain = {k: v for k, v in dict(fgrp.attrs).items() if k not in ("__item_type__", "__role__")} - attrs = restore_attrs(plain, {}) - if "attrs" in fgrp: - for key in fgrp["attrs"].array_keys(): - attrs[key] = f"" - nested[field] = attrs - yield name, nested - return + require_record_format(root.attrs.get("sampleflux_format"), "scan_zarr_metadata") for name in sorted(root.group_keys()): - yield name, dict(root[name].attrs) + group = cast(Any, root[name]) + nested: Dict[str, Any] = {} + for field in json.loads(group.attrs["__field_order__"]): + fgrp = group[field] + plain = {k: v for k, v in dict(fgrp.attrs).items() if k != "__item_type__"} + attrs = restore_attrs(plain, {}) + if "attrs" in fgrp: + for key in fgrp["attrs"].array_keys(): + attrs[key] = f"" + nested[field] = attrs + yield name, nested def _where_predicate(where: str) -> Callable[[Dict[str, Any]], bool]: @@ -142,24 +142,24 @@ def _where_predicate(where: str) -> Callable[[Dict[str, Any]], bool]: The expression evaluates in the FormulaOp restricted namespace (``math.*`` + ``abs``/``min``/``max``/``round``/``pow``, no builtins) with the metadata KEYS bound - as variables — e.g. ``"snr_db > 10 and drone == 'DJI'"`` (legacy flat metadata) or - ``"signal.samplerate > 1e6"`` (a typed scan's per-field attrs). A missing key/attr - (NameError/AttributeError) means the sample does not match (logged at debug); any + as variables — e.g. ``"snr_db > 10 and drone == 'DJI'"`` (a flat external scan) or + ``"signal.samplerate > 1e6"`` (a record scan's per-key attrs). A missing key/attr + (NameError/AttributeError) means the record does not match (logged at debug); any other evaluation error raises (a malformed expression must fail loudly). - NOTE: a field named like a Python keyword (e.g. ``class``) cannot be addressed in an - expression — query such fields via the programmatic ``predicate`` (metadata is plain - nested dicts there), or give queryable fields non-keyword names. + NOTE: a record key named like a Python keyword (e.g. ``class``) cannot be addressed in + an expression — query such keys via the programmatic ``predicate`` (metadata is plain + nested dicts there), or give queryable keys non-keyword names. """ def _predicate(metadata: Dict[str, Any]) -> bool: - # Dict-valued entries (the typed scans' per-field attr dicts) evaluate through - # _AttrView so "." reads naturally; flat legacy metadata is untouched. + # Dict-valued entries (the record scans' per-key attr dicts) evaluate through + # _AttrView so "." reads naturally; flat external metadata is untouched. namespace = {**_FORMULA_NAMESPACE, **_viewed(metadata)} try: return bool(eval(where, {"__builtins__": {}}, namespace)) # noqa: S307 - restricted namespace except (NameError, AttributeError, KeyError) as exc: - logger.debug(f"MetadataFilterSource: where={where!r} — {exc}; sample treated as non-matching") + logger.debug(f"MetadataFilterSource: where={where!r} — {exc}; record treated as non-matching") return False except Exception as exc: raise ValueError(f"MetadataFilterSource: where expression {where!r} failed: {exc}") from exc @@ -169,10 +169,10 @@ def _predicate(metadata: Dict[str, Any]) -> bool: @configurable(category="source") class MetadataFilterSource: - """A view source yielding only the samples whose metadata matches. + """A view source yielding only the records whose metadata matches. Filtering uses the wrapped source's :class:`SupportsMetadataScan` protocol when - available (metadata-only scan — data arrays load ONLY for matching samples, via the + available (metadata-only scan — data arrays load ONLY for matching records, via the source's ``__getitem__``), else falls back to full-iteration filtering (the projection-module pattern). Match criteria compose with AND: the ``where`` expression and the programmatic ``predicate`` must both pass when both are set. @@ -204,7 +204,7 @@ def _match(self, metadata: Dict[str, Any]) -> bool: @property def matches(self) -> List[int]: - """Indices of matching samples (computed once per instance; ``_matches = None`` resets).""" + """Indices of matching records (computed once per instance; ``_matches = None`` resets).""" if self._matches is None: if self.source is None: raise ValueError("MetadataFilterSource: a 'source' is required") @@ -215,34 +215,32 @@ def matches(self) -> List[int]: else: logger.debug( f"MetadataFilterSource: {type(self.source).__name__} has no iter_metadata — " - "falling back to full-iteration filtering (arrays load for every sample)." + "falling back to full-iteration filtering (arrays load for every record)." ) - self._matches = [ - i for i, sample in enumerate(self.source) if self._match(typed_sample_metadata(sample)) - ] + self._matches = [i for i, record in enumerate(self.source) if self._match(record_metadata(record))] return self._matches - def __iter__(self) -> Iterator[Sample]: + def __iter__(self) -> Iterator[Record]: matches = self.matches # validates the source before iteration source: Any = self.source if hasattr(source, "__getitem__"): for index in matches: - yield cast(Sample, source[index]) + yield cast(Record, source[index]) else: match_set = set(matches) - for i, sample in enumerate(source): + for i, record in enumerate(source): if i in match_set: - yield cast(Sample, sample) + yield cast(Record, record) def __len__(self) -> int: return len(self.matches) - def __getitem__(self, index: int) -> Sample: + def __getitem__(self, index: int) -> Record: source_index = self.matches[index] source: Any = self.source if hasattr(source, "__getitem__"): - return cast(Sample, source[source_index]) - for i, sample in enumerate(source): + return cast(Record, source[source_index]) + for i, record in enumerate(source): if i == source_index: - return cast(Sample, sample) + return cast(Record, record) raise IndexError(index) diff --git a/sampleflux/storage/zarr.py b/sampleflux/storage/zarr.py index e350667..10cdbb5 100644 --- a/sampleflux/storage/zarr.py +++ b/sampleflux/storage/zarr.py @@ -6,25 +6,44 @@ import numpy as np import zarr -from sampleflux.bag.io import EncodedItem, decode_item, encode_item -from sampleflux.bag.sample import Sample, primary -from sampleflux.storage.base import TYPED_FORMAT, DataSink, DataSource, Storage, restore_attrs, split_attrs, to_numpy - -#: Reserved field-group attr names in the typed layout (never item attrs). +from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from sampleflux.items import Record +from sampleflux.storage.base import ( + PLAIN_VALUE, + TYPED_FORMAT, + DataSink, + DataSource, + Storage, + require_record_format, + restore_attrs, + split_attrs, + to_numpy, +) + +#: Reserved key-group attr names in the record layout (never item attrs). _TYPE_ATTR = "__item_type__" -_ROLE_ATTR = "__role__" _ORDER_ATTR = "__field_order__" -def _read_typed_group(grp: "zarr.Group") -> Sample: - """Decode one ``sample_NNNNNN`` group of the typed field-group layout.""" +def _read_record(grp: "zarr.Group") -> Record: + """Decode one ``sample_NNNNNN`` group of the record key-group layout.""" order = json.loads(str(grp.attrs[_ORDER_ATTR])) - fields: Dict[str, Any] = {} - roles: Dict[str, Any] = {} + record: Record = {} + payload: Any for name in order: fgrp = cast(zarr.Group, grp[name]) fattrs = dict(fgrp.attrs) - plain = {k: v for k, v in fattrs.items() if k not in (_TYPE_ATTR, _ROLE_ATTR)} + type_name = str(fattrs[_TYPE_ATTR]) + if type_name == PLAIN_TYPE: + # A plain value: array payload as the ``data`` array, scalar payload as the + # ``value`` attr (JSON-marked when structured) — see ZarrGroupSink._write_record. + if "data" in fgrp.array_keys(): + payload = np.asarray(cast(zarr.Array, fgrp["data"])[:]) + else: + payload = restore_attrs({PLAIN_VALUE: fattrs[PLAIN_VALUE]}, {})[PLAIN_VALUE] + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs={})) + continue + plain = {k: v for k, v in fattrs.items() if k != _TYPE_ATTR} arrays: Dict[str, Any] = {} if "attrs" in fgrp: agrp = cast(zarr.Group, fgrp["attrs"]) @@ -32,16 +51,15 @@ def _read_typed_group(grp: "zarr.Group") -> Sample: arrays[key] = np.asarray(cast(zarr.Array, agrp[key])[:]) payload = np.asarray(cast(zarr.Array, fgrp["data"])[:]) if "data" in fgrp.array_keys() else None attrs = restore_attrs(plain, arrays) - fields[name] = decode_item(EncodedItem(type_name=str(fattrs[_TYPE_ATTR]), payload=payload, attrs=attrs)) - roles[name] = str(fattrs[_ROLE_ATTR]) - return Sample(fields, roles) + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=attrs)) + return record # category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. @confluid.configurable(category="sink") class ZarrGroupSink(Storage, DataSink): """ - Stores each sample as a unique array within a Zarr group. + Stores each record as a unique group within a Zarr group. Supports variable lengths while keeping data in a single bundle. """ @@ -60,41 +78,42 @@ def open(self) -> "ZarrGroupSink": pass return self - def write(self, sample: Any) -> None: + def write(self, record: Any) -> None: self.open() if self._root is None: raise RuntimeError("Zarr group not open") - if not isinstance(sample, Sample): - raise TypeError(f"ZarrGroupSink: expected a Sample bag, got {type(sample).__name__}") - self._write_typed(sample) + if not isinstance(record, dict): + raise TypeError(f"ZarrGroupSink: expected a record dict, got {type(record).__name__}") + self._write_record(record) - def _write_typed(self, sample: Sample) -> None: - """One sample in the typed field-group layout (the Zarr twin of HDF5Sink._write_typed).""" + def _write_record(self, record: Record) -> None: + """One record in the key-group layout (the Zarr twin of HDF5Sink._write_record).""" assert self._root is not None - existing_format = self._root.attrs.get("sampleflux_format") - if existing_format is None: - if any(True for _ in self._root.group_keys()) and self._counter == 0: - raise TypeError( - "ZarrGroupSink: this store carries the legacy Sample layout — cannot append a " - "Sample to it (one carrier per store)." - ) + existing = self._root.attrs.get("sampleflux_format") + if existing is None and not any(True for _ in self._root.group_keys()): self._root.attrs["sampleflux_format"] = TYPED_FORMAT - elif existing_format != TYPED_FORMAT: - raise TypeError(f"ZarrGroupSink: unknown store format {existing_format!r}") + elif existing != TYPED_FORMAT: + require_record_format(existing, "ZarrGroupSink") grp = self._root.require_group(f"sample_{self._counter:06d}") - grp.attrs[_ORDER_ATTR] = json.dumps(list(sample.keys())) - for key, item in sample.items(): - encoded = encode_item(item) + grp.attrs[_ORDER_ATTR] = json.dumps(list(record.keys())) + for key, value in record.items(): + encoded = encode_item(value) fgrp = grp.require_group(key) fgrp.attrs[_TYPE_ATTR] = encoded.type_name - fgrp.attrs[_ROLE_ATTR] = sample.role_of(key) + if encoded.type_name == PLAIN_TYPE: + plain, arrays = split_attrs({PLAIN_VALUE: encoded.payload}) + if arrays: + fgrp.create_array("data", data=np.asarray(arrays[PLAIN_VALUE]), overwrite=True) + else: + fgrp.attrs[PLAIN_VALUE] = plain[PLAIN_VALUE] + continue plain, arrays = split_attrs(encoded.attrs) fgrp.attrs.update(plain) if encoded.payload is not None: fgrp.create_array("data", data=np.asarray(to_numpy(encoded.payload)), overwrite=True) - for name, value in arrays.items(): - fgrp.create_array(f"attrs/{name}", data=np.asarray(value), overwrite=True) + for name, attr_value in arrays.items(): + fgrp.create_array(f"attrs/{name}", data=np.asarray(attr_value), overwrite=True) self._counter += 1 def flush(self) -> None: @@ -103,51 +122,36 @@ def flush(self) -> None: @confluid.configurable class ZarrGroupSource(Storage, DataSource): - """Read samples written by :class:`ZarrGroupSink` (one Zarr group per sample). + """Read records written by :class:`ZarrGroupSink` (one Zarr group per record). - Mirrors the group sink's layout: each ``sample_NNNNNN`` subgroup carries a - ``data`` array, an optional ``target`` array, and the sample metadata as - group attributes (``.zattrs``). Groups are iterated in sorted name order so - the read order matches the write order. + Mirrors the group sink's key-group layout; groups are iterated in sorted name + order so the read order matches the write order. Args: path: Path to the Zarr group written by ZarrGroupSink. - sample_key: Name of the per-sample array holding the primary input item's payload. - target_key: Name of the per-sample array holding the target-role item's payload (absent when no target). """ - def __init__( - self, - path: Union[str, Path] = "", - sample_key: str = "data", - target_key: str = "target", - ) -> None: + def __init__(self, path: Union[str, Path] = "") -> None: # Lazy / zero-arg: store config only; the group is opened lazily in open(). self.path = str(path) - self.sample_key = sample_key - self.target_key = target_key self._root: Optional[zarr.Group] = None def open(self) -> "ZarrGroupSource": if self._root is None: - self._root = zarr.open_group(self.path, mode="r") + root = zarr.open_group(self.path, mode="r") + require_record_format(root.attrs.get("sampleflux_format"), "ZarrGroupSource") + self._root = root return self def close(self) -> None: self._root = None - @property - def is_typed(self) -> bool: - """True when the store carries the typed field-group layout (``sampleflux_format`` root attr).""" - self.open() - return self._root is not None and self._root.attrs.get("sampleflux_format") == TYPED_FORMAT - - def __iter__(self) -> Iterator[Any]: + def __iter__(self) -> Iterator[Record]: self.open() if self._root is None: return for name in sorted(self._root.group_keys()): - yield _read_typed_group(cast(zarr.Group, self._root[name])) + yield _read_record(cast(zarr.Group, self._root[name])) def __len__(self) -> int: self.open() @@ -156,7 +160,7 @@ def __len__(self) -> int: return len(list(self._root.group_keys())) def iter_metadata(self) -> "Iterator[tuple[str, dict]]": - """(group name, ``.zattrs`` metadata) per sample WITHOUT loading arrays (SupportsMetadataScan).""" + """(group name, ``.zattrs`` metadata) per record WITHOUT loading arrays (SupportsMetadataScan).""" from sampleflux.storage.query import scan_zarr_metadata yield from scan_zarr_metadata(self.path) @@ -166,7 +170,7 @@ def iter_metadata(self) -> "Iterator[tuple[str, dict]]": @confluid.configurable(category="sink") class ZarrBatchSink(Storage, DataSink): """ - Optimized for uniform data. Appends samples into a single large Zarr array. + Optimized for uniform data. Appends records into a single large Zarr array. """ def __init__( @@ -200,20 +204,25 @@ def open(self) -> "ZarrBatchSink": ) return self - def write(self, sample: Any) -> None: + def write(self, record: Any) -> None: self.open() if self._data_arr is None: raise RuntimeError("Zarr array not open") - if not isinstance(sample, Sample): - raise TypeError(f"ZarrBatchSink: expected a Sample bag, got {type(sample).__name__}") - - # The batch sink stores ONE uniform array: the PRIMARY input field's payload per row, - # plus a one-time item template (type/field/attrs of the FIRST sample) so the source can - # rebuild typed rows. Uniform-batch by design — per-sample attr variation does not fit a - # single stacked array; use ZarrGroupSink for that. - key, item = primary(sample) - encoded = encode_item(item) - if "sampleflux_format" not in self._data_arr.attrs: + if not isinstance(record, dict): + raise TypeError(f"ZarrBatchSink: expected a record dict, got {type(record).__name__}") + if not record: + raise ValueError("ZarrBatchSink: cannot write an empty record") + + # The batch sink stores ONE uniform array: the FIRST record entry's payload per row + # (insertion order), plus a one-time item template (type/key/attrs of the FIRST record) + # so the source can rebuild typed rows. Uniform-batch by design — per-record attr + # variation does not fit a single stacked array; use ZarrGroupSink for that. + key, value = next(iter(record.items())) + encoded = encode_item(value) + existing = self._data_arr.attrs.get("sampleflux_format") + if existing is None: + if self._data_arr.shape[0] > 0: + require_record_format(None, "ZarrBatchSink") plain, arrays = split_attrs(encoded.attrs) if arrays: raise TypeError( @@ -223,6 +232,8 @@ def write(self, sample: Any) -> None: self._data_arr.attrs.update( {"sampleflux_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} ) + elif existing != TYPED_FORMAT: + require_record_format(existing, "ZarrBatchSink") self._data_arr.append([np.asarray(to_numpy(encoded.payload))], axis=0) self._counter += 1 @@ -232,12 +243,11 @@ def flush(self) -> None: @confluid.configurable class ZarrBatchSource(Storage, DataSource): - """Read samples written by :class:`ZarrBatchSink` (one stacked array). + """Read records written by :class:`ZarrBatchSink` (one stacked array). - The batch sink appends every sample's input along axis 0 of a single - ``data`` array and stores no per-sample target or metadata, so this source - yields input-only :class:`~sampleflux.sample.Sample` objects — one per row of - the leading axis. + The batch sink appends every record's first entry's payload along axis 0 of a + single ``data`` array plus a one-time uniform item template, so this source + yields single-key records — one per row of the leading axis. Args: path: Path to the Zarr store written by ZarrBatchSink (the directory holding the ``data`` array). @@ -250,18 +260,20 @@ def __init__(self, path: Union[str, Path] = "") -> None: def open(self) -> "ZarrBatchSource": if self._data_arr is None: - self._data_arr = zarr.open_array(store=f"{self.path}/data", mode="r") + arr = zarr.open_array(store=f"{self.path}/data", mode="r") + require_record_format(arr.attrs.get("sampleflux_format"), "ZarrBatchSource") + self._data_arr = arr return self def close(self) -> None: self._data_arr = None - def __iter__(self) -> Iterator[Any]: + def __iter__(self) -> Iterator[Record]: self.open() if self._data_arr is None: return attrs = dict(self._data_arr.attrs) - # Typed batch rows: rebuild each row as the stored item type under the stored field key + # Record batch rows: rebuild each row as the stored item type under the stored key # (uniform template — see ZarrBatchSink.write). field = str(attrs["__field__"]) type_name = str(attrs[_TYPE_ATTR]) @@ -271,7 +283,7 @@ def __iter__(self) -> Iterator[Any]: for i in range(self._data_arr.shape[0]): payload = np.asarray(self._data_arr[i]) item = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=item_attrs)) - yield Sample({field: item}) + yield {field: item} def __len__(self) -> int: self.open() diff --git a/sampleflux/transform.py b/sampleflux/transform.py new file mode 100644 index 0000000..b1c15d8 --- /dev/null +++ b/sampleflux/transform.py @@ -0,0 +1,173 @@ +"""``Transform`` — type-dispatched record ops with once-per-record parameters, plus ``Pipeline``. + +A transform samples its random / configured parameters ONCE per record (:meth:`Transform.get_params`), +then walks the dict and, for each value whose type it handles, applies the registered +kernel (:mod:`sampleflux.dispatch`). Values it does not handle pass through untouched. + +Two properties fall out of this shape for free: + +* **Cross-field consistency.** Because params are sampled once and shared, one op moves + every handled value with the SAME decision (two ``Signal`` values in one record get the + same drawn SNR) — the torchvision-v2 model. +* **Open extension.** A new value type is taught to an existing op with one + ``@MyOp.kernel(NewType)`` registration and no core edit. + +Targeting is by TYPE; the ``field`` parameter pins an op to one named key when a record +holds several values of a handled type. + +sampleflux ships NO native augmentation ops — geometric/photometric augmentation comes from +the libraries (torchvision ``transforms.v2`` / albumentations) dropped into an ops list +AS-IS; the engine invokes each op family natively (see ``sampleflux.core._apply_op``). +There are no wrapper/adapter classes. +""" + +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from confluid import configurable + +from sampleflux.dispatch import Kernel, dispatch, register_kernel +from sampleflux.items import Record, item_data, with_data + +__all__ = [ + "Transform", + "Pipeline", + "FunctionTransform", + "as_transform", +] + + +class Transform: + """Base class for type-dispatched record ops (see the module docstring). + + Subclasses declare ``handles`` (the value types they process) and register a kernel per + type via ``@MyOp.kernel(ItemType)``. Override :meth:`get_params` to sample shared + parameters once per record. + + **The type-interface attributes** (``handles`` / ``consumes`` / ``optional`` / + ``produces``) describe the op to readers and machines (a visual editor's typed sockets, + a pipeline linter) WITHOUT executing it. They are DECLARATIVE: nothing validates them + against the op's behavior, and the base ``__call__`` dispatches on the KERNEL REGISTRY, + never on ``handles`` — the one exception is :class:`FunctionTransform`, whose + ``handles`` IS its ``isinstance`` application gate. Declare them truthfully or not at + all; a kernel op keeps ``handles`` mirroring its registered kernels, and a + type-CHANGING op (``__call__`` override) states its ``consumes``/``produces`` type flow + explicitly because no kernel registration reveals it. Full guidance: + ``docs/record-model.md`` → "Declaring an op's type interface". + + Args: + field: Apply only to this record key (still type-gated). None (default) = every value of a handled type. + """ + + #: Value types this op processes (everything else passes through). DECLARATIVE for + #: kernel ops (the kernel registry decides dispatch — keep this mirroring it); + #: ENFORCED only by FunctionTransform, where it is the isinstance application gate. + handles: Tuple[type, ...] = () + #: Type interface — input types the op NEEDS to do useful work. Empty = same as + #: ``handles``. Never gates execution; validate a hard requirement lazily in __call__. + consumes: Tuple[type, ...] = () + #: Type interface — input types used when present but not required (e.g. a geometric + #: op that also moves a Mask if the record has one). Never gates execution. + optional: Tuple[type, ...] = () + #: Type interface — value types this op ADDS or CHANGES (its output contract; what a + #: downstream op can rely on finding). Never gates execution. + produces: Tuple[type, ...] = () + + def __init__(self, field: Optional[str] = None) -> None: + self.field = field + + @classmethod + def kernel(cls, item_type: type) -> Callable[[Kernel], Kernel]: + """Register a kernel for ``item_type`` on this op (decorator over :func:`register_kernel`).""" + return register_kernel(cls, item_type) + + def get_params(self, record: Record) -> Dict[str, Any]: + """Sample the shared parameters for one call. Default: no params.""" + return {} + + def __call__(self, record: Record) -> Optional[Record]: + params = self.get_params(record) + out = dict(record) + for key, value in record.items(): + if self.field is not None and key != self.field: + continue + kernel = dispatch(type(self), type(value)) + if kernel is None: + continue + out[key] = kernel(value, params) + return out + + +@configurable(category="op", group="compose") +class Pipeline: + """Sequential application of ops — ``Pipeline([a, b, c])(record)`` is ``c(b(a(record)))``. + + THE compose-group unit: wrap an ordered list of ops so they appear as one named block in + a config and one node on a visual canvas. Entries may be native ops, BARE library + transforms (torchvision ``transforms.v2`` / albumentations — invoked natively by the + engine's op-family dispatch), or config-deferred markers (built on first use). If any op + returns ``None`` the chain stops and propagates ``None`` (filter-drop semantics). + + Args: + transforms: Ordered ops applied in sequence (bare library transforms allowed). Defaults to ``[]`` (identity). + """ + + def __init__(self, transforms: Optional[Sequence[Any]] = None) -> None: + # Lazy / zero-arg: store config only; marker flow happens on first call. + self.transforms: List[Any] = list(transforms) if transforms else [] + + def __call__(self, record: Record) -> Optional[Record]: + from confluid import flow + from confluid.fluid import Fluid + + # _apply_op = the engine's op-family dispatch, so a bare albumentations / + # torchvision-v2 transform nests here exactly as in a bare ops list. + from sampleflux.core import _apply_op + + current: Optional[Record] = record + for i, op in enumerate(self.transforms): + if current is None: + return None + if isinstance(op, Fluid): + op = flow(op) + self.transforms[i] = op + if op is None: + continue + current = _apply_op(current, op) + return current + + def close(self) -> None: + """Propagate close() to inner ops that own resources (e.g. a sink op).""" + for op in self.transforms: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + def __repr__(self) -> str: + return f"Pipeline([{', '.join(type(t).__name__ for t in self.transforms)}])" + + +class FunctionTransform(Transform): + """An op that applies one plain function ``fn(data) -> data`` to every handled value. + + The escape hatch for custom ops: no kernel registration, no subclass — wrap a + function and say which value types it applies to (via :func:`as_transform`). + """ + + def __init__(self, fn: Callable[[Any], Any], handles: Sequence[type], field: Optional[str] = None) -> None: + super().__init__(field=field) + self._fn = fn + self.handles = tuple(handles) + + def __call__(self, record: Record) -> Optional[Record]: + out = dict(record) + for key, value in record.items(): + if self.field is not None and key != self.field: + continue + if isinstance(value, self.handles): + out[key] = with_data(value, self._fn(item_data(value))) + return out + + +def as_transform(fn: Callable[[Any], Any], handles: Sequence[type], field: Optional[str] = None) -> FunctionTransform: + """Wrap a plain ``fn(data) -> data`` as a :class:`FunctionTransform` over ``handles``.""" + return FunctionTransform(fn, handles, field=field) diff --git a/tests/_bag_fixtures.py b/tests/_fixtures.py similarity index 69% rename from tests/_bag_fixtures.py rename to tests/_fixtures.py index 27a3ec3..e5986e8 100644 --- a/tests/_bag_fixtures.py +++ b/tests/_fixtures.py @@ -1,19 +1,17 @@ -"""Test-local typed-bag fixtures. +"""Test-local record-model fixtures. ``FixtureFlip`` is the former native ``HorizontalFlip`` kept ONLY as a test fixture: sampleflux ships no native augmentation transforms (geometric/photometric augmentation comes from -torchvision v2 / albumentations through adapter coercion), but the kernel-dispatch machinery -(once-per-sample params, per-type kernels, MRO resolution, ``only=`` filter) still needs a -fully native transform to pin — and the fixture doubles as the ADAPTER-PARITY reference (a -`v2.RandomHorizontalFlip(p=1.0)` through the adapter must move image/mask/boxes exactly like -this native implementation does). +torchvision v2 / albumentations invoked natively by the engine's op-family dispatch), but the +kernel-dispatch machinery (once-per-record params, per-type kernels, MRO resolution, the +``field=`` pin) still needs a fully native transform to pin. """ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional import numpy as np -from sampleflux import Image, Mask, Regions, Sample, Transform, item_data, with_data +from sampleflux import Image, Mask, Record, Regions, Transform, item_data, with_data class FixtureFlip(Transform): @@ -24,13 +22,13 @@ class FixtureFlip(Transform): optional = (Mask, Regions) produces = (Image, Mask, Regions) - def __init__(self, p: float = 0.5, only: Optional[List[str]] = None) -> None: - super().__init__(only=only) + def __init__(self, p: float = 0.5, field: Optional[str] = None) -> None: + super().__init__(field=field) self.p = p - def get_params(self, sample: Sample) -> Dict[str, Any]: + def get_params(self, record: Record) -> Dict[str, Any]: do = float(np.random.random()) < self.p - return {"do": do, "width": _reference_width(sample)} + return {"do": do, "width": _reference_width(record)} @FixtureFlip.kernel(Image) @@ -59,9 +57,9 @@ def _flip_regions(item: Regions, params: Dict[str, Any]) -> Regions: return Regions(boxes=boxes, labels=item.labels, scores=item.scores, canvas=item.canvas) -def _reference_width(sample: Sample) -> Optional[int]: +def _reference_width(record: Record) -> Optional[int]: """The horizontal extent to flip boxes against — from the first Image/Mask, or a Regions canvas.""" - for _, item in sample.items(): + for _, item in record.items(): if isinstance(item, Image): arr = item_data(item) axis = 2 if getattr(item, "layout", "HWC") == "CHW" else 1 @@ -71,7 +69,7 @@ def _reference_width(sample: Sample) -> Optional[int]: arr = item_data(item) if arr.ndim >= 2: return int(arr.shape[1]) - for _, item in sample.items(): + for _, item in record.items(): if isinstance(item, Regions) and item.canvas: return int(item.canvas[1]) return None diff --git a/tests/test_bag_pipeline.py b/tests/test_bag_pipeline.py deleted file mode 100644 index 997dfdb..0000000 --- a/tests/test_bag_pipeline.py +++ /dev/null @@ -1,186 +0,0 @@ -"""The headline cross-library mixed pipeline + adapter behavior + import safety.""" - -import subprocess -import sys -from pathlib import Path - -import numpy as np -import pytest - -from sampleflux.bag import Image, Label, Mask, Pipeline, Regions, Sample -from tests._bag_fixtures import FixtureFlip - - -class TestImportSafety: - def test_typed_imports_without_torchvision(self) -> None: - # The top-level package must import without torchvision (adapters lazy-import their - # library inside method bodies), so discovery stays safe on hosts missing it. - code = "import sys; import sampleflux.bag; assert 'torchvision' not in sys.modules" - subprocess.run([sys.executable, "-c", code], check=True, cwd=str(Path(__file__).resolve().parents[1])) - - -class TestTorchvisionAdapter: - v2 = pytest.importorskip("torchvision.transforms.v2") - - def test_normalize_touches_only_image(self) -> None: - from sampleflux.bag.adapters import TorchvisionV2Adapter - - s = Sample( - {"image": Image(np.ones((4, 5, 3), dtype=np.float32)), "class": Label("x")}, - roles={"class": "target"}, - ) - out = TorchvisionV2Adapter(self.v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]))(s) - assert isinstance(out["image"], Image) and out["image"].layout == "HWC" - assert np.asarray(out["image"]).shape == (4, 5, 3) - assert np.allclose(np.asarray(out["image"]), 1.0) # (1-0.5)/0.5 - assert out["class"].value == "x" - - def test_flip_moves_image_mask_boxes_together(self) -> None: - from sampleflux.bag.adapters import TorchvisionV2Adapter - - s = Sample( - { - "image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32)), - "mask": Mask(np.arange(6 * 8).reshape(6, 8).astype(np.int64)), - "regions": Regions(boxes=[[1, 1, 4, 4]], canvas=(6, 8)), - } - ) - out = TorchvisionV2Adapter(self.v2.RandomHorizontalFlip(p=1.0))(s) - assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, ::-1]) - assert np.array_equal(np.asarray(out["mask"]), np.asarray(s["mask"])[:, ::-1]) - assert out["regions"].boxes == [[4.0, 1.0, 7.0, 4.0]] # W=8: x -> W-x - - def test_missing_transform_raises(self) -> None: - from sampleflux.bag.adapters import TorchvisionV2Adapter - - with pytest.raises(ValueError, match="must be set"): - TorchvisionV2Adapter()(Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) - - def test_no_handled_field_is_noop(self) -> None: - from sampleflux.bag.adapters import TorchvisionV2Adapter - - s = Sample({"class": Label("x")}) - assert TorchvisionV2Adapter(self.v2.RandomHorizontalFlip(p=1.0))(s) == s - - -class TestAlbumentationsAdapter: - def test_gaussnoise_touches_only_image(self) -> None: - import albumentations as A - - from sampleflux.bag.adapters import AlbumentationsAdapter - - s = Sample( - {"image": Image(np.full((6, 6, 3), 0.5, dtype=np.float32)), "class": Label("x")}, - roles={"class": "target"}, - ) - out = AlbumentationsAdapter(A.GaussNoise(p=1.0))(s) - assert isinstance(out["image"], Image) - assert not np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])) - assert out["class"].value == "x" - - def test_bboxes_wrapped_and_returned(self) -> None: - import albumentations as A - - from sampleflux.bag.adapters import AlbumentationsAdapter - - s = Sample( - { - "image": Image(np.random.rand(10, 12, 3).astype(np.float32)), - "regions": Regions(boxes=[[2, 3, 6, 7]], labels=[1], canvas=(10, 12)), - } - ) - out = AlbumentationsAdapter(A.HorizontalFlip(p=1.0))(s) - assert out["regions"].boxes[0] == pytest.approx([6.0, 3.0, 10.0, 7.0], abs=1e-4) # W=12 - assert out["regions"].labels == [1] - - def test_missing_transform_raises(self) -> None: - from sampleflux.bag.adapters import AlbumentationsAdapter - - with pytest.raises(ValueError, match="must be set"): - AlbumentationsAdapter()(Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.float32))})) - - -class TestMixedPipeline: - def test_cross_library_bare_transforms(self) -> None: - # BARE library transforms drop straight into the Pipeline — the registered adapters - # wrap them; no explicit TorchvisionV2Adapter(...) / AlbumentationsAdapter(...). Two - # torchvision v2 transforms and an albumentations transform mix in one pipeline; each - # hits only the field(s) of a type it handles, and the flip moves image+mask+boxes with - # one library draw. - v2 = pytest.importorskip("torchvision.transforms.v2") - import albumentations as A - - rng = np.random.default_rng(0) - sample = Sample( - { - "image": Image(rng.random((16, 20, 3)).astype(np.float32)), - "mask": Mask(rng.random((16, 20)) > 0.5), - "regions": Regions(boxes=[[2, 3, 6, 7]], labels=["a"], canvas=(16, 20)), - "class": Label("drone_x", classes=["noise", "drone_x"]), - }, - roles={"mask": "target", "regions": "target", "class": "target"}, - ) - out = Pipeline( - [ - v2.RandomHorizontalFlip(p=1.0), # torchvision v2: Image + Mask + Regions together - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.25, 0.25, 0.25]), # torchvision v2: Image - A.GaussNoise(p=1.0), # albumentations: Image - ] - )(sample) - assert not np.array_equal(np.asarray(out["image"]), np.asarray(sample["image"])) # flipped+normalized+noised - assert np.array_equal(np.asarray(out["mask"]), np.asarray(sample["mask"])[:, ::-1]) # flipped with the image - assert out["regions"].boxes[0] == pytest.approx([14, 3, 18, 7], abs=1e-4) # W=20: x -> W-x - assert out["class"].value == "drone_x" # label rode through untouched - assert out.roles == sample.roles # role tags preserved end-to-end - - -class TestCoercion: - def test_native_transform_passes_through(self) -> None: - from sampleflux.bag import coerce_transform - - flip = FixtureFlip() - assert coerce_transform(flip) is flip - - def test_torchvision_bare_transform_coerced(self) -> None: - v2 = pytest.importorskip("torchvision.transforms.v2") - - from sampleflux.bag import coerce_transform - from sampleflux.bag.adapters.torchvision import TorchvisionV2Adapter, is_torchvision_v2_transform - - norm = v2.Normalize(mean=[0.0], std=[1.0]) - assert is_torchvision_v2_transform(norm) - assert isinstance(coerce_transform(norm), TorchvisionV2Adapter) - - def test_albumentations_bare_transform_coerced(self) -> None: - import albumentations as A - - from sampleflux.bag import coerce_transform - from sampleflux.bag.adapters.albumentations import AlbumentationsAdapter, is_albumentations_transform - - noise = A.GaussNoise(p=1.0) - assert is_albumentations_transform(noise) - assert isinstance(coerce_transform(noise), AlbumentationsAdapter) - - def test_unknown_object_raises(self) -> None: - from sampleflux.bag import coerce_transform - - with pytest.raises(TypeError, match="don't know how to adapt"): - coerce_transform(object()) - - def test_register_custom_adapter(self) -> None: - # A user library object becomes droppable into a Pipeline with one register_adapter call. - from sampleflux.bag import FunctionTransform, coerce_transform, register_adapter - from sampleflux.bag.transform import Transform - - class MyLibDouble: # a foreign object, not a Transform - pass - - def factory(_obj: object) -> Transform: - return FunctionTransform(lambda d: d * 2, handles=(Image,)) - - register_adapter(lambda o: isinstance(o, MyLibDouble), factory) - - s = Sample({"image": Image(np.ones((2, 2, 3)))}) - out = Pipeline([MyLibDouble()])(s) - assert np.allclose(np.asarray(out["image"]), 2.0) - assert isinstance(coerce_transform(MyLibDouble()), FunctionTransform) diff --git a/tests/test_bag_sample.py b/tests/test_bag_sample.py deleted file mode 100644 index 0395d03..0000000 --- a/tests/test_bag_sample.py +++ /dev/null @@ -1,114 +0,0 @@ -"""``Sample`` — role tags, views, copy-on-write mutators, array-safe equality.""" - -import numpy as np -import pytest - -from sampleflux.bag.items import Image, Label, Regions -from sampleflux.bag.sample import ROLES, Sample - - -def _sample() -> Sample: - return Sample( - {"image": Image(np.ones(4)), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")}, - roles={"regions": "target", "class": "target"}, - ) - - -class TestRolesAndViews: - def test_default_role_is_input(self) -> None: - s = Sample({"a": Image(np.zeros((1, 1, 3))), "b": Label()}) - assert s.roles == {"a": "input", "b": "input"} - - def test_inputs_targets_aux(self) -> None: - s = _sample() - assert list(s.inputs()) == ["image"] - assert list(s.targets()) == ["regions", "class"] - assert s.aux() == {} - - def test_of_role_and_role_of(self) -> None: - s = _sample() - assert s.role_of("class") == "target" - assert list(s.of_role("input")) == ["image"] - - def test_items_of_type(self) -> None: - s = _sample() - assert [k for k, _ in s.items_of_type(Image)] == ["image"] - assert [k for k, _ in s.items_of_type(Image, Label)] == ["image", "class"] - - def test_roles_closed_set(self) -> None: - assert set(ROLES) == {"input", "target", "aux", "pred"} - - -class TestConstruction: - def test_role_for_unknown_field_raises(self) -> None: - with pytest.raises(KeyError, match="unknown field"): - Sample({"a": Label()}, roles={"b": "target"}) - - def test_invalid_role_raises(self) -> None: - with pytest.raises(ValueError, match="invalid role"): - Sample({"a": Label()}, roles={"a": "output"}) # type: ignore[dict-item] - - -class TestCopyOnWrite: - def test_set_role_returns_new(self) -> None: - s = _sample() - s2 = s.set_role("regions", "aux") - assert s2.role_of("regions") == "aux" - assert s.role_of("regions") == "target" # original untouched - - def test_set_role_unknown_and_invalid(self) -> None: - s = _sample() - with pytest.raises(KeyError): - s.set_role("nope", "aux") - with pytest.raises(ValueError): - s.set_role("class", "bogus") # type: ignore[arg-type] - - def test_replace_field_preserves_role(self) -> None: - s = _sample() - s2 = s.replace_field("class", Label("y")) - assert s2["class"].value == "y" and s2.role_of("class") == "target" - assert s["class"].value == "x" - - def test_replace_new_field_defaults_input(self) -> None: - s = _sample().replace_field("image", Image(np.zeros((2, 2, 3)))) - assert "image" in s and s.role_of("image") == "input" - - def test_drop(self) -> None: - s = _sample().drop("class") - assert "class" not in s and list(s.keys()) == ["image", "regions"] - - -class TestMappingProtocol: - def test_len_iter_contains_getitem(self) -> None: - s = _sample() - assert len(s) == 3 and "image" in s and list(iter(s)) == ["image", "regions", "class"] - assert isinstance(s["image"], Image) - - def test_fields_and_roles_are_copies(self) -> None: - s = _sample() - s.fields["image"] = None - s.roles["image"] = "target" - assert isinstance(s["image"], Image) and s.role_of("image") == "input" - - -class TestEquality: - def test_equal_with_array_fields(self) -> None: - a = Sample({"img": Image(np.zeros((2, 2, 3)))}) - b = Sample({"img": Image(np.zeros((2, 2, 3)))}) - assert a == b - - def test_unequal_arrays(self) -> None: - a = Sample({"img": Image(np.zeros((2, 2, 3)))}) - b = Sample({"img": Image(np.ones((2, 2, 3)))}) - assert a != b - - def test_unequal_roles_or_keys(self) -> None: - a = Sample({"x": Label("v")}) - assert a != Sample({"x": Label("v")}, roles={"x": "target"}) - assert a != Sample({"y": Label("v")}) - - def test_not_a_sample(self) -> None: - assert (Sample({"x": Label()}) == 5) is False - - def test_repr(self) -> None: - assert "Image[input]" in repr(_sample()) diff --git a/tests/test_bag_transform.py b/tests/test_bag_transform.py deleted file mode 100644 index 819a8a7..0000000 --- a/tests/test_bag_transform.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Transforms — type dispatch, once-per-sample params, cross-field consistency, only filter. - -Native-kernel machinery is pinned via the test fixture ``FixtureFlip`` (sampleflux ships no -native augmentation transforms — libraries cover that through adapter coercion). -""" - -import numpy as np -import pytest - -from sampleflux import Image, Label, Mask, Pipeline, Regions, Sample, Transform, as_transform -from tests._bag_fixtures import FixtureFlip - - -def _seg() -> Sample: - return Sample( - { - "image": Image(np.arange(8 * 10 * 3).reshape(8, 10, 3).astype(np.float32)), - "mask": Mask(np.arange(8 * 10).reshape(8, 10)), - "regions": Regions(boxes=[[1, 1, 4, 4]], labels=["a"], canvas=(8, 10)), - "class": Label("a"), - } - ) - - -class TestKernelDispatchMachinery: - def test_cross_field_consistency(self) -> None: - out = FixtureFlip(p=1.0)(_seg()) - seg = _seg() - assert np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])[:, ::-1]) - assert np.array_equal(np.asarray(out["mask"]), np.asarray(seg["mask"])[:, ::-1]) - assert out["regions"].boxes == [[6, 1, 9, 4]] # W=10: x -> W-x - assert out["class"].value == "a" # no handler — untouched - - def test_p_zero_is_identity(self) -> None: - out = FixtureFlip(p=0.0)(_seg()) - assert np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) - assert out["regions"].boxes == [[1, 1, 4, 4]] - - def test_only_filter(self) -> None: - out = FixtureFlip(p=1.0, only=["image"])(_seg()) - assert not np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) - assert np.array_equal(np.asarray(out["mask"]), np.asarray(_seg()["mask"])) # mask skipped - assert out["regions"].boxes == [[1, 1, 4, 4]] # regions skipped - - def test_image_layout_chw(self) -> None: - s = Sample({"image": Image(np.arange(3 * 4 * 5).reshape(3, 4, 5), layout="CHW")}) - out = FixtureFlip(p=1.0)(s) - assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, :, ::-1]) - - def test_regions_uses_canvas_without_image(self) -> None: - s = Sample({"regions": Regions(boxes=[[2, 0, 5, 3]], canvas=(8, 10))}) - assert FixtureFlip(p=1.0)(s)["regions"].boxes == [[5, 0, 8, 3]] - - def test_regions_without_reference_width_raises(self) -> None: - s = Sample({"regions": Regions(boxes=[[2, 0, 5, 3]])}) # no image, no canvas - with pytest.raises(ValueError, match="no reference width"): - FixtureFlip(p=1.0)(s) - - def test_params_sampled_once(self) -> None: - # A partial-probability flip must be all-or-nothing across fields (shared decision), - # never per-field independent draws. - seg = _seg() - for _ in range(25): - out = FixtureFlip(p=0.5)(seg) - image_flipped = not np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])) - regions_flipped = out["regions"].boxes != seg["regions"].boxes - assert image_flipped == regions_flipped - - -class TestAdapterParity: - def test_v2_flip_matches_native_fixture(self) -> None: - # The bare-library path must move image/mask/boxes EXACTLY like the native fixture — - # this is the guarantee that let sampleflux drop its native flip for the library one. - v2 = pytest.importorskip("torchvision.transforms.v2") - seg = _seg() - native = FixtureFlip(p=1.0)(seg) - adapted = Pipeline([v2.RandomHorizontalFlip(p=1.0)])(seg) - assert np.array_equal(np.asarray(adapted["image"]), np.asarray(native["image"])) - assert np.array_equal(np.asarray(adapted["mask"]), np.asarray(native["mask"])) - assert adapted["regions"].boxes[0] == pytest.approx(native["regions"].boxes[0], abs=1e-4) - assert adapted["class"].value == native["class"].value == "a" - - -class TestPipelineAndFunction: - def test_pipeline_is_sequential(self) -> None: - s = Sample({"x": Image(np.ones((2, 2, 3), dtype=np.float32))}) - double = as_transform(lambda d: d * 2, handles=(Image,)) - out = Pipeline([double, double])(s) - assert np.allclose(np.asarray(out["x"]), 4.0) - - def test_function_transform_only_filter(self) -> None: - s = Sample({"a": Image(np.ones((2, 2, 3))), "b": Image(np.ones((2, 2, 3)))}) - out = as_transform(lambda d: d + 1, handles=(Image,), only=["a"])(s) - assert np.allclose(np.asarray(out["a"]), 2.0) and np.allclose(np.asarray(out["b"]), 1.0) - - def test_pipeline_repr(self) -> None: - assert "FixtureFlip" in repr(Pipeline([FixtureFlip()])) - - -class TestBaseTransform: - def test_default_get_params_and_passthrough(self) -> None: - # A transform with no kernels leaves every field alone. - s = Sample({"x": Label("v")}) - assert Transform()(s) == s - - def test_decode_not_implemented(self) -> None: - with pytest.raises(NotImplementedError, match="no decode"): - FixtureFlip().decode(Sample({"image": Image(np.zeros((2, 2, 3)))})) - - def test_zero_arg_construction(self) -> None: - assert FixtureFlip().p == 0.5 and Transform().only is None diff --git a/tests/test_categories.py b/tests/test_categories.py index 0ee26c6..ecb6683 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -9,17 +9,19 @@ from confluid.registry import get_registry +from sampleflux import Pipeline from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.ops.albumentations import AlbumentationsOp from sampleflux.ops.configure import ConfigureOp +from sampleflux.ops.context import Apply, Capture, Drop, MergeFields, Save, Use from sampleflux.ops.debug import PrintSampleOp from sampleflux.ops.enable import Enable from sampleflux.ops.formula import FormulaOp from sampleflux.ops.image import ConvertToImage from sampleflux.ops.numpy import ConnectedComponents, Threshold from sampleflux.ops.parallel import Parallel +from sampleflux.ops.random_apply import RandomApply from sampleflux.ops.sink import SampleSinkOp -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields from sampleflux.ops.target import ( CocoToTorchVisionDetection, DecodeTarget, @@ -28,8 +30,6 @@ MetadataToTarget, ) from sampleflux.ops.torch import ToTensor -from sampleflux.ops.torchvision import TorchvisionTransformOp -from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource from sampleflux.storage.directory import DirectorySink from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source @@ -62,8 +62,9 @@ def test_op_classes_tagged() -> None: ToTensor, ConvertToImage, Enable, - TransformChain, + Pipeline, Parallel, + RandomApply, SampleSinkOp, MetadataToTarget, EncodeTarget, @@ -72,21 +73,23 @@ def test_op_classes_tagged() -> None: MasksToDetectionBoxes, ConfigureOp, FormulaOp, - AlbumentationsOp, - TorchvisionTransformOp, - SetRole, RenameField, DropField, CopyField, SelectFields, + Save, + Use, + Drop, + Apply, + Capture, + MergeFields, PrintSampleOp, ): assert cls.__confluid_category__ == "op", cls.__name__ -def test_augmentation_adapters_random_tagged() -> None: - assert AlbumentationsOp.__confluid_random__ is True - assert TorchvisionTransformOp.__confluid_random__ is True +def test_random_apply_random_tagged() -> None: + assert RandomApply.__confluid_random__ is True def test_storage_sink_classes_tagged() -> None: @@ -102,7 +105,6 @@ def test_op_group_tags() -> None: assert ConnectedComponents.__confluid_group__ == "numpy" assert ToTensor.__confluid_group__ == "torch" assert ConvertToImage.__confluid_group__ == "image" - assert SetRole.__confluid_group__ == "structure" assert SelectFields.__confluid_group__ == "structure" assert PrintSampleOp.__confluid_group__ == "debug" assert MetadataToTarget.__confluid_group__ == "structure" @@ -110,14 +112,15 @@ def test_op_group_tags() -> None: assert DecodeTarget.__confluid_group__ == "structure" assert CocoToTorchVisionDetection.__confluid_group__ == "structure" assert MasksToDetectionBoxes.__confluid_group__ == "structure" + for ctx_op in (Save, Use, Drop, Apply, Capture, MergeFields): + assert ctx_op.__confluid_group__ == "structure", ctx_op.__name__ assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" - assert TransformChain.__confluid_group__ == "compose" + assert Pipeline.__confluid_group__ == "compose" + assert RandomApply.__confluid_group__ == "compose" assert ConfigureOp.__confluid_group__ == "compose" assert FormulaOp.__confluid_group__ == "compose" assert SampleSinkOp.__confluid_group__ == "sink" - assert AlbumentationsOp.__confluid_group__ == "augment" - assert TorchvisionTransformOp.__confluid_group__ == "augment" def test_categories_enumerable_via_registry() -> None: @@ -134,13 +137,13 @@ def test_categories_enumerable_via_registry() -> None: "ToTensor", "ConvertToImage", "Enable", + "Pipeline", "SampleSinkOp", "MetadataToTarget", "EncodeTarget", "DecodeTarget", "CocoToTorchVisionDetection", "MasksToDetectionBoxes", - "TransformChain", } <= registry.list_classes(category="op") assert {"HDF5Sink", "ZarrGroupSink", "ZarrBatchSink", "DirectorySink"} <= registry.list_classes(category="sink") assert "SampleSinkOp" not in registry.list_classes(category="sink") @@ -151,7 +154,9 @@ def test_groups_enumerable_via_registry() -> None: assert {"Threshold", "ConnectedComponents"} <= registry.list_classes(group="numpy") assert {"ToTensor"} <= registry.list_classes(group="torch") assert {"ConvertToImage"} <= registry.list_classes(group="image") - assert {"Parallel", "Enable", "TransformChain"} <= registry.list_classes(group="compose") + assert {"Parallel", "Enable", "Pipeline", "RandomApply", "ConfigureOp", "FormulaOp"} <= registry.list_classes( + group="compose" + ) assert {"SampleSinkOp"} <= registry.list_classes(group="sink") assert { "MetadataToTarget", @@ -159,8 +164,9 @@ def test_groups_enumerable_via_registry() -> None: "DecodeTarget", "CocoToTorchVisionDetection", "MasksToDetectionBoxes", - "SetRole", "SelectFields", + "Save", + "Use", + "MergeFields", } <= registry.list_classes(group="structure") - assert {"AlbumentationsOp", "TorchvisionTransformOp"} <= registry.list_classes(group="augment") - assert "TransformChain" in registry.list_classes(category="op", group="compose") + assert "Pipeline" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_bag_dispatch.py b/tests/test_dispatch.py similarity index 82% rename from tests/test_bag_dispatch.py rename to tests/test_dispatch.py index 38ef288..2eea385 100644 --- a/tests/test_bag_dispatch.py +++ b/tests/test_dispatch.py @@ -3,8 +3,8 @@ from typing import Any, Dict from sampleflux import Image, Label, Mask, Regions, Transform -from sampleflux.bag.dispatch import dispatch, get_kernel, register_kernel, registered_kernels -from tests._bag_fixtures import FixtureFlip +from sampleflux.dispatch import dispatch, get_kernel, register_kernel, registered_kernels +from tests._fixtures import FixtureFlip class TestDispatch: @@ -13,7 +13,7 @@ def test_exact_hit(self) -> None: assert dispatch(FixtureFlip, Image) is get_kernel(FixtureFlip, Image) def test_miss_returns_none(self) -> None: - # FixtureFlip has no Label kernel — a Label field passes through. + # FixtureFlip has no Label kernel — a Label value passes through. assert dispatch(FixtureFlip, Label) is None assert get_kernel(FixtureFlip, Label) is None @@ -44,6 +44,10 @@ def _kernel(item: Any, params: Dict[str, Any]) -> Any: assert dispatch(T, Image) is _kernel # cache was cleared on registration + def test_plain_value_type_misses(self) -> None: + # A plain (non-item) value type — e.g. float — has no kernel: the op passes it through. + assert dispatch(FixtureFlip, float) is None + def test_registered_kernels_lists_pairs(self) -> None: pairs = registered_kernels() assert ("FixtureFlip", "Image") in pairs diff --git a/tests/test_bag_io.py b/tests/test_io.py similarity index 58% rename from tests/test_bag_io.py rename to tests/test_io.py index b7d9e34..f1a0c52 100644 --- a/tests/test_bag_io.py +++ b/tests/test_io.py @@ -1,4 +1,5 @@ -"""The item codec registry (``sampleflux.bag.io``) — default structural codec, overrides, samples.""" +"""The item codec registry (``sampleflux.io``) — default structural codec, overrides, records, +the ``"plain"`` codec path.""" from dataclasses import dataclass @@ -10,14 +11,14 @@ Image, Label, Regions, - Sample, decode_item, - decode_sample, + decode_record, encode_item, - encode_sample, + encode_record, register_io, register_item, ) +from sampleflux.io import PLAIN_TYPE @register_item @@ -58,6 +59,25 @@ def test_unknown_type_name_raises(self) -> None: decode_item(EncodedItem(type_name="Nope", payload=None, attrs={})) +class TestPlainCodec: + def test_scalar_round_trips_verbatim(self) -> None: + for value in (-3.0, 7, "text", True): + enc = encode_item(value) + assert enc.type_name == PLAIN_TYPE and enc.payload == value and enc.attrs == {} + assert decode_item(enc) == value + + def test_bare_array_round_trips_verbatim(self) -> None: + arr = np.arange(6).reshape(2, 3) # a bare ndarray is NOT a registered item -> "plain" + enc = encode_item(arr) + assert enc.type_name == PLAIN_TYPE + back = decode_item(enc) + assert type(back) is np.ndarray and np.array_equal(back, arr) + + def test_none_is_plain(self) -> None: + enc = encode_item(None) + assert enc.type_name == PLAIN_TYPE and decode_item(enc) is None + + class TestRegisteredCodec: def test_override_wins_and_round_trips(self) -> None: @register_item @@ -76,18 +96,20 @@ def __init__(self, values: list) -> None: assert isinstance(back, Compact) and back.values == [1, 2, 3] -class TestSampleCodec: - def test_sample_round_trip_fields_roles_order(self) -> None: - s = Sample( - { - "image": Image(np.zeros((2, 2, 3), dtype=np.float32)), - "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), - "class": Label("x"), - }, - roles={"regions": "target", "class": "target"}, - ) - fields = encode_sample(s) - assert [f.key for f in fields] == ["image", "regions", "class"] - assert [f.role for f in fields] == ["input", "target", "target"] - back = decode_sample(fields) - assert back == s +class TestRecordCodec: + def test_record_round_trip_keys_order_and_plain_entries(self) -> None: + record = { + "image": Image(np.zeros((2, 2, 3), dtype=np.float32)), + "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), + "class": Label("x"), + "gain_db": -3.0, # a plain scalar rides the same layout under the "plain" tag + } + fields = encode_record(record) + assert [f.key for f in fields] == ["image", "regions", "class", "gain_db"] + assert fields[3].item.type_name == PLAIN_TYPE + back = decode_record(fields) + assert list(back.keys()) == list(record.keys()) + assert np.array_equal(np.asarray(back["image"]), np.asarray(record["image"])) + assert back["regions"] == record["regions"] + assert back["class"] == record["class"] + assert back["gain_db"] == -3.0 diff --git a/tests/test_bag_items.py b/tests/test_items.py similarity index 92% rename from tests/test_bag_items.py rename to tests/test_items.py index 58a7fa5..e18f282 100644 --- a/tests/test_bag_items.py +++ b/tests/test_items.py @@ -1,9 +1,9 @@ """Typed items — array-subclass attribute preservation, wrappers, payload accessors, registry. Only the MODALITY-NEUTRAL core items live in sampleflux (Image / Mask / Regions / Label). The -data-bearing-wrapper and multi-attribute-array paths (which the signal-domain items in -``waivefront.bag`` exercise for real) are covered here with small test-local item types, so the -core stays tested without importing a domain package. +data-bearing-wrapper and multi-attribute-array paths (which the signal-domain items in a domain +package exercise for real) are covered here with small test-local item types, so the core stays +tested without importing a domain package. """ from dataclasses import dataclass @@ -11,7 +11,7 @@ import numpy as np import pytest -from sampleflux.bag.items import ( +from sampleflux.items import ( Image, Label, Mask, @@ -94,6 +94,9 @@ def test_item_data_no_payload_returns_self(self) -> None: reg = Regions(boxes=[[0, 0, 1, 1]]) assert item_data(reg) is reg # no `.data` slot — returns the item + def test_item_data_plain_value_passes_through(self) -> None: + assert item_data(3.5) == 3.5 and item_data("s") == "s" # non-items pass through verbatim + def test_with_data_array_preserves_attrs(self) -> None: img = Image(np.zeros((2, 2, 3)), layout="CHW") rebuilt = with_data(img, np.ones((2, 2, 3))) diff --git a/tests/test_labels.py b/tests/test_labels.py index e6c3274..cc08cdf 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -4,7 +4,7 @@ import pytest -from sampleflux import Label, Sample +from sampleflux import Label from sampleflux.labels import LabelMap from sampleflux.ops.target import DecodeTarget, EncodeTarget @@ -86,7 +86,7 @@ def test_encode_op_encodes_target() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) op = lm.encode_op() assert isinstance(op, EncodeTarget) - out = op(Sample({"y": Label("dog")}, roles={"y": "target"})) + out = op({"y": Label("dog")}) assert out["y"].value == 1 @@ -94,14 +94,14 @@ def test_decode_op_inverts_encoding() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) op = lm.decode_op() assert isinstance(op, DecodeTarget) - out = op(Sample({"y": Label(0)}, roles={"y": "target"})) + out = op({"y": Label(0)}) assert out["y"].value == "cat" def test_encode_op_ignore_unknown() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) op = lm.encode_op(ignore_unknown=True, default=-1) - out = op(Sample({"y": Label("fish")}, roles={"y": "target"})) + out = op({"y": Label("fish")}) assert out["y"].value == -1 diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index 18dc48b..d32e163 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -1,9 +1,9 @@ """Guard: every node-facing sampleflux Source/Op documents all its constructor params. -These classes surface in FluxStudio (as widget tooltips) and navigaitor (as -pydantic ``Field(description=...)`` in the form-spec) purely from their docstring -``Args:`` block — see ``confluid.parse_param_docs``. A param that loses its doc -silently loses its tooltip/description, so this pins the coverage. +These classes surface in visual editors (as widget tooltips) and MCP form-specs (as +pydantic ``Field(description=...)``) purely from their docstring ``Args:`` block — see +``confluid.parse_param_docs``. A param that loses its doc silently loses its +tooltip/description, so this pins the coverage. """ import inspect @@ -12,12 +12,18 @@ import pytest from confluid import parse_param_docs # type: ignore[import-not-found] +from sampleflux import Pipeline, Transform from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.ops.albumentations import AlbumentationsOp from sampleflux.ops.configure import ConfigureOp +from sampleflux.ops.context import Apply, Capture, Drop, MergeFields, Save, Use +from sampleflux.ops.debug import PrintSampleOp +from sampleflux.ops.enable import Enable +from sampleflux.ops.formula import FormulaOp from sampleflux.ops.image import ConvertToImage from sampleflux.ops.numpy import ConnectedComponents, Threshold -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole +from sampleflux.ops.parallel import Parallel +from sampleflux.ops.random_apply import RandomApply +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields from sampleflux.ops.target import ( CocoToTorchVisionDetection, DecodeTarget, @@ -26,8 +32,6 @@ MetadataToTarget, ) from sampleflux.ops.torch import ToTensor -from sampleflux.ops.torchvision import TorchvisionTransformOp -from sampleflux.ops.transform_chain import TransformChain from sampleflux.sources import HuggingFaceSource _NODE_CLASSES = [ @@ -36,6 +40,8 @@ JointFlux, FilterOp, WrappedOp, + Transform, + Pipeline, Threshold, ConnectedComponents, ConvertToImage, @@ -45,15 +51,22 @@ DecodeTarget, CocoToTorchVisionDetection, MasksToDetectionBoxes, - SetRole, RenameField, DropField, CopyField, SelectFields, + Save, + Use, + Drop, + Apply, + Capture, + MergeFields, ConfigureOp, - TransformChain, - AlbumentationsOp, - TorchvisionTransformOp, + FormulaOp, + Enable, + Parallel, + RandomApply, + PrintSampleOp, ] diff --git a/tests/test_op_families.py b/tests/test_op_families.py new file mode 100644 index 0000000..dc81589 --- /dev/null +++ b/tests/test_op_families.py @@ -0,0 +1,271 @@ +"""The engine's op-FAMILY dispatch (``core._apply_op``) — libraries run AS-IS, end to end. + +Pins the record-model headline: native type-dispatched ops, BARE albumentations transforms +(kwarg-vocabulary call, one joint draw, item re-wrap), and BARE torchvision ``transforms.v2`` +transforms (dict call) all sit in ONE ``Flux.ops`` list with no wrapper/adapter classes — +plus the family classifiers, YAML mapping-form ops docs, spawn-parallel with a bare library +op, ``field=`` targeting, and the ``WrappedOp``/``FilterOp`` raw-callable routes. +""" + +from pathlib import Path +from typing import Dict, List, Optional + +import albumentations as A +import numpy as np +import pytest +import torch +from confluid import configurable +from torchvision.transforms import v2 + +from sampleflux import FilterOp, Image, Label, Mask, Pipeline, Record, Transform, WrappedOp +from sampleflux.core import Flux, _apply_op, _is_albumentations, _is_torchvision_v2 + + +# --------------------------------------------------------------------------- # +# Module-level fixtures (spawn workers pickle records, ops, and callables). +# --------------------------------------------------------------------------- # +def _base_record(i: int = 0) -> Record: + rng = np.random.default_rng(i) + return { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), + "mask": Mask((rng.random((16, 20)) > 0.5).astype(np.uint8)), + "class": Label("drone_x", classes=["noise", "drone_x"]), + "gain_db": -3.0, + } + + +def spawn_records() -> List[Record]: + """Module-level source fixture so the spawn-parallel test's records pickle.""" + return [_base_record(i) for i in range(4)] + + +def keep_even_gain(record: Record) -> bool: + """Module-level FilterOp predicate (pickles across spawn workers).""" + return int(record["idx"]) % 2 == 0 + + +@configurable(category="op") +class AddOffset(Transform): + """Adds a fixed offset to every Image value (Awgn-style configurable native op). + + Args: + offset: The value added to every Image payload. + field: Apply only to this record key. None (default) = every Image value. + """ + + handles = (Image,) + + def __init__(self, offset: float = 0.0, field: Optional[str] = None) -> None: + super().__init__(field=field) + self.offset = float(offset) + + def get_params(self, record: Record) -> Dict[str, float]: + return {"offset": self.offset} + + +@AddOffset.kernel(Image) +def _add_offset_image(value: Image, params: Dict[str, float]) -> Image: + return Image(np.asarray(value) + params["offset"], layout=value.layout) + + +# --------------------------------------------------------------------------- # +# Family classifiers. +# --------------------------------------------------------------------------- # +class TestFamilyClassifiers: + def test_is_albumentations_positive(self) -> None: + assert _is_albumentations(A.HorizontalFlip(p=1.0)) + assert _is_albumentations(A.Compose([A.HorizontalFlip(p=1.0)])) + + def test_is_albumentations_negative(self) -> None: + assert not _is_albumentations(v2.RandomCrop(4)) + assert not _is_albumentations(AddOffset()) + assert not _is_albumentations({"image": None}) + assert not _is_albumentations(lambda r: r) + + def test_is_torchvision_v2_positive(self) -> None: + assert _is_torchvision_v2(v2.RandomCrop(4)) + assert _is_torchvision_v2(v2.ToImage()) + + def test_is_torchvision_v2_negative(self) -> None: + assert not _is_torchvision_v2(A.HorizontalFlip(p=1.0)) + assert not _is_torchvision_v2(AddOffset()) + assert not _is_torchvision_v2(object()) + + +# --------------------------------------------------------------------------- # +# Albumentations family. +# --------------------------------------------------------------------------- # +class TestAlbumentationsFamily: + def test_joint_move_under_one_bare_compose_with_bboxes(self) -> None: + # Box-carrying augmentation is albumentations' own Compose(bbox_params=...) dropped in + # BARE — ONE joint draw moves image + mask + bboxes together; item types survive. + record = {**_base_record(), "bboxes": [[2, 3, 6, 7]], "labels": ["drone"]} + flip = A.Compose( + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ) + out = _apply_op(record, flip) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(record["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(record["mask"])[:, ::-1]) + assert [round(v) for v in out["bboxes"][0]] == [14, 3, 18, 7] # W=20: x -> W-x + assert isinstance(out["image"], Image) and isinstance(out["mask"], Mask) # re-wrapped + assert out["class"].value == "drone_x" and out["gain_db"] == -3.0 # non-alb keys untouched + + def test_zero_known_keys_is_passthrough(self) -> None: + # A record with NO albumentations-vocabulary keys passes through unchanged. + record = {"spec": Mask(np.zeros((4, 4))), "gain_db": 1.0} + out = _apply_op(record, A.HorizontalFlip(p=1.0)) + assert out is record + + def test_extra_record_entries_never_reach_the_library(self) -> None: + # Scalars / Labels are not in _ALB_KEYS — the op receives only image/mask and the + # extras ride through verbatim (same objects). + record = _base_record() + out = _apply_op(record, A.HorizontalFlip(p=1.0)) + assert out is not None + assert out["class"] is record["class"] and out["gain_db"] == -3.0 + + +# --------------------------------------------------------------------------- # +# Mixed ops list end-to-end through Flux. +# --------------------------------------------------------------------------- # +class TestMixedOpsList: + _OPS = [ + AddOffset(offset=0.5), # native type-dispatched op + A.HorizontalFlip(p=1.0), # bare albumentations + v2.ToImage(), # bare torchvision v2: explicit numpy HWC -> CHW tv_tensor conversion + v2.RandomCrop(4), # bare torchvision v2 + ] + + def test_iteration(self) -> None: + flux = Flux(source=[_base_record()], ops=list(self._OPS)) + (out,) = list(flux) + assert isinstance(out["image"], torch.Tensor) + assert tuple(out["image"].shape) == (3, 4, 4) + assert out["class"].value == "drone_x" # rode through every family untouched + + def test_random_access(self) -> None: + flux = Flux(source=[_base_record(0), _base_record(1)], ops=list(self._OPS)) + out = flux[1] + assert isinstance(out["image"], torch.Tensor) and tuple(out["image"].shape) == (3, 4, 4) + + def test_pipeline_nests_the_same_families(self) -> None: + # The same mixed list nested inside Pipeline (which routes through _apply_op). + out = Pipeline(list(self._OPS))(_base_record()) + assert out is not None and tuple(out["image"].shape) == (3, 4, 4) + + +# --------------------------------------------------------------------------- # +# YAML ops docs — mapping-form bare library entries + configurable ctor binding. +# --------------------------------------------------------------------------- # +class TestOpsYaml: + def test_mapping_form_bare_albumentations_entry(self, tmp_path: Path) -> None: + path = tmp_path / "ops.yaml" + path.write_text("ops:\n - !class:albumentations.HorizontalFlip {p: 1.0}\n") + record = _base_record() + flux = Flux.from_ops_yaml(str(path), source=[record]) + (out,) = list(flux) + assert np.array_equal(np.asarray(out["image"]), np.asarray(record["image"])[:, ::-1]) + assert isinstance(out["image"], Image) + + def test_configurable_ctor_param_bound_from_yaml(self, tmp_path: Path) -> None: + # An Awgn-style native op with a ctor param set in the YAML doc: the value reaches + # the constructor and the op output reflects it. + path = tmp_path / "ops.yaml" + path.write_text("ops:\n - !class:tests.test_op_families.AddOffset {offset: 3.0}\n") + record = {"image": Image(np.zeros((2, 3, 3), dtype=np.float32))} + flux = Flux.from_ops_yaml(str(path), source=[record]) + (out,) = list(flux) # a @configurable entry stays a deferred marker until route entry + assert np.allclose(np.asarray(out["image"]), 3.0) + (op,) = flux.ops # _check_ops_materialized flowed + cached the live op in place + assert isinstance(op, AddOffset) and op.offset == 3.0 + + +# --------------------------------------------------------------------------- # +# Spawn-parallel with a bare albumentations op (+ FilterOp drop on the parallel route). +# --------------------------------------------------------------------------- # +def test_spawn_parallel_with_bare_albumentations_op() -> None: + records = [{**r, "idx": i} for i, r in enumerate(spawn_records())] + flux = Flux(source=records, ops=[A.HorizontalFlip(p=1.0), FilterOp(keep_even_gain)]).parallel(2) + results = flux.collect() + assert [int(r["idx"]) for r in results] == [0, 2] # FilterOp dropped odd records in workers + for out, want in zip(results, [records[0], records[2]]): + assert isinstance(out["image"], Image) # item type survived pickle + re-wrap + assert np.array_equal(np.asarray(out["image"]), np.asarray(want["image"])[:, ::-1]) + + +# --------------------------------------------------------------------------- # +# field= targeting. +# --------------------------------------------------------------------------- # +def test_field_targets_one_of_two_image_keys() -> None: + record = { + "a": Image(np.zeros((2, 2, 3), dtype=np.float32)), + "b": Image(np.zeros((2, 2, 3), dtype=np.float32)), + } + out = AddOffset(offset=1.0, field="a")(record) + assert out is not None + assert np.allclose(np.asarray(out["a"]), 1.0) # pinned key moved + assert np.allclose(np.asarray(out["b"]), 0.0) # sibling untouched + + +# --------------------------------------------------------------------------- # +# Raw-callable routes: WrappedOp / Flux.map / FilterOp drops everywhere. +# --------------------------------------------------------------------------- # +def double(x: np.ndarray) -> np.ndarray: + """Module-level payload function for WrappedOp (stored as an importable path).""" + return x * 2 + + +def bump_gain(record: Record) -> Record: + """Module-level whole-record function for WrappedOp(key=None).""" + return {**record, "gain_db": record["gain_db"] + 1.0} + + +class TestWrappedOpAndMap: + def test_key_targets_payload_and_preserves_item(self) -> None: + record = {"m": Mask(np.ones((2, 2))), "gain_db": 0.0} + out = WrappedOp(double, key="m")(record) + assert out is not None + assert isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 2.0) + assert record["m"] is not out["m"] # copy-on-write + + def test_key_on_plain_value_replaces_verbatim(self) -> None: + out = WrappedOp(double, key="g")({"g": 3}) + assert out is not None and out["g"] == 6 # plain value: no item to re-wrap + + def test_key_none_receives_whole_record(self) -> None: + out = WrappedOp(bump_gain)({"gain_db": -3.0}) + assert out is not None and out["gain_db"] == -2.0 + + def test_missing_key_raises(self) -> None: + with pytest.raises(KeyError, match="nope"): + WrappedOp(double, key="nope")({"m": Mask(np.ones(2))}) + + def test_flux_map_key(self) -> None: + flux = Flux(source=[{"m": Mask(np.ones((2, 2)))}]).map(double, key="m") + (out,) = list(flux) + assert isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 2.0) + + +class TestFilterDropRoutes: + def test_sequential_iteration_drops(self) -> None: + records = [{"i": 0}, {"i": 1}, {"i": 2}] + flux = Flux(source=records, ops=[FilterOp(lambda r: r["i"] != 1)]) + assert [r["i"] for r in flux] == [0, 2] + + def test_getitem_on_filtered_record_raises_index_error(self) -> None: + flux = Flux(source=[{"i": 0}], ops=[FilterOp(lambda r: False)]) + with pytest.raises(IndexError, match="filtered out"): + flux[0] + + def test_pipeline_propagates_drop(self) -> None: + assert Pipeline([FilterOp(lambda r: False)])({"i": 0}) is None + + def test_flux_filter_helper(self) -> None: + flux = Flux(source=[{"i": 0}, {"i": 1}]).filter(lambda r: r["i"] > 0) + assert [r["i"] for r in flux] == [1] + + def test_unset_predicate_raises_lazily(self) -> None: + with pytest.raises(ValueError, match="predicate"): + FilterOp()({"i": 0}) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 7b72113..8713322 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -2,32 +2,27 @@ import numpy as np -from sampleflux import Image, Sample -from sampleflux.bag.items import item_data +from sampleflux import Image, item_data from sampleflux.core import Flux def heavy_op(x: np.ndarray) -> np.ndarray: - time.sleep(0.1) + time.sleep(0.1) # simulated per-record WORKLOAD (not a synchronization wait) return x * 2 def test_parallel_execution() -> None: - source = [Sample({"x": Image(np.array([i]))}, roles={"x": "input"}) for i in range(10)] + source = [{"x": Image(np.array([i]))} for i in range(10)] start = time.time() - # Use a real top-level function for pickling - pipeline = Flux(source).map(heavy_op).parallel(workers=4) + # Use a real top-level function for pickling; key= targets the record entry's payload. + pipeline = Flux(source).map(heavy_op, key="x").parallel(workers=4) results = pipeline.collect() duration = time.time() - start assert len(results) == 10 + assert isinstance(results[0]["x"], Image) # item type survives the spawn round-trip assert int(item_data(results[0]["x"])[0]) == 0 assert int(item_data(results[9]["x"])[0]) == 18 # We only assert the pipeline completes — this isn't a benchmark. assert duration < 15.0 - - -def test_parallel_with_joint() -> None: - # Already tested elsewhere; helps coverage here too. - pass diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..73ed047 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,124 @@ +"""``Pipeline`` — the compose-group unit: identity, None-propagation, close(), Fluid entries, +bare library transforms, import safety.""" + +import subprocess +import sys +from pathlib import Path + +import confluid +import numpy as np +import pytest + +from sampleflux import FilterOp, Image, Label, Mask, Pipeline, Record +from sampleflux.ops.structure import RenameField +from tests._fixtures import FixtureFlip + + +def _rec() -> Record: + return {"image": Image(np.ones((4, 6, 3), dtype=np.float32)), "class": Label("x")} + + +class TestImportSafety: + def test_package_imports_without_torchvision(self) -> None: + # The top-level package must import without torchvision/albumentations (the engine's + # op-family dispatch detects them by MRO module NAME — no import), so discovery stays + # safe on hosts missing the libraries. + code = ( + "import sys; import sampleflux; " + "assert 'torchvision' not in sys.modules; " + "assert 'albumentations' not in sys.modules" + ) + subprocess.run([sys.executable, "-c", code], check=True, cwd=str(Path(__file__).resolve().parents[1])) + + +class TestPipelineSemantics: + def test_zero_arg_is_identity(self) -> None: + rec = _rec() + out = Pipeline()(rec) + assert out is rec # no ops -> the record passes through untouched + + def test_none_propagation_mid_chain(self) -> None: + # A filter-drop mid-chain stops the pipeline and propagates None; later ops never run. + ran = [] + + def probe(record: Record) -> Record: + ran.append(True) + return record + + p = Pipeline([FilterOp(lambda r: False), probe]) + assert p(_rec()) is None + assert ran == [] # the op after the drop never fired + + def test_close_propagates_to_inner_ops(self) -> None: + closed = [] + + class _Closeable: + def __call__(self, record: Record) -> Record: + return record + + def close(self) -> None: + closed.append(True) + + p = Pipeline([_Closeable(), FilterOp(lambda r: True)]) # FilterOp has no close -> skipped + p.close() + assert closed == [True] + + def test_fluid_entry_flowed_and_cached(self) -> None: + # A config-deferred entry (confluid Class marker) is flowed on first call and the + # live op is cached back into the transforms list. + p = Pipeline(transforms=[confluid.Class(RenameField, src="class", dst="klass")]) + out = p(_rec()) + assert out is not None and "klass" in out and "class" not in out + assert isinstance(p.transforms[0], RenameField) # cached in place + out2 = p(_rec()) # second call uses the cached live op + assert out2 is not None and "klass" in out2 + + def test_native_ops_chain(self) -> None: + s = {"image": Image(np.arange(4 * 6 * 3).reshape(4, 6, 3).astype(np.float32))} + out = Pipeline([FixtureFlip(p=1.0), FixtureFlip(p=1.0)])(s) + assert out is not None + # Two flips cancel out. + assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])) + + +class TestBareLibraryEntries: + def test_bare_albumentations_entry(self) -> None: + import albumentations as A + + rec = { + "image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32)), + "mask": Mask(np.arange(6 * 8).reshape(6, 8).astype(np.uint8)), + "class": Label("x"), + } + out = Pipeline([A.HorizontalFlip(p=1.0)])(rec) + assert out is not None + # ONE joint draw moved image and mask together; item types + metadata survive. + assert isinstance(out["image"], Image) and out["image"].layout == "HWC" + assert isinstance(out["mask"], Mask) + assert np.array_equal(np.asarray(out["image"]), np.asarray(rec["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(rec["mask"])[:, ::-1]) + assert out["class"].value == "x" # not an albumentations key — never reached the library + + def test_bare_torchvision_v2_entries(self) -> None: + import torch + + v2 = pytest.importorskip("torchvision.transforms.v2") + + # The EXPLICIT v2.ToImage() conversion first (numpy HWC -> CHW tv_tensor), then any v2 + # transform — the engine passes the dict straight through (never converts silently). + rec = {"image": np.arange(4 * 6 * 3).reshape(4, 6, 3).astype(np.float32)} + converted = Pipeline([v2.ToImage()])(rec) + assert converted is not None + assert isinstance(converted["image"], torch.Tensor) and tuple(converted["image"].shape) == (3, 4, 6) + cropped = Pipeline([v2.RandomCrop(2)])(converted) + assert cropped is not None and tuple(cropped["image"].shape) == (3, 2, 2) + + def test_mixed_native_and_bare_library(self) -> None: + import albumentations as A + + rec = {"image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32))} + # bare albumentations flip + native fixture flip = identity (both moved the image once). + out = Pipeline([A.HorizontalFlip(p=1.0), FixtureFlip(p=1.0)])(rec) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(rec["image"])) + assert isinstance(out["image"], Image) diff --git a/tests/test_structure_ops.py b/tests/test_structure_ops.py index e173d3b..e261894 100644 --- a/tests/test_structure_ops.py +++ b/tests/test_structure_ops.py @@ -1,115 +1,88 @@ -"""Typed structure ops — SetRole/RenameField/DropField/CopyField/SelectFields + primary/merge.""" +"""Structure ops over dict records — RenameField/DropField/CopyField/SelectFields.""" import numpy as np import pytest -from sampleflux import Image, Label, Regions, Sample, primary -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields, SetRole +from sampleflux import Image, Label, Record, Regions +from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields -def _sample() -> Sample: - return Sample( - {"image": Image(np.zeros((2, 2, 3))), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")}, - roles={"regions": "target", "class": "target"}, - ) - - -class TestSetRole: - def test_retags(self) -> None: - out = SetRole(key="regions", role="aux")(_sample()) - assert out.role_of("regions") == "aux" and out.role_of("class") == "target" - - def test_lazy_validation(self) -> None: - assert SetRole().key == "" # zero-arg constructible - with pytest.raises(ValueError, match="'key'"): - SetRole()(_sample()) - # An invalid Literal is rejected at CONSTRUCTION (confluid schema enforcement) … - with pytest.raises(Exception, match="input_value='bogus'"): - SetRole(key="class", role="bogus") # type: ignore[arg-type] - # … and the defensive __call__ re-check guards post-construction mutation. - op = SetRole(key="class") - op.role = "bogus" # type: ignore[assignment] - with pytest.raises(ValueError, match="invalid role"): - op(_sample()) +def _record() -> Record: + return {"image": Image(np.zeros((2, 2, 3))), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")} class TestRenameField: - def test_renames_role_travels(self) -> None: - out = RenameField(src="regions", dst="boxes")(_sample()) - assert "regions" not in out and out.role_of("boxes") == "target" + def test_renames_preserving_order(self) -> None: + out = RenameField(src="regions", dst="boxes")(_record()) + assert "regions" not in out and isinstance(out["boxes"], Regions) + assert list(out.keys()) == ["image", "boxes", "class"] # renamed in place def test_rename_onto_existing_replaces(self) -> None: - out = RenameField(src="class", dst="image")(_sample()) - assert isinstance(out["image"], Label) and out.role_of("image") == "target" + out = RenameField(src="class", dst="image")(_record()) + assert isinstance(out["image"], Label) def test_validation(self) -> None: with pytest.raises(ValueError, match="both 'src' and 'dst'"): - RenameField()(_sample()) - with pytest.raises(KeyError, match="unknown field"): - RenameField(src="nope", dst="x")(_sample()) + RenameField()(_record()) + with pytest.raises(KeyError, match="unknown key"): + RenameField(src="nope", dst="x")(_record()) class TestDropField: def test_drops(self) -> None: - out = DropField(key="class")(_sample()) + out = DropField(key="class")(_record()) assert "class" not in out and list(out.keys()) == ["image", "regions"] def test_missing_raises_unless_ok(self) -> None: - with pytest.raises(KeyError, match="unknown field"): - DropField(key="nope")(_sample()) - assert DropField(key="nope", missing_ok=True)(_sample()) == _sample() + with pytest.raises(KeyError, match="unknown key"): + DropField(key="nope")(_record()) + rec = _record() + assert DropField(key="nope", missing_ok=True)(rec) is rec + + def test_missing_key_config_raises(self) -> None: + with pytest.raises(ValueError, match="'key'"): + DropField()(_record()) class TestCopyField: - def test_copies_with_source_role(self) -> None: - out = CopyField(src="regions", dst="regions_backup")(_sample()) - assert out["regions_backup"] is out["regions"] and out.role_of("regions_backup") == "target" + def test_copies_same_object(self) -> None: + out = CopyField(src="regions", dst="regions_backup")(_record()) + assert out["regions_backup"] is out["regions"] # same value object (values are immutable) - def test_copy_with_explicit_role(self) -> None: - out = CopyField(src="regions", dst="regions_aux", role="aux")(_sample()) - assert out.role_of("regions_aux") == "aux" + def test_copy_replaces_existing_dst(self) -> None: + out = CopyField(src="class", dst="image")(_record()) + assert isinstance(out["image"], Label) def test_validation(self) -> None: with pytest.raises(ValueError, match="both 'src' and 'dst'"): - CopyField()(_sample()) - with pytest.raises(KeyError, match="unknown field"): - CopyField(src="nope", dst="x")(_sample()) + CopyField()(_record()) + with pytest.raises(KeyError, match="unknown key"): + CopyField(src="nope", dst="x")(_record()) class TestSelectFields: def test_keeps_only_and_orders(self) -> None: - out = SelectFields(keys=["class", "image"])(_sample()) - assert list(out.keys()) == ["class", "image"] and out.role_of("class") == "target" + out = SelectFields(keys=["class", "image"])(_record()) + assert list(out.keys()) == ["class", "image"] def test_validation(self) -> None: with pytest.raises(ValueError, match="'keys'"): - SelectFields()(_sample()) - with pytest.raises(KeyError, match="unknown fields"): - SelectFields(keys=["image", "nope"])(_sample()) - - -class TestPrimaryAndMerge: - def test_primary_by_role(self) -> None: - s = _sample() - assert primary(s)[0] == "image" - assert primary(s, "target")[0] == "regions" # first target in insertion order - - def test_primary_missing_role_raises(self) -> None: - with pytest.raises(KeyError, match="no field with role 'pred'"): - primary(_sample(), "pred") - - def test_merge_union_last_wins(self) -> None: - a = Sample({"x": Label("a"), "shared": Label("from_a")}) - b = Sample({"y": Label("b"), "shared": Label("from_b")}, roles={"shared": "target"}) - m = Sample.merge(a, b) - assert list(m.keys()) == ["x", "shared", "y"] # union keeps first-seen position - assert m["shared"].value == "from_b" and m.role_of("shared") == "target" # last wins, role travels - - def test_merge_rejects_non_sample(self) -> None: - with pytest.raises(TypeError, match="expected Sample"): - Sample.merge(_sample(), "nope") # type: ignore[arg-type] - - def test_configurable_marks(self) -> None: - for cls in (SetRole, RenameField, DropField, CopyField, SelectFields): - assert getattr(cls, "__confluid_category__", None) == "op" - assert getattr(cls, "__confluid_group__", None) == "structure" + SelectFields()(_record()) + with pytest.raises(KeyError, match="unknown keys"): + SelectFields(keys=["image", "nope"])(_record()) + + +class TestCopyOnWrite: + def test_ops_never_mutate_the_incoming_record(self) -> None: + rec = _record() + RenameField(src="class", dst="klass")(rec) + DropField(key="class")(rec) + CopyField(src="class", dst="klass")(rec) + SelectFields(keys=["image"])(rec) + assert list(rec.keys()) == ["image", "regions", "class"] # untouched + + +def test_configurable_marks() -> None: + for cls in (RenameField, DropField, CopyField, SelectFields): + assert getattr(cls, "__confluid_category__", None) == "op" + assert getattr(cls, "__confluid_group__", None) == "structure" diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..41b1da1 --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,121 @@ +"""Transforms — type dispatch, once-per-record params, cross-key consistency, ``field=`` pin. + +Native-kernel machinery is pinned via the test fixture ``FixtureFlip`` (sampleflux ships no +native augmentation transforms — libraries drop into ops lists bare, invoked natively by the +engine's op-family dispatch). +""" + +import numpy as np +import pytest + +from sampleflux import Image, Label, Mask, Pipeline, Record, Regions, Transform, as_transform +from tests._fixtures import FixtureFlip + + +def _seg() -> Record: + return { + "image": Image(np.arange(8 * 10 * 3).reshape(8, 10, 3).astype(np.float32)), + "mask": Mask(np.arange(8 * 10).reshape(8, 10)), + "regions": Regions(boxes=[[1, 1, 4, 4]], labels=["a"], canvas=(8, 10)), + "class": Label("a"), + "gain_db": -3.0, # a plain scalar side value is just another key + } + + +class TestKernelDispatchMachinery: + def test_cross_key_consistency(self) -> None: + out = FixtureFlip(p=1.0)(_seg()) + seg = _seg() + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(seg["mask"])[:, ::-1]) + assert out["regions"].boxes == [[6, 1, 9, 4]] # W=10: x -> W-x + assert out["class"].value == "a" # no handler — untouched + assert out["gain_db"] == -3.0 # plain value — untouched + + def test_p_zero_is_identity(self) -> None: + out = FixtureFlip(p=0.0)(_seg()) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert out["regions"].boxes == [[1, 1, 4, 4]] + + def test_field_pin(self) -> None: + # field= pins the op to ONE key: only "image" moves, the other handled types stay. + out = FixtureFlip(p=1.0, field="image")(_seg()) + assert out is not None + assert not np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(_seg()["mask"])) # mask skipped + assert out["regions"].boxes == [[1, 1, 4, 4]] # regions skipped + + def test_field_pin_on_unhandled_type_is_noop(self) -> None: + # Still type-gated: pinning to a key whose value type has no kernel changes nothing. + out = FixtureFlip(p=1.0, field="class")(_seg()) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert out["class"].value == "a" + + def test_image_layout_chw(self) -> None: + s = {"image": Image(np.arange(3 * 4 * 5).reshape(3, 4, 5), layout="CHW")} + out = FixtureFlip(p=1.0)(s) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, :, ::-1]) + + def test_regions_uses_canvas_without_image(self) -> None: + s = {"regions": Regions(boxes=[[2, 0, 5, 3]], canvas=(8, 10))} + out = FixtureFlip(p=1.0)(s) + assert out is not None and out["regions"].boxes == [[5, 0, 8, 3]] + + def test_regions_without_reference_width_raises(self) -> None: + s = {"regions": Regions(boxes=[[2, 0, 5, 3]])} # no image, no canvas + with pytest.raises(ValueError, match="no reference width"): + FixtureFlip(p=1.0)(s) + + def test_params_sampled_once(self) -> None: + # A partial-probability flip must be all-or-nothing across keys (shared decision), + # never per-key independent draws. + seg = _seg() + for _ in range(25): + out = FixtureFlip(p=0.5)(seg) + assert out is not None + image_flipped = not np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])) + regions_flipped = out["regions"].boxes != seg["regions"].boxes + assert image_flipped == regions_flipped + + def test_input_record_not_mutated(self) -> None: + # Transform.__call__ is copy-on-write: the incoming dict keeps its original values. + seg = _seg() + FixtureFlip(p=1.0)(seg) + assert np.array_equal(np.asarray(seg["image"]), np.asarray(_seg()["image"])) + + +class TestPipelineAndFunction: + def test_pipeline_is_sequential(self) -> None: + s = {"x": Image(np.ones((2, 2, 3), dtype=np.float32))} + double = as_transform(lambda d: d * 2, handles=(Image,)) + out = Pipeline([double, double])(s) + assert out is not None and np.allclose(np.asarray(out["x"]), 4.0) + + def test_function_transform_field_pin(self) -> None: + s = {"a": Image(np.ones((2, 2, 3))), "b": Image(np.ones((2, 2, 3)))} + out = as_transform(lambda d: d + 1, handles=(Image,), field="a")(s) + assert out is not None + assert np.allclose(np.asarray(out["a"]), 2.0) and np.allclose(np.asarray(out["b"]), 1.0) + + def test_function_transform_preserves_item_type(self) -> None: + s = {"m": Mask(np.ones((2, 2)))} + out = as_transform(lambda d: d * 3, handles=(Mask,))(s) + assert out is not None and isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 3.0) + + def test_pipeline_repr(self) -> None: + assert "FixtureFlip" in repr(Pipeline([FixtureFlip()])) + + +class TestBaseTransform: + def test_default_get_params_and_passthrough(self) -> None: + # A transform with no kernels leaves every value alone. + s = {"x": Label("v"), "g": 1.5} + out = Transform()(s) + assert out == s and out is not s # equal copy, not the same dict + + def test_zero_arg_construction(self) -> None: + assert FixtureFlip().p == 0.5 and Transform().field is None diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py index 0b92f0f..08a4dc1 100644 --- a/tests/test_typed_collate.py +++ b/tests/test_typed_collate.py @@ -1,4 +1,4 @@ -"""The typed collate — batched Sample convention (golden shapes consumers rely on).""" +"""The record collate — batched record convention (golden shapes consumers rely on).""" from dataclasses import dataclass @@ -6,7 +6,7 @@ import pytest import torch -from sampleflux import Image, Label, Mask, Sample, collate, get_collate, register_item +from sampleflux import Image, Label, Mask, Record, collate, collate_records, get_collate, register_item @register_item @@ -16,53 +16,66 @@ class _CollateBlob: rate: float = 1.0 -def _sample(i: int) -> Sample: - return Sample( - { - "image": Image(np.full((4, 5, 3), float(i), dtype=np.float32)), - "mask": Mask(np.full((4, 5), i, dtype=np.int64)), - "class": Label(i, classes=["a", "b", "c"]), - }, - roles={"mask": "target", "class": "target"}, - ) +def _record(i: int) -> Record: + return { + "image": Image(np.full((4, 5, 3), float(i), dtype=np.float32)), + "mask": Mask(np.full((4, 5), i, dtype=np.int64)), + "class": Label(i, classes=["a", "b", "c"]), + "gain_db": float(i) - 3.0, # a plain scalar entry + } -class TestTypedCollate: +class TestRecordCollate: def test_golden_shapes(self) -> None: - # THE batch convention consumers rely on: batched Sample, payloads stacked - # per field, per-item attrs as lists, roles preserved. - batch = collate([_sample(0), _sample(1), _sample(2)]) - assert isinstance(batch, Sample) + # THE batch convention consumers rely on: ONE batched record dict, payloads stacked + # per key, per-record item attrs as lists, plain values as plain lists. + batch = collate([_record(0), _record(1), _record(2)]) + assert isinstance(batch, dict) assert np.asarray(batch["image"]).shape == (3, 4, 5, 3) # stacked payload + assert isinstance(batch["image"], Image) assert np.asarray(batch["mask"]).shape == (3, 4, 5) - assert batch["class"].value == [0, 1, 2] # per-item attrs become lists + assert batch["class"].value == [0, 1, 2] # per-record attrs become lists assert batch["class"].classes == [["a", "b", "c"]] * 3 - assert batch.roles == {"image": "input", "mask": "target", "class": "target"} + assert batch["gain_db"] == [-3.0, -2.0, -1.0] # plain values -> a plain list - def test_auto_dispatch_and_explicit_key(self) -> None: - samples = [_sample(0), _sample(1)] - auto = collate(samples) # Sample batch routes to "typed" automatically - explicit = get_collate("typed")(samples) - assert isinstance(auto, Sample) and isinstance(explicit, Sample) - assert np.array_equal(np.asarray(auto["image"]), np.asarray(explicit["image"])) + def test_default_key_and_explicit_key(self) -> None: + records = [_record(0), _record(1)] + default = collate(records) # the default registry key is "record" + explicit = get_collate("record")(records) + assert isinstance(default, dict) and isinstance(explicit, dict) + assert np.array_equal(np.asarray(default["image"]), np.asarray(explicit["image"])) + assert get_collate("record") is collate_records + + def test_item_attr_lists_decode_back_into_one_item(self) -> None: + batch = collate_records([_record(0), _record(1)]) + # The batched Image is ONE Image whose layout attr is the per-record list. + assert isinstance(batch["image"], Image) and batch["image"].layout == ["HWC", "HWC"] def test_torch_payloads_stack_to_tensor(self) -> None: - samples = [ - Sample({"sig": _CollateBlob(torch.ones(8) * i, rate=float(i))}, roles={"sig": "input"}) for i in range(2) - ] - batch = collate(samples) + records = [{"sig": _CollateBlob(torch.ones(8) * i, rate=float(i))} for i in range(2)] + batch = collate(records) assert isinstance(batch["sig"].data, torch.Tensor) and batch["sig"].data.shape == (2, 8) assert batch["sig"].rate == [0.0, 1.0] + def test_plain_string_values_batch_as_list(self) -> None: + batch = collate_records([{"f": "a.iq"}, {"f": "b.iq"}]) + assert batch["f"] == ["a.iq", "b.iq"] + def test_heterogeneous_batch_raises(self) -> None: - odd = Sample({"other": Label("x")}) - with pytest.raises(ValueError, match="do not match the batch fields"): - collate([_sample(0), odd]) + odd = {"other": Label("x")} + with pytest.raises(ValueError, match="do not match the batch keys"): + collate([_record(0), odd]) def test_empty_batch_raises(self) -> None: with pytest.raises(ValueError, match="empty batch"): - get_collate("typed")([]) + get_collate("record")([]) + with pytest.raises(ValueError, match="empty batch"): + collate([]) + + def test_non_dict_items_raise(self) -> None: + with pytest.raises(TypeError, match="expected record dicts"): + get_collate("record")([1, 2, 3]) - def test_non_typed_items_raise(self) -> None: - with pytest.raises(TypeError, match="expected Sample"): - get_collate("typed")([1, 2, 3]) + def test_unknown_key_raises_with_known_keys(self) -> None: + with pytest.raises(KeyError, match="no collate registered"): + get_collate("nope") diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py index ca61cfa..9ce8f69 100644 --- a/tests/test_typed_detection_target_ops.py +++ b/tests/test_typed_detection_target_ops.py @@ -1,16 +1,14 @@ -"""Typed-bag TWINS of the two detection target-shaping ops. +"""The two detection target-shaping ops over dict records. -Pins the native typed transforms that let a ``Sample`` detection pipeline build its -torchvision-style ``{boxes, labels}`` target as a :class:`~sampleflux.Regions` item without the -legacy ``Sample`` path: +Pins the native transforms that build a detection pipeline's torchvision-style +``{boxes, labels}`` target as a :class:`~sampleflux.Regions` item: * :class:`sampleflux.ops.target.CocoToTorchVisionDetection` — a HuggingFace / COCO ``objects`` annotation → a target ``Regions``; * :class:`sampleflux.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Regions``. -Each twin REUSES its legacy op's conversion math, so the twin's ``boxes`` / ``labels`` tensors are -pinned byte-identical to a legacy run on the equivalent ``Sample`` (parity). sampleflux-only — no -waivefront import. +Each op REUSES its conversion helper, so the op's ``boxes`` / ``labels`` tensors are pinned +byte-identical to the helper (parity). sampleflux-only — no domain-package import. """ import numpy as np @@ -18,8 +16,7 @@ import torch from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Label, Mask, Regions, Sample -from sampleflux.collate import typed_collate +from sampleflux import Image, Label, Mask, Regions, collate_records from sampleflux.ops.target import ( CocoToTorchVisionDetection, MasksToDetectionBoxes, @@ -45,60 +42,56 @@ def _instance_mask() -> np.ndarray: # --------------------------------------------------------------------------- # class TestCocoToTorchVisionDetection: def test_produces_target_regions(self) -> None: - s = Sample({"objects": Label(_OBJECTS)}, roles={"objects": "aux"}) - out = CocoToTorchVisionDetection(field="objects")(s) + out = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) regions = out["target"] assert isinstance(regions, Regions) - assert out.role_of("target") == "target" assert isinstance(regions.boxes, torch.Tensor) assert isinstance(regions.labels, torch.Tensor) assert regions.boxes.shape == (2, 4) assert regions.labels.shape == (2,) def test_parity_with_helper(self) -> None: - typed = CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label(_OBJECTS)})) + out = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) expected = coco_to_detection(_OBJECTS) - assert torch.equal(typed["target"].boxes, expected["boxes"]) - assert torch.equal(typed["target"].labels, expected["labels"]) + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) def test_parity_xyxy_and_label_offset(self) -> None: objects = {"bbox": [[10.0, 20.0, 40.0, 60.0]], "category": [2]} - typed = CocoToTorchVisionDetection(field="objects", bbox_format="xyxy", label_offset=1)( - Sample({"objects": Label(objects)}) + out = CocoToTorchVisionDetection(field="objects", bbox_format="xyxy", label_offset=1)( + {"objects": Label(objects)} ) expected = coco_to_detection(objects, bbox_format="xyxy", label_offset=1) - assert torch.equal(typed["target"].boxes, expected["boxes"]) - assert torch.equal(typed["target"].labels, expected["labels"]) + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) def test_empty_annotation_yields_empty_tensors(self) -> None: - out = CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label({"bbox": [], "category": []})})) + out = CocoToTorchVisionDetection(field="objects")({"objects": Label({"bbox": [], "category": []})}) assert out["target"].boxes.shape == (0, 4) assert out["target"].labels.shape == (0,) def test_default_picks_first_label(self) -> None: - s = Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "objects": Label(_OBJECTS)}) - out = CocoToTorchVisionDetection()(s) + rec = {"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "objects": Label(_OBJECTS)} + out = CocoToTorchVisionDetection()(rec) assert out["target"].boxes.shape == (2, 4) - def test_new_output_field_keeps_source(self) -> None: - s = Sample({"objects": Label(_OBJECTS)}) - out = CocoToTorchVisionDetection(field="objects", output="det")(s) + def test_new_output_key_keeps_source(self) -> None: + out = CocoToTorchVisionDetection(field="objects", output="det")({"objects": Label(_OBJECTS)}) assert isinstance(out["det"], Regions) - assert out.role_of("det") == "target" assert out["objects"].value == _OBJECTS # source left intact def test_missing_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - CocoToTorchVisionDetection(field="nope")(Sample({"objects": Label(_OBJECTS)})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + CocoToTorchVisionDetection(field="nope")({"objects": Label(_OBJECTS)}) - def test_empty_sample_raises(self) -> None: - with pytest.raises(ValueError, match="sample is empty"): - CocoToTorchVisionDetection()(Sample({})) + def test_empty_record_raises(self) -> None: + with pytest.raises(ValueError, match="record is empty"): + CocoToTorchVisionDetection()({}) def test_non_dict_source_raises(self) -> None: - # The reused legacy op rejects a non-objects-shaped value loudly. + # The shared helper rejects a non-objects-shaped value loudly. with pytest.raises(TypeError, match="objects mapping"): - CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label("not a dict")})) + CocoToTorchVisionDetection(field="objects")({"objects": Label("not a dict")}) # --------------------------------------------------------------------------- # @@ -106,74 +99,70 @@ def test_non_dict_source_raises(self) -> None: # --------------------------------------------------------------------------- # class TestMasksToDetectionBoxes: def test_instance_mask_produces_target_regions(self) -> None: - s = Sample({"mask": Mask(_instance_mask())}, roles={"mask": "aux"}) - out = MasksToDetectionBoxes(field="mask")(s) + out = MasksToDetectionBoxes(field="mask")({"mask": Mask(_instance_mask())}) regions = out["target"] assert isinstance(regions, Regions) assert isinstance(regions.boxes, torch.Tensor) assert isinstance(regions.labels, torch.Tensor) - assert out.role_of("target") == "target" assert regions.boxes.shape == (3, 4) # three instances assert regions.labels.tolist() == [1, 1, 1] # every box → foreground class 1 def test_instance_parity_with_helper(self) -> None: mask = _instance_mask() - typed = MasksToDetectionBoxes(field="mask")(Sample({"mask": Mask(mask)})) + out = MasksToDetectionBoxes(field="mask")({"mask": Mask(mask)}) expected = masks_to_detection(mask) - assert torch.equal(typed["target"].boxes, expected["boxes"]) - assert torch.equal(typed["target"].labels, expected["labels"]) + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) def test_connected_components_parity(self) -> None: # A binary/semantic mask (all objects share value 1): connected=True splits into blobs. binary = (_instance_mask() != 0).astype(np.uint8) - typed = MasksToDetectionBoxes(field="mask", connected=True, label=2)(Sample({"mask": Mask(binary)})) + out = MasksToDetectionBoxes(field="mask", connected=True, label=2)({"mask": Mask(binary)}) expected = masks_to_detection(binary, connected=True, label=2) - assert typed["target"].boxes.shape[0] == 3 # three connected blobs - assert torch.equal(typed["target"].boxes, expected["boxes"]) - assert torch.equal(typed["target"].labels, expected["labels"]) + assert out["target"].boxes.shape[0] == 3 # three connected blobs + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) def test_min_area_drops_small_instances(self) -> None: mask = _instance_mask() - typed = MasksToDetectionBoxes(field="mask", min_area=10)(Sample({"mask": Mask(mask)})) + out = MasksToDetectionBoxes(field="mask", min_area=10)({"mask": Mask(mask)}) expected = masks_to_detection(mask, min_area=10) - assert torch.equal(typed["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].boxes, expected["boxes"]) def test_empty_mask_yields_empty_tensors(self) -> None: - out = MasksToDetectionBoxes(field="mask")(Sample({"mask": Mask(np.zeros((4, 4), dtype=np.uint8))})) + out = MasksToDetectionBoxes(field="mask")({"mask": Mask(np.zeros((4, 4), dtype=np.uint8))}) assert out["target"].boxes.shape == (0, 4) assert out["target"].labels.shape == (0,) def test_default_picks_first_mask(self) -> None: - s = Sample({"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "seg": Mask(_instance_mask())}) - out = MasksToDetectionBoxes()(s) + rec = {"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "seg": Mask(_instance_mask())} + out = MasksToDetectionBoxes()(rec) assert out["target"].boxes.shape == (3, 4) - def test_new_output_field_keeps_source(self) -> None: - s = Sample({"mask": Mask(_instance_mask())}) - out = MasksToDetectionBoxes(field="mask", output="det")(s) + def test_new_output_key_keeps_source(self) -> None: + out = MasksToDetectionBoxes(field="mask", output="det")({"mask": Mask(_instance_mask())}) assert isinstance(out["det"], Regions) - assert out.role_of("det") == "target" assert isinstance(out["mask"], Mask) # source left intact def test_missing_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - MasksToDetectionBoxes(field="nope")(Sample({"mask": Mask(_instance_mask())})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + MasksToDetectionBoxes(field="nope")({"mask": Mask(_instance_mask())}) def test_no_mask_or_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no Mask or array-bearing field"): - MasksToDetectionBoxes()(Sample({"lbl": Label("x")})) + MasksToDetectionBoxes()({"lbl": Label("x")}) # --------------------------------------------------------------------------- # -# Collate — per-sample Regions gather into a list of detection targets. +# Collate — per-record Regions gather into a list of detection targets. # --------------------------------------------------------------------------- # -def test_typed_collate_gathers_regions_as_list() -> None: - a = CocoToTorchVisionDetection(field="objects")(Sample({"objects": Label(_OBJECTS)})) +def test_record_collate_gathers_regions_as_list() -> None: + a = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) c = CocoToTorchVisionDetection(field="objects")( - Sample({"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [5]})}) + {"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [5]})} ) - batch = typed_collate([a, c]) - # Variable-N boxes can't be stacked → the collate gathers them as a per-sample list of tensors. + batch = collate_records([a, c]) + # Variable-N boxes can't be stacked → the collate gathers them as a per-record list of tensors. assert isinstance(batch["target"], Regions) assert isinstance(batch["target"].boxes, list) and len(batch["target"].boxes) == 2 assert batch["target"].boxes[0].shape == (2, 4) diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index 174e1c0..669f4ae 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -1,14 +1,14 @@ -"""Typed FlowGraph — merge_from fan-in, step[key] bind, typed carriers through Flux, parity.""" +"""FlowGraph over dict records — merge_from fan-in, step[key]/bare-step bind, lowering parity.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional import numpy as np import pytest -from sampleflux import FlowGraph, Flux, Image, Label, Mask, Sample, Transform, to_ops +from sampleflux import FlowGraph, Flux, Image, Label, Mask, Pipeline, Record, Transform, to_ops from sampleflux.flow import from_ops, parse_flow from sampleflux.ops.context import MergeFields -from sampleflux.ops.structure import RenameField, SetRole +from sampleflux.ops.structure import RenameField, SelectFields class _AddOffset(Transform): @@ -16,46 +16,40 @@ class _AddOffset(Transform): handles = (Image,) - def __init__(self, offset: float = 0.0, only: Optional[List[str]] = None) -> None: - super().__init__(only=only) + def __init__(self, offset: float = 0.0, field: Optional[str] = None) -> None: + super().__init__(field=field) self.offset = offset - def __call__(self, sample: Sample) -> Sample: - out = sample - for key, item in sample.items(): - if isinstance(item, Image) and (self.only is None or key in self.only): - out = out.replace_field(key, Image(np.asarray(item) + self.offset, layout=item.layout)) + def __call__(self, record: Record) -> Record: + out = dict(record) + for key, item in record.items(): + if isinstance(item, Image) and (self.field is None or key == self.field): + out[key] = Image(np.asarray(item) + self.offset, layout=item.layout) return out class _MakeMask(Transform): - """Derives a Mask field from the first Image (a branch producer).""" + """Derives a Mask entry from the first Image (a branch producer).""" - def __call__(self, sample: Sample) -> Sample: - image = next(item for item in sample.fields.values() if isinstance(item, Image)) - out = sample.replace_field("mask", Mask(np.asarray(image)[..., 0] > 0.5)) - return out.set_role("mask", "target") + def __call__(self, record: Record) -> Record: + image = next(item for item in record.values() if isinstance(item, Image)) + return {**record, "mask": Mask(np.asarray(image)[..., 0] > 0.5)} -def _seed(value: float = 0.0) -> Sample: - return Sample( - {"image": Image(np.full((2, 3, 3), value, dtype=np.float32)), "label": Label("x")}, - roles={"label": "target"}, - ) +def _seed(value: float = 0.0) -> Record: + return {"image": Image(np.full((2, 3, 3), value, dtype=np.float32)), "label": Label("x")} -class TestTypedFlowGraph: - def test_linear_typed_flow(self) -> None: +class TestFlowGraph: + def test_linear_flow(self) -> None: graph = FlowGraph(source=[_seed(1.0)], flow={"plus": _AddOffset(offset=2.0)}) (out,) = list(graph) - assert isinstance(out, Sample) and np.allclose(np.asarray(out["image"]), 3.0) + assert isinstance(out, dict) and np.allclose(np.asarray(out["image"]), 3.0) def test_merge_from_union(self) -> None: - # Fork: derive a mask on a branch, SELECT the new field, union it back into the main - # stream. (Selecting is the idiom — a full branch bag would also carry its own + # Fork: derive a mask on a branch, SELECT the new entry, union it back into the main + # stream. (Selecting is the idiom — a full branch record would also carry its own # 'image', and last-wins would overwrite the boosted one.) - from sampleflux.ops.structure import SelectFields - flow = { "start": {}, "masked": {"op": _MakeMask(), "from": "start"}, @@ -66,11 +60,12 @@ def test_merge_from_union(self) -> None: graph = FlowGraph(source=[_seed(0.75)], flow=flow, outputs="out") (out,) = list(graph) assert np.allclose(np.asarray(out["image"]), 1.75) # the boosted branch's image survives - assert "mask" in out and out.role_of("mask") == "target" # the selected branch field + assert "mask" in out and isinstance(out["mask"], Mask) # the selected branch entry assert out["label"].value == "x" def test_merge_collision_last_wins(self) -> None: - # Both branches carry 'image'; the merge source is listed LAST -> its image wins. + # Both branches carry 'image'; the merge source is listed LAST -> its image wins + # (dict-union semantics, listed order). flow = { "start": {}, "a": {"op": _AddOffset(offset=1.0), "from": "start"}, @@ -92,20 +87,18 @@ def test_rename_avoids_collision(self) -> None: assert "image_b" in out # branch b united under its renamed key def test_step_key_bind(self) -> None: - # bind offset := the 'probe' step's image payload mean is NOT expressible without a - # value op — bind the FIELD instead and let the op read it: offset receives the - # Image item from probe via step[image]. + # step[key] binds the NAMED ENTRY of the bound step's record result. class _OffsetFromItem(Transform): def __init__(self, item: Any = None) -> None: super().__init__() self.item = item - def __call__(self, sample: Sample) -> Sample: + def __call__(self, record: Record) -> Record: offset = float(np.asarray(self.item).mean()) - out = sample - for key, value in sample.items(): + out = dict(record) + for key, value in record.items(): if isinstance(value, Image): - out = out.replace_field(key, Image(np.asarray(value) + offset, layout=value.layout)) + out[key] = Image(np.asarray(value) + offset, layout=value.layout) return out flow = { @@ -116,26 +109,27 @@ def __call__(self, sample: Sample) -> Sample: (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) assert np.allclose(np.asarray(out["image"]), 2.0) # 0.0 + mean(2.0) - def test_bare_step_bind_is_primary(self) -> None: - class _CapturePrimary(Transform): + def test_bare_step_bind_is_whole_record(self) -> None: + class _CaptureWhole(Transform): def __init__(self, item: Any = None) -> None: super().__init__() self.item = item - def __call__(self, sample: Sample) -> Sample: - assert isinstance(self.item, Image) # primary input-role field of the bound step - return sample + def __call__(self, record: Record) -> Record: + # bare "probe" bind = the step's WHOLE result record dict. + assert isinstance(self.item, dict) and isinstance(self.item["image"], Image) + return record flow = { "start": {}, "probe": {"op": _AddOffset(offset=1.0), "from": "start"}, - "final": {"op": _CapturePrimary(), "from": "start", "bind": {"item": "probe"}}, + "final": {"op": _CaptureWhole(), "from": "start", "bind": {"item": "probe"}}, } (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) - assert isinstance(out, Sample) + assert isinstance(out, dict) def test_legacy_fanin_key_removed(self) -> None: - # target_from / metadata_from (the legacy Sample fan-in) were purged; they are now + # target_from / metadata_from (the legacy role fan-in) were purged; they are now # unknown step keys — a flow document using one fails loudly at parse. flow = { "start": {}, @@ -155,10 +149,8 @@ def test_merge_from_forward_ref_raises(self) -> None: parse_flow({"a": {"merge_from": ["b"]}, "b": {}}) -class TestTypedLoweringParity: +class TestLoweringParity: def _flow(self) -> Dict[str, Any]: - from sampleflux.ops.structure import SelectFields - return { "start": {}, "masked": {"op": _MakeMask(), "from": "start"}, @@ -173,7 +165,9 @@ def test_to_ops_runs_on_flux(self) -> None: native = list(FlowGraph(source=[_seed(0.25)], flow=self._flow(), outputs="out")) lowered = list(Flux(source=[_seed(0.25)], ops=to_ops(steps, outputs))) assert len(native) == len(lowered) == 1 - assert native[0] == lowered[0] + assert list(native[0].keys()) == list(lowered[0].keys()) + assert np.array_equal(np.asarray(native[0]["image"]), np.asarray(lowered[0]["image"])) + assert np.array_equal(np.asarray(native[0]["mask"]), np.asarray(lowered[0]["mask"])) def test_round_trip_from_ops(self) -> None: steps, outputs = parse_flow(self._flow()) @@ -190,8 +184,8 @@ def __init__(self, item: Any = None) -> None: super().__init__() self.item = item - def __call__(self, sample: Sample) -> Sample: - return sample.replace_field("echo", self.item) + def __call__(self, record: Record) -> Record: + return {**record, "echo": self.item} flow = { "start": {}, @@ -208,20 +202,19 @@ def __call__(self, sample: Sample) -> Sample: assert np.allclose(np.asarray(out["echo"]), 3.0) -class TestTypedThroughFlux: - def test_default_flux_carries_typed_verbatim(self) -> None: - # No native=True needed: a Sample source item is NEVER coerced to legacy Sample. +class TestRecordsThroughFlux: + def test_flux_carries_record_dicts_verbatim(self) -> None: flux = Flux(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) (out,) = list(flux) - assert isinstance(out, Sample) and np.allclose(np.asarray(out["image"]), 2.0) - - def test_getitem_typed(self) -> None: - flux = Flux(source=[_seed(1.0), _seed(2.0)], ops=[SetRole(key="image", role="aux")]) - assert flux[1].role_of("image") == "aux" + assert isinstance(out, dict) and np.allclose(np.asarray(out["image"]), 2.0) - def test_compose_ops_route_typed(self) -> None: - from sampleflux.ops.transform_chain import TransformChain + def test_getitem(self) -> None: + flux = Flux(source=[_seed(1.0), _seed(2.0)], ops=[RenameField(src="label", dst="klass")]) + out = flux[1] + assert "klass" in out and np.allclose(np.asarray(out["image"]), 2.0) - flux = Flux(source=[_seed(1.0)], ops=[TransformChain(ops=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])]) + def test_compose_ops_route_records(self) -> None: + # Pipeline (the compose-group grouping op — TransformChain's replacement). + flux = Flux(source=[_seed(1.0)], ops=[Pipeline(transforms=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])]) (out,) = list(flux) assert np.allclose(np.asarray(out["image"]), 4.0) diff --git a/tests/test_typed_generic_ops.py b/tests/test_typed_generic_ops.py index b6870dd..0aeccd4 100644 --- a/tests/test_typed_generic_ops.py +++ b/tests/test_typed_generic_ops.py @@ -1,21 +1,21 @@ -"""Typed-bag TWINS of the generic array→Image→Mask→Regions ops. +"""The generic array→Image→Mask→Regions ops over dict records. -Pins the three native typed transforms that let a ``Sample`` pipeline run the -detection/segmentation front-end without the legacy ``Sample`` path: +Pins the three native type-changing transforms that run the detection/segmentation +front-end on plain record dicts: -* :class:`sampleflux.ops.image.ConvertToImage` — array-bearing field → ``Image`` item; -* :class:`sampleflux.ops.numpy.Threshold` — array field → boolean ``Mask`` item; +* :class:`sampleflux.ops.image.ConvertToImage` — array-bearing key → ``Image`` item; +* :class:`sampleflux.ops.numpy.Threshold` — array key → boolean ``Mask`` item; * :class:`sampleflux.ops.numpy.ConnectedComponents` — ``Mask`` → ``Regions`` item. -Each twin REUSES its legacy op's math, so the twin's output is pinned to be byte-identical -to a legacy run on the equivalent ``Sample`` (parity). sampleflux-only — no waivefront import. +Each op REUSES its shared math helper, so the op output is pinned identical to the helper +(parity). sampleflux-only — no domain-package import. """ import numpy as np import pytest from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Mask, Regions, Sample +from sampleflux import Image, Mask, Regions from sampleflux.ops.image import ConvertToImage, _bound_longest_side, _render_rgb from sampleflux.ops.numpy import ConnectedComponents, Threshold, connected_component_bboxes, threshold_array @@ -35,132 +35,129 @@ def _blob_mask() -> np.ndarray: # ConvertToImage # --------------------------------------------------------------------------- # class TestConvertToImage: - def test_produces_image_item_shape_dtype_role(self) -> None: - out = ConvertToImage(colormap="gray")(Sample({"spec": Mask(_ramp_2d())})) + def test_produces_image_item_shape_dtype(self) -> None: + out = ConvertToImage(colormap="gray")({"spec": Mask(_ramp_2d())}) assert "image" in out img = out["image"] assert isinstance(img, Image) assert np.asarray(img).shape == (8, 10, 3) assert np.asarray(img).dtype == np.uint8 assert img.layout == "HWC" - assert out.role_of("image") == "input" - # source field untouched + # source entry untouched assert isinstance(out["spec"], Mask) def test_parity_with_render_helper_default_sizing(self) -> None: arr = _ramp_2d() - typed = ConvertToImage(colormap="viridis")(Sample({"spec": Mask(arr)})) + out = ConvertToImage(colormap="viridis")({"spec": Mask(arr)}) expected = _bound_longest_side(_render_rgb(arr, "viridis"), 512) - assert np.array_equal(expected, np.asarray(typed["image"])) + assert np.array_equal(expected, np.asarray(out["image"])) def test_exact_resize_and_flip(self) -> None: arr = _ramp_2d() - typed = ConvertToImage(colormap="gray", width=20, height=16, flip_vertical=True)(Sample({"spec": Mask(arr)})) - assert np.asarray(typed["image"]).shape == (16, 20, 3) + out = ConvertToImage(colormap="gray", width=20, height=16, flip_vertical=True)({"spec": Mask(arr)}) + assert np.asarray(out["image"]).shape == (16, 20, 3) def test_explicit_field_and_custom_output(self) -> None: - s = Sample({"a": Mask(_ramp_2d()), "b": Mask(np.zeros((4, 4), dtype=np.float32))}) - out = ConvertToImage(field="b", output="preview")(s) + rec = {"a": Mask(_ramp_2d()), "b": Mask(np.zeros((4, 4), dtype=np.float32))} + out = ConvertToImage(field="b", output="preview")(rec) assert np.asarray(out["preview"]).shape == (4, 4, 3) - def test_does_not_publish_image_dims_metadata(self) -> None: - # There is no shared metadata dict in the typed model; the Image SHAPE carries the dims. - out = ConvertToImage()(Sample({"spec": Mask(_ramp_2d())})) - assert set(out.keys()) == {"spec", "image"} # no image_width_px / image_height_px field + def test_does_not_publish_image_dims_keys(self) -> None: + # The Image SHAPE carries the pixel dims; no image_width_px / image_height_px entries. + out = ConvertToImage()({"spec": Mask(_ramp_2d())}) + assert set(out.keys()) == {"spec", "image"} assert np.asarray(out["image"]).shape[:2] == (8, 10) def test_missing_explicit_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - ConvertToImage(field="nope")(Sample({"spec": Mask(_ramp_2d())})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + ConvertToImage(field="nope")({"spec": Mask(_ramp_2d())}) def test_no_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - ConvertToImage()(Sample({"lbl": Regions(boxes=[[0, 0, 1, 1]])})) + ConvertToImage()({"lbl": Regions(boxes=[[0, 0, 1, 1]])}) # --------------------------------------------------------------------------- # # Threshold # --------------------------------------------------------------------------- # class TestThreshold: - def test_produces_mask_parity_role(self) -> None: + def test_produces_mask_parity(self) -> None: arr = _ramp_2d() - typed = Threshold(low_level=20.0)(Sample({"spec": Mask(arr)})) - assert isinstance(typed["mask"], Mask) - assert np.asarray(typed["mask"]).dtype == np.bool_ - assert typed.role_of("mask") == "aux" + out = Threshold(low_level=20.0)({"spec": Mask(arr)}) + assert isinstance(out["mask"], Mask) + assert np.asarray(out["mask"]).dtype == np.bool_ expected = threshold_array(arr, low_level=20.0) - assert np.array_equal(np.asarray(typed["mask"]), expected) + assert np.array_equal(np.asarray(out["mask"]), expected) def test_string_literal_bound(self) -> None: arr = _ramp_2d() - typed = Threshold(low_level="20")(Sample({"spec": Mask(arr)})) + out = Threshold(low_level="20")({"spec": Mask(arr)}) expected = threshold_array(arr, low_level="20") - assert np.array_equal(np.asarray(typed["mask"]), expected) + assert np.array_equal(np.asarray(out["mask"]), expected) def test_env_var_expression_bound(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("TEST_THRESH_LEVEL", "20") arr = _ramp_2d() - typed = Threshold(low_level="$TEST_THRESH_LEVEL")(Sample({"spec": Mask(arr)})) - assert np.array_equal(np.asarray(typed["mask"]), arr > 20.0) + out = Threshold(low_level="$TEST_THRESH_LEVEL")({"spec": Mask(arr)}) + assert np.array_equal(np.asarray(out["mask"]), arr > 20.0) - def test_meta_key_expression_has_no_typed_source(self) -> None: - # {key} expressions have no typed metadata home -> loud KeyError (documented). + def test_meta_key_expression_has_no_source(self) -> None: + # {key} expressions have no metadata home in the record model -> loud KeyError. with pytest.raises(KeyError): - Threshold(low_level="{some_key}")(Sample({"spec": Mask(_ramp_2d())})) + Threshold(low_level="{some_key}")({"spec": Mask(_ramp_2d())}) def test_band_pass_both_bounds_and_ops(self) -> None: arr = _ramp_2d() - typed = Threshold(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")(Sample({"spec": Mask(arr)})) + out = Threshold(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")({"spec": Mask(arr)}) expected = threshold_array(arr, low_level=20.0, high_level=60.0, low_op=">=", high_op="<=") - assert np.array_equal(np.asarray(typed["mask"]), expected) - assert np.array_equal(np.asarray(typed["mask"]), (arr >= 20.0) & (arr <= 60.0)) + assert np.array_equal(np.asarray(out["mask"]), expected) + assert np.array_equal(np.asarray(out["mask"]), (arr >= 20.0) & (arr <= 60.0)) def test_no_bound_raises(self) -> None: with pytest.raises(ValueError, match="at least one"): - Threshold()(Sample({"spec": Mask(_ramp_2d())})) + Threshold()({"spec": Mask(_ramp_2d())}) def test_default_field_picks_first_array(self) -> None: # No explicit field: first array-bearing item (insertion order). - s = Sample({"raw": Mask(_ramp_2d()), "other": Regions(boxes=[])}) - out = Threshold(low_level=20.0)(s) + rec = {"raw": Mask(_ramp_2d()), "other": Regions(boxes=[])} + out = Threshold(low_level=20.0)(rec) assert np.array_equal(np.asarray(out["mask"]), _ramp_2d() > 20.0) def test_missing_explicit_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - Threshold(low_level=1.0, field="nope")(Sample({"spec": Mask(_ramp_2d())})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + Threshold(low_level=1.0, field="nope")({"spec": Mask(_ramp_2d())}) def test_non_array_field_raises(self) -> None: with pytest.raises(TypeError, match="expected an array"): - Threshold(low_level=1.0, field="reg")(Sample({"reg": Regions(boxes=[])})) + Threshold(low_level=1.0, field="reg")({"reg": Regions(boxes=[])}) def test_no_array_field_default_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - Threshold(low_level=1.0)(Sample({"reg": Regions(boxes=[])})) + Threshold(low_level=1.0)({"reg": Regions(boxes=[])}) # --------------------------------------------------------------------------- # # ConnectedComponents # --------------------------------------------------------------------------- # class TestConnectedComponents: - def test_produces_regions_bin_box_contract_and_role(self) -> None: - out = ConnectedComponents()(Sample({"m": Mask(_blob_mask())})) + def test_produces_regions_bin_box_contract(self) -> None: + out = ConnectedComponents()({"m": Mask(_blob_mask())}) regions = out["boxes"] assert isinstance(regions, Regions) - assert out.role_of("boxes") == "aux" # The pinned generic contract: (row_min, row_max, col_min, col_max) inclusive tuples. assert regions.boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] - def test_parity_with_legacy(self) -> None: + def test_parity_with_helper(self) -> None: mask = _blob_mask() - typed = ConnectedComponents()(Sample({"m": Mask(mask)})) + out = ConnectedComponents()({"m": Mask(mask)}) expected = connected_component_bboxes(mask) - assert typed["boxes"].boxes == expected + assert out["boxes"].boxes == expected def test_min_area_bins_filters_small_blobs(self) -> None: m = np.zeros((6, 6), dtype=bool) m[0:2, 0:2] = True # area 4 m[5, 5] = True # area 1 -> dropped when min_area_bins=2 - out = ConnectedComponents(min_area_bins=2)(Sample({"m": Mask(m)})) + out = ConnectedComponents(min_area_bins=2)({"m": Mask(m)}) assert out["boxes"].boxes == [(0, 1, 0, 1)] def test_connectivity_parity(self) -> None: @@ -168,43 +165,43 @@ def test_connectivity_parity(self) -> None: m = np.zeros((4, 4), dtype=bool) m[0, 0] = True m[1, 1] = True - four = ConnectedComponents(connectivity=4)(Sample({"m": Mask(m)})) - eight = ConnectedComponents(connectivity=8)(Sample({"m": Mask(m)})) + four = ConnectedComponents(connectivity=4)({"m": Mask(m)}) + eight = ConnectedComponents(connectivity=8)({"m": Mask(m)}) assert len(four["boxes"].boxes) == 2 assert len(eight["boxes"].boxes) == 1 def test_default_prefers_mask_over_other_array(self) -> None: # An Image is inserted first, but a Mask is preferred by the default resolver. - s = Sample({"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())}) - out = ConnectedComponents()(s) + rec = {"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())} + out = ConnectedComponents()(rec) assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] def test_falls_back_to_first_array_when_no_mask(self) -> None: # No Mask item — a 2-D array item is used. - out = ConnectedComponents()(Sample({"m": Image(_blob_mask())})) + out = ConnectedComponents()({"m": Image(_blob_mask())}) assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] def test_non_2d_mask_raises(self) -> None: with pytest.raises(ValueError, match="2-D mask"): - ConnectedComponents()(Sample({"m": Mask(np.zeros((2, 2, 2), dtype=bool))})) + ConnectedComponents()({"m": Mask(np.zeros((2, 2, 2), dtype=bool))}) def test_missing_explicit_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - ConnectedComponents(field="nope")(Sample({"m": Mask(_blob_mask())})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + ConnectedComponents(field="nope")({"m": Mask(_blob_mask())}) def test_no_mask_or_array_raises(self) -> None: with pytest.raises(ValueError, match="no Mask or array-bearing field"): - ConnectedComponents()(Sample({"reg": Regions(boxes=[])})) + ConnectedComponents()({"reg": Regions(boxes=[])}) # --------------------------------------------------------------------------- # -# End-to-end chain: array -> Image -> Mask -> Regions, all typed, sampleflux-only. +# End-to-end chain: array -> Image -> Mask -> Regions, all on one record dict. # --------------------------------------------------------------------------- # def test_array_to_image_to_mask_to_regions_chain() -> None: arr = _ramp_2d() - sample = Sample({"spec": Mask(arr)}) - out = ConnectedComponents(field="mask")(Threshold(field="spec", low_level=20.0)(ConvertToImage()(sample))) - # Every stage produced its typed field. + record = {"spec": Mask(arr)} + out = ConnectedComponents(field="mask")(Threshold(field="spec", low_level=20.0)(ConvertToImage()(record))) + # Every stage produced its typed entry. assert isinstance(out["image"], Image) assert isinstance(out["mask"], Mask) assert isinstance(out["boxes"], Regions) @@ -214,7 +211,7 @@ def test_array_to_image_to_mask_to_regions_chain() -> None: assert len(box) == 4 row_min, row_max, col_min, col_max = box assert row_min <= row_max and col_min <= col_max - # The image field carries the pixel dims via its shape (no separate metadata). + # The image entry carries the pixel dims via its shape (no separate metadata). assert np.asarray(out["image"]).shape[:2] == arr.shape diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py index eec08c9..cd5eaa2 100644 --- a/tests/test_typed_storage.py +++ b/tests/test_typed_storage.py @@ -1,16 +1,17 @@ -"""Typed field-group storage — HDF5/Zarr/Directory round-trips, carrier guards, typed queries.""" +"""Record key-group storage — HDF5/Zarr/Directory round-trips, format-tag guard, metadata queries.""" from dataclasses import dataclass from pathlib import Path +import h5py import numpy as np import pytest -from sampleflux import Image, Label, Regions, Sample, register_item -from sampleflux.storage.base import restore_attrs, split_attrs +from sampleflux import Image, Label, Regions, register_item +from sampleflux.storage.base import TYPED_FORMAT, require_record_format, restore_attrs, split_attrs from sampleflux.storage.directory import DirectorySink, DirectorySource from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.query import MetadataFilterSource, scan_hdf5_metadata, scan_zarr_metadata +from sampleflux.storage.query import MetadataFilterSource, record_metadata, scan_hdf5_metadata, scan_zarr_metadata from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource @@ -24,34 +25,34 @@ class _StoreSig: mask: object = None # an ARRAY-valued attr — exercises the attrs/ dataset path -def _samples() -> list: - # Ragged across samples: different box counts, one field with an array-valued attr. - s0 = Sample( - { - "image": Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW"), - "sig": _StoreSig(np.arange(8, dtype=np.float32), samplerate=20e6, mask=np.array([1, 0, 1], dtype=np.uint8)), - "regions": Regions(boxes=[[0, 0, 1, 1], [1, 1, 2, 2]], labels=["a", "b"], canvas=(2, 2)), - "label": Label("drone", classes=["x", "drone"]), - }, - roles={"regions": "target", "label": "target", "sig": "aux"}, - ) - s1 = Sample( - { - "image": Image(np.ones((2, 2, 3), dtype=np.float32)), - "sig": _StoreSig(np.zeros(4, dtype=np.float32), samplerate=1e6, mask=np.array([0], dtype=np.uint8)), - "regions": Regions(boxes=[[0, 0, 2, 2]], labels=["c"], canvas=(2, 2)), - "label": Label("x", classes=["x", "drone"]), - }, - roles={"regions": "target", "label": "target", "sig": "aux"}, - ) - return [s0, s1] +def _records() -> list: + # Ragged across records: different box counts, one value with an array-valued attr, + # plus PLAIN entries (scalar / string / bare array) riding the "plain" type tag. + r0 = { + "image": Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW"), + "sig": _StoreSig(np.arange(8, dtype=np.float32), samplerate=20e6, mask=np.array([1, 0, 1], dtype=np.uint8)), + "regions": Regions(boxes=[[0, 0, 1, 1], [1, 1, 2, 2]], labels=["a", "b"], canvas=(2, 2)), + "label": Label("drone", classes=["x", "drone"]), + "gain_db": -3.0, + "source_file": "a.iq", + "window": np.hanning(4), + } + r1 = { + "image": Image(np.ones((2, 2, 3), dtype=np.float32)), + "sig": _StoreSig(np.zeros(4, dtype=np.float32), samplerate=1e6, mask=np.array([0], dtype=np.uint8)), + "regions": Regions(boxes=[[0, 0, 2, 2]], labels=["c"], canvas=(2, 2)), + "label": Label("x", classes=["x", "drone"]), + "gain_db": 1.5, + "source_file": "b.iq", + "window": np.hanning(4), + } + return [r0, r1] def _assert_round_trip(back: list, expect: list) -> None: assert len(back) == len(expect) for got, want in zip(back, expect): assert list(got.keys()) == list(want.keys()) # insertion order preserved - assert got.roles == want.roles assert got["image"].layout == want["image"].layout assert np.array_equal(np.asarray(got["image"]), np.asarray(want["image"])) assert got["sig"].samplerate == want["sig"].samplerate @@ -61,6 +62,9 @@ def _assert_round_trip(back: list, expect: list) -> None: assert got["regions"].canvas == want["regions"].canvas # tuple preserved assert isinstance(got["regions"].canvas, tuple) assert got["label"].value == want["label"].value and got["label"].classes == want["label"].classes + assert got["gain_db"] == want["gain_db"] # plain scalar round-trips + assert got["source_file"] == want["source_file"] # plain string round-trips + assert np.allclose(np.asarray(got["window"]), want["window"]) # plain bare-array round-trips class TestAttrWireFormat: @@ -89,83 +93,138 @@ def test_numpy_scalars_become_python(self) -> None: assert restore_attrs(plain, {})["x"] == 2.5 -class TestHDF5Typed: +class TestFormatGuard: + def test_require_record_format_accepts_current_tag(self) -> None: + require_record_format(TYPED_FORMAT, "test") # no raise + + @pytest.mark.parametrize("found", [None, "typedsample-v1", "bogus"]) + def test_require_record_format_rejects_everything_else(self, found: object) -> None: + with pytest.raises(ValueError, match="typedrecord-v1"): + require_record_format(found, "test") + + def test_hdf5_source_rejects_old_typedsample_tag(self, tmp_path: Path) -> None: + # NO backward compat: a pre-record-model store must fail loudly with the clear error. + path = tmp_path / "old.h5" + with h5py.File(path, "w") as handle: + handle.attrs["sampleflux_format"] = "typedsample-v1" + with pytest.raises(ValueError, match="typedrecord-v1"): + HDF5Source(path=path).open() + + def test_hdf5_sink_rejects_appending_to_old_tag(self, tmp_path: Path) -> None: + path = tmp_path / "old.h5" + with h5py.File(path, "w") as handle: + handle.attrs["sampleflux_format"] = "typedsample-v1" + sink = HDF5Sink(path=path) + with sink: + with pytest.raises(ValueError, match="typedrecord-v1"): + sink.write(_records()[0]) + + def test_zarr_group_source_rejects_old_tag(self, tmp_path: Path) -> None: + import zarr + + path = str(tmp_path / "old.zarr") + root = zarr.open_group(path, mode="a") + root.attrs["sampleflux_format"] = "typedsample-v1" + with pytest.raises(ValueError, match="typedrecord-v1"): + ZarrGroupSource(path=path).open() + + +class TestHDF5: def test_round_trip(self, tmp_path: Path) -> None: path = tmp_path / "t.h5" sink = HDF5Sink(path=path, overwrite=True) with sink: - for s in _samples(): - sink.write(s) + for r in _records(): + sink.write(r) sink.flush() source = HDF5Source(path=path) with source: - assert source.is_typed and len(source) == 2 - _assert_round_trip(list(source), _samples()) + assert len(source) == 2 + _assert_round_trip(list(source), _records()) + + def test_format_tag_stamped(self, tmp_path: Path) -> None: + path = tmp_path / "t.h5" + with HDF5Sink(path=path, overwrite=True) as sink: + sink.write(_records()[0]) + with h5py.File(path, "r") as handle: + assert handle.attrs["sampleflux_format"] == TYPED_FORMAT - def test_non_sample_write_raises(self, tmp_path: Path) -> None: - # The sink only accepts a typed Sample bag; a bare array is rejected loudly. + def test_non_dict_write_raises(self, tmp_path: Path) -> None: + # The sink only accepts a record dict; a bare array is rejected loudly. sink = HDF5Sink(path=tmp_path / "typed.h5", overwrite=True) with sink: - sink.write(_samples()[0]) - with pytest.raises(TypeError, match="expected a Sample bag"): + sink.write(_records()[0]) + with pytest.raises(TypeError, match="expected a record dict"): sink.write(np.zeros(3)) -class TestZarrTyped: +class TestZarr: def test_group_round_trip(self, tmp_path: Path) -> None: path = str(tmp_path / "g.zarr") sink = ZarrGroupSink(path=path) sink.open() - for s in _samples(): - sink.write(s) + for r in _records(): + sink.write(r) source = ZarrGroupSource(path=path) - assert source.is_typed and len(source) == 2 - _assert_round_trip(list(source), _samples()) + assert len(source) == 2 + _assert_round_trip(list(source), _records()) - def test_group_non_sample_write_raises(self, tmp_path: Path) -> None: + def test_group_non_dict_write_raises(self, tmp_path: Path) -> None: path = str(tmp_path / "g.zarr") sink = ZarrGroupSink(path=path) sink.open() - sink.write(_samples()[0]) - with pytest.raises(TypeError, match="expected a Sample bag"): + sink.write(_records()[0]) + with pytest.raises(TypeError, match="expected a record dict"): sink.write(np.zeros(3)) def test_batch_typed_rows(self, tmp_path: Path) -> None: path = str(tmp_path / "b.zarr") sink = ZarrBatchSink(path=path, shape=[2, 2, 3], dtype="float32", overwrite=True) sink.open() - for s in _samples(): - sink.write(s) # appends the PRIMARY input field's payload + for r in _records(): + sink.write(r) # appends the FIRST record entry's payload source = ZarrBatchSource(path=path) rows = list(source) - assert len(rows) == 2 and all(isinstance(r, Sample) for r in rows) + assert len(rows) == 2 and all(isinstance(r, dict) for r in rows) assert isinstance(rows[0]["image"], Image) and rows[0]["image"].layout == "CHW" # uniform template assert np.asarray(rows[1]["image"]).shape == (2, 2, 3) -class TestDirectoryTyped: +class TestDirectory: def test_round_trip(self, tmp_path: Path) -> None: path = tmp_path / "dir" sink = DirectorySink(path=path) sink.open() - for s in _samples(): - sink.write(s) + for r in _records(): + sink.write(r) source = DirectorySource(path=path) assert len(source) == 2 - _assert_round_trip(list(source), _samples()) + _assert_round_trip(list(source), _records()) def test_missing_root_raises(self, tmp_path: Path) -> None: with pytest.raises(FileNotFoundError): len(DirectorySource(path=tmp_path / "nope")) + def test_non_dict_write_raises(self, tmp_path: Path) -> None: + with pytest.raises(TypeError, match="expected a record dict"): + DirectorySink(path=tmp_path / "dir").write(np.zeros(3)) + + +class TestMetadataQueries: + def test_record_metadata_shape(self) -> None: + meta = record_metadata(_records()[0]) + assert meta["image"] == {"layout": "CHW"} + assert meta["sig"]["samplerate"] == 20e6 + assert meta["gain_db"] == {"value": -3.0} # plain scalar -> its "value" attr + assert meta["source_file"] == {"value": "a.iq"} + assert meta["window"] == {} # a plain ARRAY payload contributes no scalar metadata -class TestTypedQueries: def test_hdf5_scan_is_nested_and_payload_free(self, tmp_path: Path) -> None: path = tmp_path / "q.h5" sink = HDF5Sink(path=path, overwrite=True) with sink: - for s in _samples(): - sink.write(s) + for r in _records(): + sink.write(r) scans = list(scan_hdf5_metadata(path)) assert len(scans) == 2 _, meta = scans[0] @@ -177,29 +236,32 @@ def test_zarr_scan_nested(self, tmp_path: Path) -> None: path = str(tmp_path / "q.zarr") sink = ZarrGroupSink(path=path) sink.open() - for s in _samples(): - sink.write(s) + for r in _records(): + sink.write(r) scans = list(scan_zarr_metadata(path)) assert scans[1][1]["sig"]["samplerate"] == 1e6 - def test_where_field_attr_expression(self, tmp_path: Path) -> None: + def test_where_key_attr_expression(self, tmp_path: Path) -> None: path = tmp_path / "w.h5" sink = HDF5Sink(path=path, overwrite=True) with sink: - for s in _samples(): - sink.write(s) + for r in _records(): + sink.write(r) source = HDF5Source(path=path) source.open() fast = MetadataFilterSource(source=source, where="sig.samplerate > 1e7") assert len(fast) == 1 (match,) = list(fast) - assert isinstance(match, Sample) and match["sig"].samplerate == 20e6 + assert isinstance(match, dict) and match["sig"].samplerate == 20e6 - def test_full_iteration_fallback_on_typed_samples(self) -> None: - # A plain list source (no iter_metadata protocol) of Samples still filters. - filt = MetadataFilterSource(source=_samples(), where="image.layout == 'CHW'") + def test_full_iteration_fallback_on_records(self) -> None: + # A plain list source (no iter_metadata protocol) of record dicts still filters, + # via record_metadata — plain scalars addressable as .value. + filt = MetadataFilterSource(source=_records(), where="gain_db.value < 0") assert len(filt) == 1 + (match,) = list(filt) + assert match["gain_db"] == -3.0 def test_missing_attr_is_non_match(self) -> None: - filt = MetadataFilterSource(source=_samples(), where="sig.nonexistent > 0") + filt = MetadataFilterSource(source=_records(), where="sig.nonexistent > 0") assert len(filt) == 0 diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py index 8856e06..24d7067 100644 --- a/tests/test_typed_target_ops.py +++ b/tests/test_typed_target_ops.py @@ -1,22 +1,22 @@ -"""Typed-bag TWINS of the tensorization + target-shaping ops. +"""The tensorization + target-shaping ops over dict records. -Pins the native typed transforms that let a ``Sample`` classification pipeline build its -model INPUT tensor and its encoded TARGET ``Label`` without the legacy ``Sample`` path: +Pins the native transforms that build a classification pipeline's model INPUT array and its +encoded TARGET ``Label`` on plain record dicts: -* :class:`sampleflux.ops.torch.ToTensor` — array-bearing field → CHW-float ``Image`` item; -* :class:`sampleflux.ops.target.MetadataToTarget` — a field / attr value → a target ``Label``; +* :class:`sampleflux.ops.torch.ToTensor` — array-bearing key → a LIVE CHW-float ``torch.Tensor`` (a plain record value); +* :class:`sampleflux.ops.target.MetadataToTarget` — a key / attr value → a target ``Label``; * :class:`sampleflux.ops.target.EncodeTarget` / ``DecodeTarget`` — class-name ↔ class-id ``Label``. -Each twin REUSES its legacy op's math, so the twin's output is pinned byte-identical to a legacy -run on the equivalent ``Sample`` (parity). sampleflux-only — no waivefront import. +Each op REUSES its shared conversion helper, so the op output is pinned identical to the +helper (parity). sampleflux-only — no domain-package import. """ import numpy as np import pytest +import torch from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Label, Mask, Sample -from sampleflux.collate import typed_collate +from sampleflux import Image, Label, Mask, collate_records, item_data from sampleflux.ops.image import ConvertToImage from sampleflux.ops.target import DecodeTarget, EncodeTarget, MetadataToTarget from sampleflux.ops.torch import ToTensor, to_tensor @@ -33,69 +33,66 @@ def _hwc_uint8() -> np.ndarray: # ToTensor # --------------------------------------------------------------------------- # class TestToTensor: - def test_produces_chw_float_image_role_preserved(self) -> None: + def test_produces_chw_float_tensor(self) -> None: arr = _hwc_uint8() - out = ToTensor()(Sample({"image": Image(arr)}, roles={"image": "input"})) - img = out["image"] - assert isinstance(img, Image) - assert img.layout == "CHW" - payload = np.asarray(img) - assert payload.shape == (3, 4, 5) # HWC -> CHW - assert payload.dtype == np.float32 - assert payload.max() <= 1.0 # normalized - assert out.role_of("image") == "input" # replaced in place -> role preserved + out = ToTensor()({"image": Image(arr)}) + tensor = out["image"] + assert isinstance(tensor, torch.Tensor) + assert tuple(tensor.shape) == (3, 4, 5) # HWC -> CHW + assert tensor.dtype == torch.float32 + assert float(tensor.max()) <= 1.0 # normalized def test_parity_with_to_tensor_helper(self) -> None: arr = _hwc_uint8() - typed = ToTensor()(Sample({"image": Image(arr)})) + out = ToTensor()({"image": Image(arr)}) expected = to_tensor(arr).numpy() - assert np.array_equal(np.asarray(typed["image"]), expected) + assert np.array_equal(np.asarray(out["image"]), expected) def test_parity_no_normalize(self) -> None: arr = _hwc_uint8() - typed = ToTensor(normalize=False)(Sample({"image": Image(arr)})) + out = ToTensor(normalize=False)({"image": Image(arr)}) expected = to_tensor(arr, normalize=False).numpy() - assert np.array_equal(np.asarray(typed["image"]), expected) - - def test_payload_is_numpy_not_live_tensor(self) -> None: - # NDArrayItem coerces its payload via np.asarray, so an Image CANNOT hold a live tensor; - # the stored CHW-float payload is a numpy array (values identical to the legacy tensor). - from sampleflux.bag.items import item_data - - out = ToTensor()(Sample({"image": Image(_hwc_uint8())})) - assert isinstance(item_data(out["image"]), np.ndarray) - - def test_new_output_field_tagged_input(self) -> None: + assert np.array_equal(np.asarray(out["image"]), expected) + + def test_output_is_a_plain_live_tensor(self) -> None: + # The record model holds arbitrary values: the tensor rides AS-IS (no Image wrap — an + # NDArrayItem coerces via np.asarray and cannot hold a live tensor). item_data passes + # a plain value through unchanged. + out = ToTensor()({"image": Image(_hwc_uint8())}) + assert isinstance(out["image"], torch.Tensor) + assert not isinstance(out["image"], Image) + assert item_data(out["image"]) is out["image"] + + def test_new_output_key_keeps_source(self) -> None: arr = _hwc_uint8() - out = ToTensor(output="tensor")(Sample({"image": Image(arr)}, roles={"image": "input"})) + out = ToTensor(output="tensor")({"image": Image(arr)}) assert np.asarray(out["tensor"]).shape == (3, 4, 5) - assert out.role_of("tensor") == "input" - # original field left as-is (HWC uint8) + # original entry left as-is (HWC uint8) assert np.asarray(out["image"]).shape == (4, 5, 3) def test_explicit_field(self) -> None: - s = Sample({"a": Mask(np.zeros((2, 2), dtype=np.uint8)), "b": Image(_hwc_uint8())}) - out = ToTensor(field="b")(s) + rec = {"a": Mask(np.zeros((2, 2), dtype=np.uint8)), "b": Image(_hwc_uint8())} + out = ToTensor(field="b")(rec) assert np.asarray(out["b"]).shape == (3, 4, 5) - def test_default_picks_first_array_field(self) -> None: - s = Sample({"lbl": Label("cat"), "image": Image(_hwc_uint8())}) - out = ToTensor()(s) + def test_default_picks_first_array_key(self) -> None: + rec = {"lbl": Label("cat"), "image": Image(_hwc_uint8())} + out = ToTensor()(rec) assert np.asarray(out["image"]).shape == (3, 4, 5) def test_missing_explicit_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - ToTensor(field="nope")(Sample({"image": Image(_hwc_uint8())})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + ToTensor(field="nope")({"image": Image(_hwc_uint8())}) def test_no_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - ToTensor()(Sample({"lbl": Label("cat")})) + ToTensor()({"lbl": Label("cat")}) - def test_typed_collate_stacks_payloads(self) -> None: - # The typed collate stacks the CHW-float Image payloads into a batched array. - a = ToTensor()(Sample({"image": Image(_hwc_uint8())})) - b = ToTensor()(Sample({"image": Image(_hwc_uint8())})) - batch = typed_collate([a, b]) + def test_record_collate_stacks_payloads(self) -> None: + # The record collate stacks the CHW-float Image payloads into a batched array. + a = ToTensor()({"image": Image(_hwc_uint8())}) + b = ToTensor()({"image": Image(_hwc_uint8())}) + batch = collate_records([a, b]) assert np.asarray(batch["image"]).shape == (2, 3, 4, 5) @@ -104,134 +101,120 @@ def test_typed_collate_stacks_payloads(self) -> None: # --------------------------------------------------------------------------- # class TestMetadataToTarget: def test_promotes_label_value_to_target(self) -> None: - s = Sample({"class": Label("cat")}, roles={"class": "aux"}) - out = MetadataToTarget(field="class", output="target")(s) + out = MetadataToTarget(field="class", output="target")({"class": Label("cat")}) assert isinstance(out["target"], Label) assert out["target"].value == "cat" - assert out.role_of("target") == "target" def test_default_picks_first_label(self) -> None: - s = Sample({"image": Image(_hwc_uint8()), "y": Label("dog")}) - out = MetadataToTarget()(s) + rec = {"image": Image(_hwc_uint8()), "y": Label("dog")} + out = MetadataToTarget()(rec) assert out["target"].value == "dog" - assert out.role_of("target") == "target" def test_read_named_attribute(self) -> None: - # Read a carried attribute off a field (a value that rode as item-scoped metadata). - s = Sample({"y": Label("cat", classes=["cat", "dog"])}) - out = MetadataToTarget(field="y", key="classes", output="vocab")(s) + # Read a carried attribute off an item (metadata lives ON the value that owns it). + out = MetadataToTarget(field="y", key="classes", output="vocab")({"y": Label("cat", classes=["cat", "dog"])}) assert out["vocab"].value == ["cat", "dog"] def test_missing_attribute_raises(self) -> None: with pytest.raises(AttributeError, match="no attribute 'nope'"): - MetadataToTarget(field="y", key="nope")(Sample({"y": Label("cat")})) + MetadataToTarget(field="y", key="nope")({"y": Label("cat")}) def test_missing_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in sample"): - MetadataToTarget(field="nope")(Sample({"y": Label("cat")})) + with pytest.raises(ValueError, match="field 'nope' not in record"): + MetadataToTarget(field="nope")({"y": Label("cat")}) - def test_empty_sample_raises(self) -> None: - with pytest.raises(ValueError, match="sample is empty"): - MetadataToTarget()(Sample({})) + def test_empty_record_raises(self) -> None: + with pytest.raises(ValueError, match="record is empty"): + MetadataToTarget()({}) # --------------------------------------------------------------------------- # # EncodeTarget / DecodeTarget # --------------------------------------------------------------------------- # class TestEncodeDecodeTarget: - def test_encode_name_to_id_role_target(self) -> None: - out = EncodeTarget(mapping=_MAP)(Sample({"y": Label("cat")}, roles={"y": "target"})) + def test_encode_name_to_id(self) -> None: + out = EncodeTarget(mapping=_MAP)({"y": Label("cat")}) assert isinstance(out["y"], Label) assert out["y"].value == 0 - assert out.role_of("y") == "target" def test_encode_maps_every_name(self) -> None: for name in _MAP: - typed = EncodeTarget(mapping=_MAP)(Sample({"y": Label(name)})) - assert typed["y"].value == _MAP[name] + out = EncodeTarget(mapping=_MAP)({"y": Label(name)}) + assert out["y"].value == _MAP[name] def test_encode_preserves_classes_vocab(self) -> None: - out = EncodeTarget(mapping=_MAP)(Sample({"y": Label("dog", classes=list(_MAP))})) + out = EncodeTarget(mapping=_MAP)({"y": Label("dog", classes=list(_MAP))}) assert out["y"].value == 1 assert out["y"].classes == list(_MAP) - def test_encode_new_output_field(self) -> None: - out = EncodeTarget(mapping=_MAP, output="target_id")(Sample({"y": Label("fox")})) + def test_encode_new_output_key(self) -> None: + out = EncodeTarget(mapping=_MAP, output="target_id")({"y": Label("fox")}) assert out["target_id"].value == 2 - assert out.role_of("target_id") == "target" assert out["y"].value == "fox" # source left intact def test_encode_ignore_unknown(self) -> None: - out = EncodeTarget(mapping=_MAP, ignore_unknown=True, default=-1)(Sample({"y": Label("bird")})) + out = EncodeTarget(mapping=_MAP, ignore_unknown=True, default=-1)({"y": Label("bird")}) assert out["y"].value == -1 def test_encode_unknown_raises(self) -> None: with pytest.raises(KeyError): - EncodeTarget(mapping=_MAP)(Sample({"y": Label("bird")})) + EncodeTarget(mapping=_MAP)({"y": Label("bird")}) def test_encode_empty_mapping_raises_lazily(self) -> None: op = EncodeTarget() # constructible with no mapping (lazy) with pytest.raises(ValueError, match="at least one entry"): - op(Sample({"y": Label("cat")})) + op({"y": Label("cat")}) def test_decode_id_to_name(self) -> None: for cid in _INV: - typed = DecodeTarget(mapping=_INV)(Sample({"y": Label(cid)})) - assert typed["y"].value == _INV[cid] + out = DecodeTarget(mapping=_INV)({"y": Label(cid)}) + assert out["y"].value == _INV[cid] def test_encode_then_decode_round_trip(self) -> None: - s = Sample({"y": Label("dog")}) - encoded = EncodeTarget(mapping=_MAP)(s) + encoded = EncodeTarget(mapping=_MAP)({"y": Label("dog")}) assert encoded["y"].value == 1 decoded = DecodeTarget(mapping=_INV)(encoded) assert decoded["y"].value == "dog" def test_decode_empty_mapping_raises_lazily(self) -> None: with pytest.raises(ValueError, match="at least one entry"): - DecodeTarget()(Sample({"y": Label(0)})) + DecodeTarget()({"y": Label(0)}) def test_encode_non_label_field_raises(self) -> None: with pytest.raises(TypeError, match="expected a Label"): - EncodeTarget(mapping=_MAP, field="image")(Sample({"image": Image(_hwc_uint8())})) + EncodeTarget(mapping=_MAP, field="image")({"image": Image(_hwc_uint8())}) def test_encode_no_label_field_raises(self) -> None: with pytest.raises(ValueError, match="no Label field"): - EncodeTarget(mapping=_MAP)(Sample({"image": Image(_hwc_uint8())})) + EncodeTarget(mapping=_MAP)({"image": Image(_hwc_uint8())}) # --------------------------------------------------------------------------- # -# End-to-end typed classification input/target path (sampleflux-only). +# End-to-end classification input/target path (sampleflux-only). # --------------------------------------------------------------------------- # -def test_typed_classification_input_and_target_chain() -> None: - # Source-shaped bag: an HWC image (role input) + a class-NAME label (role target). - sample = Sample( - {"image": Image(_hwc_uint8()), "class": Label("cat", classes=list(_MAP))}, - roles={"image": "input", "class": "target"}, - ) - # Build the model INPUT tensor (CHW float) and the encoded TARGET id — no legacy Sample. - out = EncodeTarget(mapping=_MAP, field="class")(ToTensor(field="image")(sample)) - - # Input field: a CHW-float Image tagged input. - assert isinstance(out["image"], Image) - assert out["image"].layout == "CHW" - assert np.asarray(out["image"]).shape == (3, 4, 5) - assert np.asarray(out["image"]).dtype == np.float32 - assert out.role_of("image") == "input" - assert out.inputs().keys() == {"image"} - - # Target field: an int-id Label tagged target. +def test_classification_input_and_target_chain() -> None: + # Source-shaped record: an HWC image + a class-NAME label — key names carry meaning. + record = {"image": Image(_hwc_uint8()), "class": Label("cat", classes=list(_MAP))} + # Build the model INPUT (CHW float) and the encoded TARGET id. + out = EncodeTarget(mapping=_MAP, field="class")(ToTensor(field="image")(record)) + + # Input entry: a LIVE CHW-float tensor (a plain record value). + assert isinstance(out["image"], torch.Tensor) + assert tuple(out["image"].shape) == (3, 4, 5) + assert out["image"].dtype == torch.float32 + + # Target entry: an int-id Label carrying the vocabulary. assert isinstance(out["class"], Label) assert out["class"].value == 0 - assert out.role_of("class") == "target" - assert out.targets().keys() == {"class"} + assert out["class"].classes == list(_MAP) def test_convert_then_tensor_chain() -> None: - # A raw 2-D array field runs ConvertToImage -> ToTensor into a CHW-float input. + # A raw 2-D array entry runs ConvertToImage -> ToTensor into a CHW-float input. arr = np.arange(6 * 4).reshape(6, 4).astype(np.float32) - out = ToTensor(field="image")(ConvertToImage(colormap="gray")(Sample({"spec": Mask(arr)}))) - assert out["image"].layout == "CHW" - assert np.asarray(out["image"]).shape == (3, 6, 4) + out = ToTensor(field="image")(ConvertToImage(colormap="gray")({"spec": Mask(arr)})) + assert isinstance(out["image"], torch.Tensor) + assert tuple(out["image"].shape) == (3, 6, 4) # --------------------------------------------------------------------------- # From 575856d38643958f149431be8646c26b1f7566a1 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 26 Jul 2026 03:48:33 +0200 Subject: [PATCH 041/102] =?UTF-8?q?feat:=20record-model=20review=20hardeni?= =?UTF-8?q?ng=20=E2=80=94=20open=20op-family=20registry,=20live-tensor=20T?= =?UTF-8?q?oTensor,=20doc/test=20truth=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review-driven refinements on top of the record-model conversion: - core: the op-family dispatch is an OPEN registry (register_op_family / registered_op_families, built-ins registered through the same API, last-registered-first shadowing, spawn workers rebuild via pickled module-level matchers/invokers shipped by the parallel routes). - ToTensor emits the LIVE CHW-float torch.Tensor as a plain record value (typed-bag constraint reversed; architecture record updated). - MetadataToTarget DELETED (role-era twin of CopyField, zero users); SampleSinkOp renamed RecordSinkOp; Switch.selector renamed select. - FormulaOp: sandboxed array reducers amax/amin/mean/std/median (numpy C-reductions lazy-import through the caller frame — attribute form documented as not guaranteed). - Flow YAML truth pass: bind: requires the plain-mapping (op:) step form (a nested mapping under a !class: marker is consumed by confluid as addressed config — pinned); graph.md/configure.md examples rewritten to execution-verified shapes; stale Flux-rejects-markers claim fixed. - Docs: architecture.md restructured (system map + 5 numbered records, superseded records deleted with contracts folded into successors); runnable.md + workflow.md written with verified examples; collate detection walkthrough; type-interface (handles/consumes/produces) section; Enable overlay.enable pattern documented + pinned. - Examples: workflow_pipeline.py (resume-safe Sequence/Conditional/ Switch, never-built !lazy: guarantee asserted), storage_roundtrip.py (3 sink/source pairs + metadata-only query); stale pre-record .zarr/ .h5/dir_store artifacts and orphaned dataset_split.yaml removed. - Tests: 421 green — new pins for the registry, YAML binds, reducers, Enable toggles, collate registry, live-tensor ToTensor. --- AGENTS.md | 4 +- README.md | 2 + docs/architecture.md | 435 ++++++++++++--------------------- docs/augmentation.md | 9 + docs/configure.md | 66 ++++- docs/graph.md | 23 +- docs/kinds.md | 2 +- docs/record-model.md | 101 +++++++- docs/runnable.md | 104 ++++++++ docs/storage.md | 4 + docs/workflow.md | 62 +++++ examples/dataset_split.yaml | 53 ---- examples/storage_roundtrip.py | 92 +++++++ examples/workflow_pipeline.py | 140 +++++++++++ sampleflux/__init__.py | 4 +- sampleflux/core.py | 167 ++++++++++--- sampleflux/ops/__init__.py | 17 +- sampleflux/ops/configure.py | 2 +- sampleflux/ops/enable.py | 26 +- sampleflux/ops/formula.py | 9 +- sampleflux/ops/parallel.py | 5 +- sampleflux/ops/sink.py | 10 +- sampleflux/ops/target.py | 59 ----- sampleflux/runnable.py | 28 ++- sampleflux/workflow.py | 20 +- tests/test_categories.py | 24 +- tests/test_enable.py | 69 ++++++ tests/test_node_docs.py | 2 - tests/test_op_families.py | 107 ++++++++ tests/test_typed_collate.py | 54 +++- tests/test_typed_flow.py | 62 +++++ tests/test_typed_target_ops.py | 37 +-- tests/test_workflow.py | 8 +- 33 files changed, 1252 insertions(+), 555 deletions(-) create mode 100644 docs/runnable.md create mode 100644 docs/workflow.md delete mode 100644 examples/dataset_split.yaml create mode 100644 examples/storage_roundtrip.py create mode 100644 examples/workflow_pipeline.py create mode 100644 tests/test_enable.py diff --git a/AGENTS.md b/AGENTS.md index 1f6e614..5dcab0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,7 @@ - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`sampleflux.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `sampleflux.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. - **The RECORD Is THE Data Model (2026-07-25):** A sample is a **PLAIN `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `sampleflux.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`sampleflux.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `sampleflux.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from sampleflux import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `sampleflux/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `sampleflux.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). NEVER add a wrapper/adapter class for a library — supporting a NEW library family means adding a new branch in `_apply_op` (an MRO module-name matcher + the library's native calling convention), nothing else. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Flux._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel). +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `sampleflux.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Flux._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-sample flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. - **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. @@ -25,7 +25,7 @@ - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The sampleflux buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` / `FlowGraph` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. FluxStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`). The sampleflux groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `MetadataToTarget` / `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`sampleflux.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`sampleflux.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`SampleSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. FluxStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`). The sampleflux groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`sampleflux.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`sampleflux.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. - **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). - **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) diff --git a/README.md b/README.md index 0bee9fe..e141b7f 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ ops: | [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap` | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | +| [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `sampleflux run`), the `@entrypoint` task/role markers with a worked example, `TorchRunner` / `ProgressReporting` | +| [docs/workflow.md](docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | | [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | diff --git a/docs/architecture.md b/docs/architecture.md index ed55a6f..a9e27b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,78 +1,113 @@ -# Architecture decisions - -The *why* behind sampleflux's non-obvious module boundaries and mechanisms. The user-facing -documentation ([README](../README.md), the per-topic `docs/*.md`) shows **how to use** each surface; -this document records **why the surface is shaped the way it is** — the context, the decision, and -the consequences — so a reader who asks "why does this module exist?" finds the answer here instead -of reverse-engineering it from git history. - -Each entry is a short decision record: **Context → Decision → Consequences → Example → What you may -change**. When a change alters one of these mechanisms, update its record in the same change (see -the workspace `AGENTS.md` → "Architecture Decisions Are Documented"). Superseded records are kept -as history, banner-marked with a pointer to their successor. +# SampleFlux architecture + +The *why* behind sampleflux's module boundaries and mechanisms. The user-facing documentation +([README](../README.md), the per-topic `docs/*.md`) shows **how to use** each surface; this +document records **why the surface is shaped the way it is** — so a reader who asks "why does this +module exist?" finds the answer here instead of reverse-engineering it from git history. + +Maintenance rules: + +- Every record keeps the five elements **Context → Decision → Consequences → Example → What you + may change**, dated (see the workspace `AGENTS.md` → "Architecture Decisions Are Documented"). +- A change that alters a mechanism updates its record **in the same change**. +- A superseded decision is **deleted**, not archived: whatever it still binds is folded into its + successor record. History lives in git, not here. + +## The system at a glance + +| Layer | Modules | What it is | Where the *why* lives | +|---|---|---|---| +| Data model | `items.py`, `io.py` | A sample is a plain `dict` of typed values; one codec serializes any value | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Native ops | `transform.py`, `dispatch.py`, `ops/*` | Type-dispatched `Transform`s (kernels, `field=`) + structural/compose/context ops | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Library interop | `core._apply_op`, `register_op_family` | External libraries run as-is via the op-family dispatch — no adapters | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Engines | `core.py` (`Flux`/`JointFlux`), `flow.py` (`FlowGraph`) | One op-application chokepoint, four routes; a named-step graph engine with pinned lowering parity | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17), [§5](#5-the-engines-own-callable-wrappers-live-in-corepy-2026-07-20) | +| Graph wiring | `context.py`, `ops/context.py` | Fan-out/fan-in/cross-branch values on the plain sequential engine | [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17) | +| Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17) | +| Storage & query | `storage/*` | The `typedrecord-v1` key-group layout over the codec; metadata scans without array loads | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) (contracts) + [storage.md](storage.md) | +| Introspection & serialization | `discovery.py` | Callable↔string identity + registration-free module scans | [§4](#4-callablestring-serialization--passive-introspection-samplefluxdiscovery-2026-07-20) | +| Runnables & workflows | `runnable.py`, `workflow.py`, `processing.py`, `cli.py` | `run()` objects, entry-point markers, combinators, the one `sampleflux run` runner | no record yet — [runnable.md](runnable.md), [workflow.md](workflow.md) | --- -## One type-dispatched op engine — plain-dict records, libraries as-is (2026-07-25) +## 1. The record data model and the type-dispatched op engine (2026-07-25) ### Context -The previous data model (the typed-bag `Sample`, recorded below and now superseded) got the item -half right — typed values owning their metadata — but wrapped them in a bespoke container with -per-key role tags. That container was the friction point: every external library needed an adapter -before it could touch a sample (`coerce_transform` + a matcher/factory registry + two adapter -classes + ~170 GENERATED per-transform op wrappers, all maintenance surface), the role tags -duplicated what key names already say (`"mask"` *is* the mask), and dict-native libraries — -torchvision `transforms.v2` walks dicts, albumentations takes named kwargs — were kept at arm's -length from a carrier they could have consumed directly. Meanwhile a second op-authoring surface -(the adapter/generated families) competed with the native type-dispatched `Transform`, so "where -does augmentation come from?" had three answers. +The rejected alternative was a bespoke sample container: typed items (that part was right) +wrapped in a `Sample` class with per-key role tags, plus an adapter registry that wrapped every +external library transform in an adapter object before it could touch a sample (two adapter +classes, a coercion registry, and ~170 generated per-transform wrapper ops — all maintenance +surface). The container was the friction point: role tags duplicated what key names already say +(`"mask"` *is* the mask), and dict-native libraries — torchvision `transforms.v2` walks dicts, +albumentations takes named kwargs — were kept at arm's length from a carrier they could have +consumed directly. With two op-authoring surfaces (native transforms vs the adapter/generated +families), "where does augmentation come from?" had three answers. ### Decision Collapse to ONE carrier and ONE op engine: -- **A sample is a plain `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **typed values** - (`Image`/`Mask`/`Regions`/`Label`, base `NDArrayItem`; open registry `register_item`; uniform - payload accessors `item_data`/`with_data`). No container class, no roles, no `primary()`: - **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"class"`), and a scalar side value - is just another key. Metadata is attrs on the typed value (`Image.layout`, `Label.classes`) or - more dict keys (`"samplerate": 30.72e6`). -- **Native ops are type-dispatched `Transform`s** (`sampleflux/transform.py`): `get_params(record)` - draws shared parameters ONCE per record, per-type kernels (`@MyOp.kernel(ItemType)`, MRO-aware - registry in `sampleflux/dispatch.py`) apply to every handled value, `field=` pins one key. The - second sanctioned shape — type-CHANGING ops (`Threshold`, `ConvertToImage`, the target ops) — - overrides `__call__`. +- **A sample is a plain `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **typed + values** (`Image`/`Mask`/`Regions`/`Label`, base `NDArrayItem`; open registry `register_item`; + uniform payload accessors `item_data`/`with_data`). No container class, no roles, no + `primary()`: **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"class"`), and a + scalar side value is just another key. Metadata is attrs on the typed value (`Image.layout`, + `Label.classes`) or more dict keys (`"samplerate": 30.72e6`). Items are deliberately NOT + confluid-`@configurable`: an ndarray subclass builds through `__new__`, which fights the + `__init__` validation wrap — they live in their own registry. +- **Native ops are type-dispatched `Transform`s** (`sampleflux/transform.py`): + `get_params(record)` draws shared parameters ONCE per record, per-type kernels + (`@MyOp.kernel(ItemType)`, MRO-aware registry in `sampleflux/dispatch.py`) apply to every + handled value, `field=` pins one key. The second sanctioned shape — type-CHANGING ops + (`Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, `ConnectedComponents`: + `Mask`→`Regions`, the target ops) — overrides `__call__`, resolves its source by an explicit + `field=` or the first value of the natural type, and raises a `ValueError` naming the record's + keys on every miss. - **External libraries run AS-IS through the engine's op-family dispatch** - (`sampleflux.core._apply_op`, three branches): an albumentations op receives exactly its own kwarg - vocabulary (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record; one - call = one joint draw; array outputs re-wrapped in the incoming `NDArrayItem` type so - `Image`/`Mask` survive); a torchvision-v2 op is called on the dict as-is; everything else is - `op(record)` with `None` = drop. Family detection is by MRO module name — no eager imports, no - adapters, no generated wrappers. Box-carrying augmentation is the library's own - `A.Compose(..., bbox_params=...)`; seeding is the libraries' own mechanisms. -- **`Pipeline(transforms=[...])`** (`sampleflux/transform.py`) is THE sequential composer — - `TransformChain` was deleted; every composing op routes inner ops through `_apply_op`. -- **Storage is the record key-group layout** (`typedrecord-v1`): everything serializes through the - `sampleflux/io.py` codec; plain values ride the `"plain"` tag; NO backward compatibility with the - pre-record layout (an old/untagged store raises via `storage/base.py::require_record_format` — - an explicit decision: re-generate, don't accrete legacy readers). + (`sampleflux.core._apply_op`): an albumentations op receives exactly its own kwarg vocabulary + (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record; one call = + one joint draw; array outputs re-wrapped in the incoming `NDArrayItem` type so `Image`/`Mask` + survive); a torchvision-v2 op is called on the dict as-is; everything else is `op(record)` with + `None` = drop. **The families are an open registry** — `register_op_family(name, matcher, + invoker)`; the built-ins register through the same API, dispatch checks last-registered first, + and matchers/invokers are module-level functions so spawn workers rebuild the registry. Family + detection is by MRO module name — no eager imports, no adapters, no generated wrappers. + Box-carrying augmentation is the library's own `A.Compose(..., bbox_params=...)`; seeding is + the libraries' own mechanisms. +- **`Pipeline(transforms=[...])`** (`sampleflux/transform.py`) is THE sequential composer; every + composing op routes inner ops through `_apply_op`, so bare library transforms nest anywhere a + native op does. +- **Tensors are plain values.** `ToTensor` writes a LIVE CHW-float `torch.Tensor` under its key — + a record value can be anything (`collate_records` stacks tensors natively, storage converts via + `to_numpy` on write, a downstream tv2 op transforms them as-is). An `Image` itself cannot hold + a tensor (`NDArrayItem.__new__` runs `np.asarray`); a typed tensor ITEM base is a tracked + follow-up (root `TASKS.md`). +- **Storage is the record key-group layout** (`typedrecord-v1`): everything serializes through + the `sampleflux/io.py` codec; plain values ride the `"plain"` tag; NO backward compatibility + with the pre-record layout (an old/untagged store raises via + `storage/base.py::require_record_format` — an explicit decision: re-generate, never accrete + legacy readers). - **Projection and collation are key-addressed**: `project(source, keys)` / `iter_key` / `num_classes(key="class")`; the collate registry's default is `"record"` = `collate_records`. ### Consequences -- Zero adapter surface: the two adapter classes, the coercion registry, and both generated op - families are gone; a new library version's transforms are available the moment the library is — - nothing to regenerate. -- Cross-key consistency is the LIBRARY's own joint draw (albumentations Compose / tv2's dict walk) - for augmentation, and `get_params`-once for native ops — one mechanism per world, both automatic. +- Zero adapter surface: a new library version's transforms are available the moment the library + is — nothing to regenerate; a NEW library family is one `register_op_family` call from any + package. +- Cross-key consistency is the LIBRARY's own joint draw (albumentations Compose / tv2's dict + walk) for augmentation, and `get_params`-once for native ops — one mechanism per world, both + automatic. - YAML needs no special forms: a bare `!class:albumentations.HorizontalFlip {p: 0.5}` sits in an `ops:` list like any native op (deferred markers flow at route entry). - The albumentations vocabulary is load-bearing: a value augments only if it rides one of the library's key names — routing is an explicit `RenameField`, never engine magic. -- Anything that used `Sample`, roles, `primary()`, `typed_collate`, `ProjectionField`, or a - `typedsample-v1` store must migrate — there are deliberately no aliases and no legacy read path. +- **Contracts that outlive refactors:** the `(row_min, row_max, col_min, col_max)` inclusive + integer bin-box order of `connected_component_bboxes` (a downstream back-projection reads + exactly that order); the `typedrecord-v1` tag + no-back-compat rule; the albumentations key + vocabulary; the `"module:qualname"` callable-path format (§4). +- Anything that used the old container API must migrate — there are deliberately no aliases and + no legacy read path. ### Example @@ -105,15 +140,18 @@ ops: - **A new item type** — one class + `@register_item` (array-backed: subclass `NDArrayItem`, declare `_item_attrs`); usage in [record-model.md](record-model.md). - **A new per-type behaviour for an existing op** — `@Op.kernel(ItemType)`, no core edit. -- **A new library family** — a new branch in `core._apply_op` (MRO module-name matcher + the - library's native calling convention). Never an adapter/wrapper class; update this record when a - branch is added. +- **A new library family** — one `register_op_family(name, matcher, invoker)` call from any + package (MRO module-name matcher + the library's native calling convention; module-level + functions so spawn workers rebuild the registry). Never an adapter/wrapper class. The + built-ins register through the same API; dispatch is last-registered-first, so + forks/extensions shadow their base library by registering later. Usage: + [record-model.md](record-model.md) → "A new library family". - **The `typedrecord-v1` tag and the no-back-compat rule are contracts** — changing the on-disk layout means a NEW tag and a re-generation story, never a silent dual-read path. --- -## Batching is two-stage; collation is a pluggable registry (`sampleflux.collate`, 2026-07-17, updated 2026-07-25) +## 2. Batching is two-stage; collation is a pluggable registry (`sampleflux.collate`, 2026-07-17) ### Context @@ -121,8 +159,7 @@ Turning N pipeline items into one batched carrier has two distinct halves: 1. **Grouping** — the engine yields groups of N items (`Flux.batch` / `FlowGraph.batch` yield `list`s, and a torch `DataLoader` hands its `collate_fn` a list). -2. **Stacking** — a *collate function* turns one group into one batched carrier (stacked tensors + - batched metadata). +2. **Stacking** — a *collate function* turns one group into one batched carrier. The engine owns grouping; it must NOT own stacking, because stacking is task-shaped: historically every consuming project shipped its own task collate (classification, segmentation, detection), @@ -132,38 +169,35 @@ and divergent batched-metadata conventions emerged between them. `sampleflux/collate.py` is a **pluggable registry of collate functions keyed by representation**: `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`, where an omitted key -uses the default **`"record"`** collate (`collate_records`) — N plain record dicts into ONE batched -record: per key, typed values encode through the `sampleflux/io.py` codec, payloads stack +uses the default **`"record"`** collate (`collate_records`) — N plain record dicts into ONE +batched record: per key, typed values encode through the `sampleflux/io.py` codec, payloads stack (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into one batched item of the same type), and a `"plain"`-tagged value batches as the plain list. Batches must be key-homogeneous — a mismatch -raises. Consuming projects may register task aliases (`"yolo"`, `"segmentation"`, …) -**additively**; re-registering a key deliberately overwrites (logged at debug) so a consumer can -replace a default. The divergent consumer conventions were deliberately NOT unified here — the -registry is an addressable home consumers opt into, not a forced migration. - -### Primary intended consumer: the MCP tool surface +raises. Consuming projects register task aliases (`"detection"`, `"yolo"`, …) **additively**; +re-registering a key deliberately overwrites so a consumer can replace a default. The divergent +consumer conventions were deliberately NOT unified here — the registry is an addressable home +consumers opt into, not a forced migration. The open, string-keyed half of the registry exists first and foremost for **AI-callable tools** (the workspace converges on an MCP tool surface — see the root `AGENTS.md` end-goal): a JSON tool argument can carry `"collate": "yolo"` but never a Python function object, and a tool schema can -offer the legal values only if the set is discoverable at runtime (`registered_collates()`). The -registry is the collate layer's MCP-readiness — a stable, JSON-serializable, enumerable name per -batch layout. In ordinary Python (and in YAML via a dotted `!ref:` to the function), passing the -collate function directly remains the normal path; the registry never replaces it. +offer the legal values only if the set is discoverable at runtime (`registered_collates()`). In +ordinary Python (and in YAML via a dotted `!ref:` to the function), passing the collate function +directly remains the normal path. ### Consequences - The engine stays task-agnostic: sampleflux stacks by key + item type, never - classification/detection/… + classification/detection/…. - Item metadata batches deterministically: per-record attrs become lists on the ONE batched item - (`batch["image"].layout == ["HWC", "HWC", ...]`), plain values become plain lists — there is no - second batched-metadata convention in this package. -- One addressable lookup (`get_collate("yolo")`) replaces scattered cross-package imports — once a - consumer registers. Registration happens at module import, so a key exists only after its - defining module has been imported. -- **Current usage:** only the `"record"` default is registered here; the open registration surface - is capacity held for the MCP tool surface above. + (`batch["image"].layout == ["HWC", "HWC", ...]`), plain values become plain lists — there is + no second batched-metadata convention in this package. +- A task whose batch shape the generic rules cannot express (detection's ragged per-record + boxes) opts OUT entirely and emits its model family's native contract — see the worked + example in [record-model.md](record-model.md) → "Batching". +- Registration happens at module import, so a key exists only after its defining module has been + imported. ### Example @@ -192,28 +226,27 @@ loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("yolo")) ### What you may change (and where it's documented) -- **Plugging in your own batch layout** is the supported extension point — decorate a function with - `@register_collate("your-key")` and select it via `get_collate`/`collate`. Usage lives in - [kinds.md](kinds.md). +- **Plugging in your own batch layout** is the supported extension point — decorate a function + with `@register_collate("your-key")` and select it via `get_collate`/`collate`. Usage: + [kinds.md](kinds.md); the detection walkthrough: [record-model.md](record-model.md). - **Changing the default collate's semantics** (how `"record"` stacks, the attrs-become-lists convention) is an architectural change: every batch consumer depends on it. Update this record and the sampleflux `AGENTS.md` metadata mandate together. --- -## The per-record Context is an ambient wiring plane (`sampleflux.context`, 2026-07-17) +## 3. The per-record Context is an ambient wiring plane (`sampleflux.context`, 2026-07-17) ### Context Graph-shaped pipelines — fan-out, fan-in, cross-branch values — need somewhere to hold a value between the op that produces it and the op that consumes it. The obvious candidate, extra keys on -the record itself, was rejected: the record is the carrier that **persists** — it flows into sinks, -crosses process boundaries, and is the sample's serialized identity — while wiring data is +the record itself, was rejected: the record is the carrier that **persists** — it flows into +sinks, crosses process boundaries, and is the sample's serialized identity — while wiring data is transient scaffolding that should be gone by the end of a well-formed graph. Three constraints shaped the mechanism: ops keep the plain `__call__(record)` signature (no threading a context parameter through every op), the executor stays a bare `for op in ops` loop (graphs run on the -*plain sequential engine*), and a linear pipeline's behavior — its records, byte-for-byte — must be -completely untouched. +*plain sequential engine*), and a linear pipeline's records must stay byte-for-byte untouched. ### Decision @@ -222,25 +255,24 @@ creates one fresh `Context` per source item and activates it around the op loop `contextvars.ContextVar`; the six wiring ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields` in `sampleflux.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature change anywhere. Deliberate semantics: cells are stored **by reference** and copy-on-read is the -*reading* op's decision (`Use` deep-copies unless `drop` frees the cell = move); a missing cell on -read or delete **raises loudly** with the live-cell list (a liveness bug must never pass +*reading* op's decision (`Use` deep-copies unless `drop` frees the cell = move); a missing cell +on read or delete **raises loudly** with the live-cell list (a liveness bug must never pass silently); 1→N expansion children get `Context.copy()` (shallow — independent cell *sets*, shared values); cells may NOT cross a stream-level op boundary (`Parallel` raises on live cells — each inner chain gets its own contexts). A `Context` is never `@configurable` and never appears in YAML — it is pure runtime plumbing. The public surface is two-tier by design: the `Context` class -is a package-root export, while `activate`/`current`/`require` stay module-qualified -(`sampleflux.context.…`) — reachable, but visibly plumbing. `FlowGraph` deliberately does NOT use -this module: its named-step documents give the compiler full knowledge of cell lifetimes, so it -manages its own per-record env directly, held to the context-op semantics by the pinned -flow⇄ops execution-parity contract. +is a package-root export, while `activate`/`current`/`require` stay module-qualified — reachable, +but visibly plumbing. `FlowGraph` deliberately does NOT use this module: its named-step documents +give the compiler full knowledge of cell lifetimes, so it manages its own per-record env +directly, held to the context-op semantics by the pinned flow⇄ops execution-parity contract. ### Consequences - A plain sequential `ops:` list executes a real fan-out/fan-in graph — which is exactly what graph exporters (a visual canvas, the `flow:` compiler) lower to, so ONE executor serves both linear and graph pipelines. -- Linear pipelines are provably untouched: no context op ⇒ the Context is created and never used; - the record-byte-identical invariant is pinned in the record-model suite under `tests/`. +- Linear pipelines are provably untouched: no context op ⇒ the Context is created and never + used; the record-byte-identical invariant is pinned in the record-model suite under `tests/`. - Spawn-parallelism is safe by construction: contexts are created *inside* the worker and never pickled or shared across processes. - Ambient state cuts both ways: running an op list containing context ops *outside* an engine @@ -278,9 +310,9 @@ with activate(Context()): ### What you may change (and where it's documented) -- **Writing a custom wiring op** is the supported extension point: call - `require("YourOpName")` inside `__call__`, follow the by-reference/copy-on-read discipline, and - free cells you consume. Usage of the six built-in ops lives in [graph.md](graph.md). +- **Writing a custom wiring op** is the supported extension point: call `require("YourOpName")` + inside `__call__`, follow the by-reference/copy-on-read discipline, and free cells you consume. + Usage of the six built-in ops lives in [graph.md](graph.md). - **Keep the surface narrow.** Don't root-export `activate`/`current`/`require`, and don't grow `Context` into a general blackboard — anything that should *persist with the record* belongs in the record itself, not in a cell. @@ -291,7 +323,7 @@ with activate(Context()): --- -## Callable↔string serialization + passive introspection (`sampleflux.discovery`, recorded 2026-07-20) +## 4. Callable↔string serialization + passive introspection (`sampleflux.discovery`, 2026-07-20) ### Context @@ -299,11 +331,11 @@ Two workspace mandates — *Serialization Symmetry* (every pipeline round-trips YAML) and *Passive Introspection* (tools discover pipeline pieces without hand-written definitions) — need a bridge the Confluid registry deliberately does not provide. The registry is a **curated, opt-in catalog**: classes *and* builder functions participate, but only after an -explicit `@configurable`/`register()` (the Registry Discipline mandate), keyed by -name/category/task/role, resolving *strings → callables* for config materialization. What it does -NOT do: produce a string **from** a live callable (the dump direction a bare-function value like a -mapped transform needs), resolve a callable out of a plain `.py` script or `__main__`, or walk a -module to introspect every callable *defined in it* — registered or not. +explicit `@configurable`/`register()`, keyed by name/category/task/role, resolving *strings → +callables* for config materialization. What it does NOT do: produce a string **from** a live +callable (the dump direction a bare-function value like a mapped transform needs), resolve a +callable out of a plain `.py` script or `__main__`, or walk a module to introspect every callable +*defined in it* — registered or not. ### Decision @@ -314,15 +346,13 @@ module to introspect every callable *defined in it* — registered or not. `resolve_callable(path)` back to the live object (module import, `.py`-file load, or an already-callable passthrough). - **Introspection** — `introspect_callable(fn)` → a JSON-serializable schema (path, name, doc, - per-parameter type/default/required), and - `scan_module(module_or_py)` applying it to every callable *defined in* a module - (`__module__`-filtered, so imports don't leak in). + per-parameter type/default/required), and `scan_module(module_or_py)` applying it to every + callable *defined in* a module (`__module__`-filtered, so imports don't leak in). Curated discovery (MCP form-specs, task/category option pickers) deliberately does **not** use -this module — it builds on the Confluid registry, which registers classes AND builder functions, -opt-in by name. This module is the **registration-free complement**: the two surfaces answer -different questions — `scan_module` reflects over *a module, no curation required*; the registry -resolves *a curated name/category*. +this module — it builds on the Confluid registry. The two surfaces answer different questions: +`scan_module` reflects over *a module, no curation required*; the registry resolves *a curated +name/category*. ### Consequences @@ -338,10 +368,9 @@ resolves *a curated name/category*. for that case). - **One acknowledged overlap**: `resolve_callable`'s plain module-import branch resolves the same importable-function targets confluid's `resolve_class` module-path branch / `!ref:` grammar can - — two spellings of one job (`"module:qualname"` here vs `"module.attr"` there). The - non-overlapping remainder (path *production* via `get_callable_path`, `.py`-file and `__main__` - handling, module scans) is why the module exists; whether the resolution half should delegate - to confluid is a tracked follow-up in the root `TASKS.md`. + — two spellings of one job. The non-overlapping remainder (path *production*, `.py`-file and + `__main__` handling, module scans) is why the module exists; whether the resolution half should + delegate to confluid is a tracked follow-up in the root `TASKS.md`. ### Example @@ -362,12 +391,12 @@ schemas = scan_module("sampleflux.ops.numpy") # one JSON schema per op defined - **Adding a string-callable knob to your own class**: reuse `resolve_callable` (the `WrappedOp.f` pattern) — never write a bespoke import dance. - **The `"module:qualname"` format and the module-local scan filter are contracts** — serialized - pipelines and node bridges depend on both; changing either is an architectural change that - must update this record. + pipelines and node bridges depend on both; changing either is an architectural change that must + update this record. --- -## The engine's own callable wrappers live in `core.py` (`FilterOp`/`WrappedOp`/`JointFlux`, recorded 2026-07-20) +## 5. The engine's own callable wrappers live in `core.py` (2026-07-20) ### Context @@ -398,8 +427,8 @@ off visual canvases. - `WrappedOp` is a package-root export (the public "lift a plain function" surface, and its stored-string `f` is the reference use of the discovery serialization half); `FilterOp` is not root-exported (normally reached via `Flux.filter`; importable as `sampleflux.core.FilterOp`). -- `JointFlux` is YAML-addressable (`!class:sampleflux.core.JointFlux()`) and canvas-composable - as an engine node; its indexable counterpart for raw sources is `ConcatSource`. +- `JointFlux` is YAML-addressable (`!class:sampleflux.core.JointFlux()`) and canvas-composable as + an engine node; its indexable counterpart for raw sources is `ConcatSource`. ### Example @@ -419,165 +448,3 @@ both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flu a category and group. - **Do not add a discovery category to `FilterOp`/`WrappedOp`** — surfacing a raw-callable parameter on a canvas is a dead widget; the taxonomy is pinned in `tests/test_categories.py`. - ---- - -## ~~The typed-bag model: a named bag of typed items (`sampleflux.bag`, 2026-07-21)~~ — SUPERSEDED - -> **Superseded (2026-07-25)** by -> [One type-dispatched op engine — plain-dict records, libraries as-is](#one-type-dispatched-op-engine--plain-dict-records-libraries-as-is-2026-07-25). -> The `Sample` container, role tags, `primary()`, the adapter coercion registry, and the -> `sampleflux.bag` package were removed; the typed items, the kernel-dispatch idea, and the item -> codec carried forward into the record model. Kept as history — do not follow. - -### Context (historical) - -Before the typed model, the carrier was a fixed `(input, target, metadata)` 3-tuple where `metadata` -was one flat `dict` shared by the whole sample. Everything that is not literally the model input or -target rode that dict by string key: segmentation masks, `[f0,f1,t0,t1]` region lists, window locators, -`spectrogram_params`, power stats, `snr_db`, a signal's samplerate, an image's canvas size, a -label's class names. Two structural costs follow. First, **metadata has no owner** — `samplerate` -belongs to *the signal*, `canvas` to *the image*, but the flat dict severs that link. Second, **a -transform cannot move several fields together** — flipping an image and its mask and its boxes with -one shared decision is inexpressible when the fields are `input`, `target`, and `metadata["regions"]` -respectively, so the era's augmentation adapters hard-coded a `TargetMode = Literal["none","mask","boxes"]` -knob per op instead. `target` was also overloaded — sometimes a bare string (`"drone_x"`), sometimes a -`{boxes, labels}` dict. - -### Decision (historical) - -`sampleflux.bag` modeled a sample as a **named bag of typed items with per-field role tags** -(`Sample`, roles `input`/`target`/`aux`/`pred`, immutable copy-on-write mutators), dispatched -transforms on item TYPE via a kernel registry, batched via `typed_collate` (a batched `Sample`), -and plugged external libraries in through a **coercion registry of adapters** -(`register_adapter`/`coerce_transform` — a `Pipeline` wrapped each bare torchvision-v2 / -albumentations transform in an adapter object at composition time). - -### What survived, and what was undone (2026-07-25) - -- **Survived into the record model:** typed items owning their metadata (the HYBRID - ndarray-subclass / dataclass-wrapper realization, `item_data`/`with_data`, `register_item`), the - once-per-record kernel dispatch (`sampleflux.dispatch`), and the item codec idea - (`sampleflux/io.py` — storage backends never inspect item internals). -- **Undone:** the `Sample` container (a plain dict now), role tags (key names carry meaning), - `primary()` (key addressing), `typed_collate` (→ `collate_records`), and the ENTIRE adapter plane - — coercion registry, adapter classes, `only=` per-key filters (→ `field=`) — replaced by the - engine-level op-family dispatch (`core._apply_op`), which calls each library natively instead of - wrapping it. - -### Example (historical shape — no longer runs) - -```python -sample = Sample({"image": Image(rgb), "regions": Regions(boxes)}, roles={"regions": "target"}) -out = Pipeline([v2.RandomHorizontalFlip(p=1.0), A.GaussNoise(p=1.0)])(sample) # adapter-coerced -``` - -### What you may change - -Nothing — superseded. Extension points live in the successor record above. - ---- - -## ~~Native typed transforms that change a field's TYPE (`ConvertToImage`/`Threshold`/`ConnectedComponents`, 2026-07-22)~~ — SUPERSEDED - -> **Superseded (2026-07-25)** by -> [One type-dispatched op engine — plain-dict records, libraries as-is](#one-type-dispatched-op-engine--plain-dict-records-libraries-as-is-2026-07-25), -> which promotes this record's core insight — the type-changing `__call__`-override op as the -> second sanctioned shape — to a rule of the data model itself. The ops survive (`ConvertToImage`: -> array → `Image`, `Threshold`: array → `Mask`, `ConnectedComponents`: `Mask` → `Regions`, -> plus the target ops) but now read/write plain record KEYS (`field=` in, `output=` out) — the -> role tags, the `Sample` shims, and the legacy-op delegation described below are gone. -> Kept as history — do not follow the role/shim details. - -### Context (historical) - -Two shapes of typed transform exist. The first is the augmentation shape the base `Transform` -was built for: it `handles` an item type and, per handled field, applies a registered kernel that -returns *the same type* (a flip returns a flipped `Image`), so `image`, `mask`, and `boxes` move -together. But a running detection/segmentation front-end needs a different shape: **read one -field, write a field of a DIFFERENT type**. Turning a numeric array into a displayable image, -thresholding an array into a boolean mask, and labelling that mask into a set of bin boxes are -each a *type change* (`array → Image`, `array → Mask`, `Mask → Regions`), not an in-place -per-type edit. No library provides them. - -### Decision (historical) - -Add native typed **twins** that subclass `Transform` and OVERRIDE `__call__` (rather than register -a kernel), reading one field and writing a different-typed item; resolve the source field by an -explicit `field=` name or the first item of the natural type, with every miss raising a -`ValueError` naming the sample's fields; write the output with role tags chosen semantically; and -delegate each twin to its legacy op's math verbatim for byte-parity. - -### What survived, and what was undone (2026-07-25) - -- **Survived:** the two-shapes rule; the `field=`-or-first-natural-type source resolution with loud - `ValueError` misses; the `(row_min, row_max, col_min, col_max)` inclusive integer bin-box - contract of `connected_component_bboxes` (**still load-bearing** — a downstream back-projection - reads exactly that order); truthful `consumes`/`produces` graph metadata. -- **Undone:** role tags on outputs (an op now writes a named `output` key — `Threshold`'s default - `output="mask"`, `ConvertToImage`'s `output="image"`); the legacy `(input, target, metadata)` ops - and the shim-`Sample` delegation (the legacy ops are deleted; the math lives in the shared free - functions `threshold_array` / `connected_component_bboxes` / `value_to_image`). - -### Example (current successor shape) - -```python -from sampleflux.ops.numpy import ConnectedComponents, Threshold - -record = Threshold(field="spec", low_level=-30.0)(record) # + record["mask"] (a Mask) -record = ConnectedComponents(field="mask")(record) # + record["regions"] (a Regions) -``` - -### What you may change - -The bin-box tuple order remains a contract (see the successor record); everything else here is -history. - ---- - -## ~~A typed field cannot hold a live torch tensor — `ToTensor` stores CHW-float numpy (2026-07-22)~~ — SUPERSEDED (decision REVERSED 2026-07-25) - -> **Superseded (2026-07-25, user decision)**: the constraint below was a TYPED-BAG artifact — -> every field had to be a typed item, and an `NDArrayItem` coerces its payload through -> `np.asarray`, so a live tensor could not ride a field. In the RECORD model a value can be -> ANYTHING (the `"plain"` codec tag covers storage, `collate_records._stack` stacks torch -> tensors natively, a bare torchvision-v2 op transforms them as-is), so **`ToTensor` now writes -> the LIVE CHW-float `torch.Tensor` under the key** (in place by default, `output=` for a new -> key) — no numpy round-trip, and the op's name is again the truth. `Image` itself still cannot -> hold a tensor (it IS an ndarray subclass); the torch-`Tensor`-subclass ITEM base (a typed -> tensor value with attrs) remains the documented follow-up (root `TASKS.md`). - -### Context (historical) - -A typed classification front-end needs to turn the working image into the model's input tensor and -the class-name label into the encoded target id. Two facts shape the ops: (1) there is no shared -metadata dict — the label already rides a `Label` value that owns its metadata; (2) an array item -is an `np.ndarray` SUBCLASS whose `__new__` runs `np.asarray(data)`, so **a payload is coerced to -numpy** — an `Image` cannot hold a live `torch.Tensor`, and a bare tensor stored directly has no -registered item type for the collate / storage codec. - -### Decision (historical, largely still in force) - -`ToTensor` resolves an array-bearing key, runs the HWC→CHW + `normalize` conversion, and writes an -`Image(layout="CHW")` whose payload is CHW `float32` numpy — NOT a live tensor; in place by -default so the working key keeps its name. `EncodeTarget` / `DecodeTarget` map a `Label`'s value -through a config-pinned `mapping` and write the encoded `Label` back (carrying the source label's -`classes`). `MetadataToTarget` stays as the escape hatch for a label that rode as another value's -attribute — largely redundant when a source emits the label as a `Label` under its own key. - -### Example (current successor shape) - -```python -from sampleflux.ops.target import EncodeTarget -from sampleflux.ops.torch import ToTensor - -record = {"image": Image(hwc_uint8), "class": Label("cat")} -record = ToTensor(field="image")(record) # record["image"] is now a LIVE CHW float32 torch.Tensor -record = EncodeTarget(mapping={"cat": 0, "dog": 1}, field="class")(record) # Label(0), classes kept -``` - -### What you may change - -- **The Tensor-subclass item follow-up** — the tensor currently rides as a PLAIN value (no item - attrs); a torch-`Tensor`-subclass item base would make it a typed value with metadata again. - Update this record and the `sampleflux/items.py` note together when it lands. diff --git a/docs/augmentation.md b/docs/augmentation.md index 36c1a4b..2f2ccb0 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -112,6 +112,15 @@ Stochasticity lives where each library puts it — the engine adds no seed plumb - per-record gating of any op (native or library): `RandomApply(op=..., probability=..., random_state=N)`. +## Other libraries — register an op family + +albumentations and torchvision v2 are the built-in families, registered through the same OPEN +registry any package can use: `register_op_family(name, matcher, invoker)` teaches the engine a +new library's native calling convention (kornia, DALI, a fork extending albumentations, a +signal-processing library), and bare ops of that library then sit in ANY ops list — every engine +route and composing op, including spawn-parallel workers. Full example + rules: +[record-model.md → "A new library family"](record-model.md#a-new-library-family). + ## Example [`examples/record_pipeline.py`](../examples/record_pipeline.py) — the tour: a bare diff --git a/docs/configure.md b/docs/configure.md index eb3259f..3f27d96 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -5,27 +5,85 @@ Some op parameters are only known *per record*. Two mechanisms cover this: - **`ConfigureOp(ops, target, param, source)`** — runs the `ops` compute-chain on the record as a SIDE branch (its transformations are discarded — the original record continues); the `source`-keyed entry of the chain's final record becomes the VALUE (payload-unwrapped via `item_data`), which is set as the `param` attribute of `target` — post-construction configuration, the confluid paradigm — and then `target` is applied to the original record. Use it when the value is *derived from the record itself* (e.g. a threshold from the record's own max) — the whole derivation reads as one node/YAML block. - **`Capture` + `Apply`** (`sampleflux.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. +Concretely — a producer that draws a random gain per record and publishes what it ACTUALLY drew +as a confluid `@output` (apply `@output` UNDER `@property`), and a consumer whose `level` gets set +per record. The pair is deliberately a roundtrip: compensating with the captured gain restores the +original image, which proves the LIVE draw — not a recomputation — reached the consumer +(verified: `np.allclose(out["image"], original)` holds for every record): + +```python +# mypackage/ops.py +from confluid import configurable, output +from sampleflux import Record, item_data, with_data +import numpy as np + +@configurable(category="op", random=True) +class AugmentOp: + """Scale the image by a random gain drawn per record. + + Args: + max_gain: Upper bound of the uniform gain draw. + """ + + def __init__(self, max_gain: float = 2.0) -> None: + self.max_gain = max_gain + self._applied = 1.0 + self._rng = np.random.default_rng() + + @property + @output + def applied_level(self) -> float: + """The gain the LAST call actually drew — the live @output that Capture records.""" + return self._applied + + def __call__(self, record: Record) -> Record: + self._applied = float(self._rng.uniform(1.0, self.max_gain)) + img = record["image"] + return {**record, "image": with_data(img, item_data(img) * self._applied)} + +@configurable(category="op") +class CompensateOp: + """Divide the image by ``level`` — undo a gain applied earlier in the chain. + + Args: + level: The gain to divide out; set per record by Apply (or ConfigureOp). + """ + + def __init__(self, level: float = 1.0) -> None: + self.level = level + + def __call__(self, record: Record) -> Record: + img = record["image"] + return {**record, "image": with_data(img, item_data(img) / float(self.level))} +``` + ```yaml ops: - # AugmentOp draws a random level each call; capture it into a cell. + # AugmentOp draws a random gain each call; capture the LIVE @output into a cell. - !class:sampleflux.ops.context.Capture - op: !class:mypackage.ops.AugmentOp {} # any op exposing a confluid @output + op: !class:mypackage.ops.AugmentOp {} # or the registered short name: !class:AugmentOp {} output: applied_level name: __captured_level - # …then inject the captured value into a later op's parameter per record. + # …then inject the captured value into the consumer's parameter, per record. - !class:sampleflux.ops.context.Apply op: !class:mypackage.ops.CompensateOp {} param: level source: __captured_level ``` +Reading the pair: `Capture` runs `AugmentOp` once (the record's image is scaled by, say, 1.7×) and +stores `applied_level` = 1.7 in the `__captured_level` cell; `Apply` does +`setattr(compensate_op, "level", 1.7)` and then runs it — post-construction configuration, the +confluid paradigm. Because the value is read off the op AFTER it ran, a stochastic draw is captured +exactly; recomputing it (the naive alternative) would draw a DIFFERENT number. + A self-contained `ConfigureOp` example — derive a per-record threshold from the record's own statistics: ```yaml ops: - !class:sampleflux.ops.configure.ConfigureOp ops: - - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "a.max() * 0.5"} + - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} source: image target: !class:sampleflux.ops.numpy.Threshold low_op: ">=" diff --git a/docs/graph.md b/docs/graph.md index 1e5ec23..a1655c6 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -6,16 +6,25 @@ The **readable authoring form** of a graph pipeline is a `flow:` document — na ```yaml flow: - spec: !class:mypkg.MakeSpectrogram() # input: the source record - masked: !class:sampleflux.ops.numpy.Threshold(low_level=0.5) {from: spec} # 2nd reader of `spec` = fan-out - thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5", field=spec) {from: spec} - gated: !class:sampleflux.ops.numpy.Threshold() + spec: !class:mypkg.MakeSpectrogram {} # input: the source record (writes key `image`) + masked: !class:sampleflux.ops.numpy.Threshold {low_level: 0.5, from: spec} # 2nd reader of `spec` = fan-out + thresh: !class:sampleflux.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} + gated: # a step with bind: uses the plain-mapping form (op: + reserved keys) + op: !class:sampleflux.ops.numpy.Threshold {output: gated_mask} from: spec - bind: {low_level: thresh[spec]} # per-record param := the `spec` entry of thresh's result + bind: + low_level: thresh[image] # per-record param := the `image` entry of thresh's result out: {from: gated, merge_from: [masked]} # fan-in (no op) outputs: out ``` +Two YAML spelling rules (both verified): SCALAR/list reserved keys (`from:`, `merge_from:`) may ride +inside a `!class:` marker's mapping alongside its kwargs — but **`bind:` (a nested mapping) MUST use +the plain-mapping step form** (`op:` + reserved keys, the `gated` step above): a nested mapping under +a `!class:` marker is consumed by Confluid as addressed configuration and never reaches the step +grammar. Write bind refs in block style or quoted — `{low_level: thresh[image]}` inline is a YAML +parse error (`[` opens a flow sequence). + Step grammar (three reserved keys, stripped before the op is built): - **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document order is the schedule and cycles are inexpressible). @@ -75,7 +84,7 @@ with activate(Context()): record = op(record) ``` -Cells hold whole records (from `Save`) or raw values (from `Capture`); `Apply` reads a record cell's `key`-named entry (whole record when `key` is blank), `MergeFields` unions each cell's entries. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). On a deliberate key collision at the fan-in, rename on the producing branch first (`RenameField`, `sampleflux.ops.structure`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-record store instead of extra record keys (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). +Cells hold whole records (from `Save`) or raw values (from `Capture`); `Apply` reads a record cell's `key`-named entry (whole record when `key` is blank), `MergeFields` unions each cell's entries. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). On a deliberate key collision at the fan-in, rename on the producing branch first (`RenameField`, `sampleflux.ops.structure`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-record store instead of extra record keys (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#3-the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). > **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the value under its own key with `CopyField` (`sampleflux.ops.structure`); the snapshot then rides the record as a real entry. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. @@ -90,4 +99,4 @@ from sampleflux.sources import HuggingFaceSource flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) ``` -The helper **materializes** the deferred `!class:` markers before attaching (via `confluid.materialize`) — necessary because `confluid.load` leaves markers nested under a mapping key deferred, and a `Flux` rejects deferred markers at iteration by design. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. +The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: `Flux` also flows any still-deferred marker in place at engine-route entry (the same lazy-flow convention the composing ops use), which is what lets a bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. diff --git a/docs/kinds.md b/docs/kinds.md index 894665f..43241a3 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -61,7 +61,7 @@ def yolo_collate(items): ... loader = DataLoader(flux, collate_fn=get_collate("yolo")) ``` -The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17-updated-2026-07-25). +The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#2-batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17). ## 1→N expanding ops (iterable-only pipelines) diff --git a/docs/record-model.md b/docs/record-model.md index 8253717..740566b 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -4,7 +4,7 @@ A sample is a **plain `dict`** of **typed values**. Import the whole surface fro LEVEL (`from sampleflux import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). The design rationale is recorded in -[architecture.md](architecture.md#one-type-dispatched-op-engine--plain-dict-records-libraries-as-is-2026-07-25). +[architecture.md](architecture.md#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25). ## Why @@ -324,9 +324,41 @@ storable — with no core edit. ### A new library family -Supporting a new external transform library is NOT an adapter class — it is one new branch in -`core._apply_op` (an MRO module-name matcher plus the library's native calling convention), so every -engine route and composing op picks it up at once. +Supporting a new external transform library (kornia, DALI, an albumentations fork, a +signal-processing library) is NOT an adapter class — it is one **registered op family**: a matcher +that recognises the library's op objects plus an invoker that applies one op with the library's own +calling convention. Every engine route (sequential, spawn-parallel, streamed, random-access) and +every composing op picks it up at once, because they all funnel through `_apply_op`: + +```python +from sampleflux import register_op_family + +def is_kornia(op) -> bool: + # Keep the matcher IMPORT-FREE: inspect MRO module names, never import the library. + return any(c.__module__.startswith("kornia.augmentation") for c in type(op).__mro__) + +def invoke_kornia(record, op): + # kornia augmentations are nn.Modules over batched BCHW tensors — one draw per call. + img = record["image"] # a CHW torch.Tensor (e.g. after ToTensor) + out = op(img.unsqueeze(0)).squeeze(0) + return {**record, "image": out} + +register_op_family("kornia", is_kornia, invoke_kornia) + +# From here on, bare kornia ops sit in ANY ops list — Flux, Pipeline, RandomApply, flow steps: +flux = Flux(source=records, ops=[ToTensor(field="image"), K.RandomHorizontalFlip(p=1.0)]) +``` + +The rules: dispatch checks families **last-registered first**, so a more specific family (say a +fork extending albumentations) registers after the built-ins and wins the overlap; re-registering a +name replaces that family in place; matcher and invoker must be **module-level functions** — the +spawn-parallel routes pickle them by reference to rebuild the registry inside worker processes +(defining them in a script's `__main__` or a REPL breaks `.parallel()`; a module import side effect +is the sanctioned place, exactly like `register_item`/`register_kernel`). The built-in +`albumentations` / `torchvision_v2` families register through this same API at import — there is no +privileged code path. When a library's convention needs per-op configuration instead (which key to +read, per-op state), write a normal `Transform` op that wraps it explicitly — the registry is for +AS-IS drop-in. ## Engines — Flux and FlowGraph carry the record @@ -411,13 +443,62 @@ like a Python keyword (e.g. `class`) can't be addressed in an expression — use `predicate` or a non-keyword key name. Live records expose the same nested shape via `sampleflux.storage.query.record_metadata(record)`. See [storage.md](storage.md). -## Batching — `collate_records` +## Batching — `collate_records` and the collate registry + +A torch `DataLoader` (or `Flux.batch`) hands a collate function a LIST of N records and expects +ONE object back. `collate_records` — the registry's `"record"` default — folds per key with three +rules (all records must share the same key set; a mismatch raises): + +1. **array-backed item** → payloads stacked into one array/tensor with a leading batch dim, SAME + item type back; each declared attr becomes a per-record list; +2. **wrapper item** (`Label`, `Regions`) → ONE item whose fields are per-record LISTS — deliberately + not auto-tensorized (turning class names into an `[N]` int64 tensor is the model boundary's one + explicit step, not a generic-engine guess); +3. **plain value** → a plain list. + +```python +records = [{"image": Image(...2×2×3...), "class": Label(i % 2, classes=["noise", "drone"]), + "snr_db": 10.0 * i} for i in range(3)] +batch = collate_records(records) +# image: Image (3, 2, 2, 3) layout: ['HWC', 'HWC', 'HWC'] +# class: Label value=[0, 1, 0] classes: [['noise', 'drone'], ×3] +# snr_db: [0.0, 10.0, 20.0] +``` + +### When the generic rules cannot work: register a task collate + +Stacking is task-shaped, and detection is the canonical failure: each record carries a DIFFERENT +number of boxes, and rule 2 can only give you `Regions(boxes=[<1 box>, <3 boxes>])` — per-record +lists no detection model accepts. A detection model family has its own batch contract (stacked +images + RAGGED per-record target dicts), so the task package registers a collate that produces +exactly that: + +```python +from sampleflux import Image, Regions, collate, register_collate + +@register_collate("detection") +def detection_collate(items): + """The torchvision detection contract: stacked images + ragged per-record targets.""" + images = torch.stack([torch.as_tensor(np.asarray(r["image"])).permute(2, 0, 1) for r in items]) + targets = [ + {"boxes": torch.as_tensor(r["target"].boxes, dtype=torch.float32).reshape(-1, 4), + "labels": torch.as_tensor(r["target"].labels, dtype=torch.int64)} + for r in items + ] + metadata = [{k: v for k, v in r.items() if k not in ("image", "target")} for r in items] + return {"images": images, "targets": targets, "metadata": metadata} + +batch = collate(records, key="detection") # or: DataLoader(..., collate_fn=get_collate("detection")) +# images: [2, 3, 4, 4] — uniform, so stacked +# targets[0]: {'boxes': [1, 4], 'labels': [1]} +# targets[1]: {'boxes': [3, 4], 'labels': [3]} — raggedness PRESERVED, per record +``` -`collate_records` (the collate registry's `"record"` default) turns N record dicts into ONE batched -record: per key, typed payloads stack (torch → stacked tensor, numpy → stacked array, else a list) -and each declared item attr becomes a LIST of per-record values, decoded back into one batched item -of the same type; a plain value batches as the plain list. Batches must be key-homogeneous — a -mismatch raises. See [kinds.md](kinds.md). +The registration is what "solves" detection: the registry lets the task OPT OUT of the generic +folding entirely and emit its model family's native batch shape — while the engine keeps owning +only the GROUPING (yielding lists of records) and never grows task knowledge. Registration is +additive (an import side effect of the task package); a config wires the collate by reference +(`collate_fn: !ref:mypkg.detection_collate`) like any other slot. See [kinds.md](kinds.md). ## What is NOT here yet (follow-ups) diff --git a/docs/runnable.md b/docs/runnable.md new file mode 100644 index 0000000..ca0af5e --- /dev/null +++ b/docs/runnable.md @@ -0,0 +1,104 @@ +# Runnables and entry points + +A **runnable** is any object exposing a no-arg `run()` — a trainer, an evaluator, a dataset +processor, a workflow. It is the unit `sampleflux run` executes: + +```yaml +# config.yaml — the ONE runner shape for every kind of run +runnable: !class:mypkg.Classifier + task: fit # ← the one knob: fit / evaluate / test / predict + train_set: !ref:my_split.train +``` + +```bash +python -m sampleflux.cli run config.yaml # builds `runnable:`, calls .run() +``` + +## The problem entry points solve + +A merged train+eval class exposes SEVERAL capabilities from one class, dispatched off its +`task` knob. Without extra information, a discovery consumer (a config generator, a visual +editor) would have to *assume* one class per capability — it cannot know that `Classifier` +both trains and evaluates, nor which `task` value means "evaluate". The `@entrypoint` marker +declares exactly that, per method. + +## A straightforward example + +```python +from sampleflux import ProgressReporting, TorchRunner, entrypoint + +class Classifier(TorchRunner, ProgressReporting): + """One class, four capabilities — run() dispatches off the ``task`` knob.""" + + def __init__(self, task: str = "fit"): + self.task = task + + def run(self) -> None: + {"fit": self.fit, "evaluate": self.evaluate, + "test": self.test, "predict": self.predict}[self.task]() + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> None: ... # gradient training + + @entrypoint("evaluate", role="evaluator") + def evaluate(self) -> None: ... # metrics over the VALIDATION split + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: ... # metrics over the held-out TEST split + + @entrypoint("predict", role="predictor", primary=True) + def predict(self) -> None: ... # stream predictions +``` + +Each marker states three things: the **`task` value** that reaches this method through +`run()`, the **`role`** capability label (conventionally `"trainer"` / `"evaluator"` / +`"predictor"`; free-form for new capabilities), and — when several methods share a role — +which one is the **`primary`** (here `test` is the default evaluator; `evaluate` is the +secondary, validation-split variant). + +## What the introspectors return + +Real output for the class above (these are executed facts, not sketches): + +```python +>>> from sampleflux import runnable_entrypoints, entrypoint_tasks +>>> runnable_entrypoints(Classifier) +{'fit': {'task': 'fit', 'role': 'trainer', 'primary': True}, + 'evaluate': {'task': 'evaluate', 'role': 'evaluator', 'primary': False}, + 'test': {'task': 'test', 'role': 'evaluator', 'primary': True}, + 'predict': {'task': 'predict', 'role': 'predictor', 'primary': True}} + +>>> entrypoint_tasks(Classifier, "trainer") +['fit'] +>>> entrypoint_tasks(Classifier, "evaluator") +['test', 'evaluate'] # PRIMARY FIRST — "run this as an evaluator" means task: test +>>> entrypoint_tasks(Classifier, "exporter") +[] # unknown role: empty, never an error +``` + +`runnable_entrypoints` walks the MRO (an inherited entry point is found; a subclass override +wins) and reads the marker off the raw function object, so property getters never fire. + +## How a consumer uses this + +A config generator asked for "an evaluator config for `Classifier`" calls +`entrypoint_tasks(Classifier, "evaluator")[0]` → `"test"` and pins `task: test` in the YAML +it emits — one `sampleflux run` then dispatches correctly with no human editing. The same +walk over every discovered class tells a visual editor which classes to offer in a +"trainer" picker versus an "evaluator" picker, even when both answers are the same class. + +## The two marker mixins + +Orthogonal to entry points, a runnable may inherit two stateless mixins: + +- **`TorchRunner`** — declares "my `run()` needs autograd" (`__torch_runner__ = True`, + duck-typed). A GUI executor that evaluates nodes under `torch.inference_mode()` re-enables + autograd for the duration of `run()`. Inference-only runnables deliberately do NOT inherit + it. A merged class can even make it dynamic — a property returning `self.task == "fit"`, + so the same class trains under autograd and predicts under inference mode. +- **`ProgressReporting`** — a framework-free progress sink: the executor injects + `(value, total, desc) -> None` via `set_progress_callback()`, the runnable drains it via + `self._report_progress(step, total, "epoch 3")` from its loop. With no sink injected + (a plain CLI run) every call is a silent no-op. + +Pins: `tests/test_runnable.py` / `tests/test_entrypoint.py`. diff --git a/docs/storage.md b/docs/storage.md index 5df3cc1..6cc0933 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,5 +1,9 @@ # Storage — sinks, sources and queryable metadata (`sampleflux.storage`) +> Runnable tour: [`examples/storage_roundtrip.py`](../examples/storage_roundtrip.py) — the same +> records through all three sink/source pairs (typed values + a plain scalar, byte-identical +> round-trips) plus a `MetadataFilterSource` query that never loads an array. + SampleFlux makes it easy to move data between different formats: ```python diff --git a/docs/workflow.md b/docs/workflow.md new file mode 100644 index 0000000..5473b15 --- /dev/null +++ b/docs/workflow.md @@ -0,0 +1,62 @@ +# Workflows — composing runnables + +The workflow combinators (`sampleflux.workflow`) are the RUNNABLE-level analogue of the +composing ops: they HOLD other runnables and orchestrate them, so a multi-stage pipeline +(prepare → train → evaluate) is ONE Confluid document executed by the same +`sampleflux run workflow.yaml` as any single runnable. + +| Combinator | Runs | +|---|---| +| `Sequence(steps=[...])` | every step in order — the workflow itself (`None` steps skip) | +| `Conditional(condition, if_true, if_false)` | one of two branches, by a condition (`None` branch = do nothing, the sequence continues) | +| `Switch(select, cases, default)` | one of several branches, keyed by the select's value (coerced to `str`) | + +Conditions are `@configurable` predicates — a no-arg `__call__() -> bool`: `PathExists(path)` +(the canonical cache check), `Not(condition)`, `AllOf(conditions)`, `AnyOf(conditions)` — or any +zero-arg callable, a plain `bool`, or a deferred marker resolving to one. + +## The compelling case: a resume-safe pipeline + +Re-run the SAME document after a crash (or just again tomorrow) and it skips the work whose +artifact already exists — *memoise and continue*: + +```yaml +runnable: !class:sampleflux.workflow.Sequence + steps: + # Train ONLY when the checkpoint is missing. On a cache hit the !lazy: branch is + # not just skipped — it is never even BUILT (no model / dataset materialised). + - !class:sampleflux.workflow.Conditional + condition: !class:sampleflux.workflow.PathExists + path: $MODEL_ROOT/run1/model.ckpt + if_true: null # cache hit -> skip, Sequence continues + if_false: !lazy:TrainModel # @configurable classes resolve by registered NAME + ckpt: $MODEL_ROOT/run1/model.ckpt + + # Always runs; the report FORMAT is a Switch on a plain config value — one key a + # CLI override can flip (--select text) without touching the workflow shape. + - !class:sampleflux.workflow.Switch + select: json + cases: + json: !lazy:Evaluate { report: $MODEL_ROOT/run1/report.json, fmt: json } + text: !lazy:Evaluate { report: $MODEL_ROOT/run1/report.txt, fmt: text } +``` + +**The guarantee:** only the selected branch's `run()` is ever called, and a branch wired +`!lazy:` is only *constructed* when selected. [`examples/workflow_pipeline.py`](../examples/workflow_pipeline.py) +runs this exact shape twice and asserts both: pass 2 re-evaluates without retraining, and the +trainer class records exactly ONE construction across both passes. + +## Semantics worth knowing + +- **Branches are lazy twice over**: unchosen `!lazy:` branches are never built; chosen ones are + flowed inside `run()` (zero-arg construction of the combinators themselves does no work). +- **`Conditional` with a `None` branch is "skip and continue"** — the enclosing `Sequence` + proceeds to the next step; nothing blocks. +- **`Switch` select** may be a plain value (`select: json` — overridable from a CLI), a + predicate, or any zero-arg callable; an unmatched key (or `None`) runs `default` (`None` = + no-op). +- **GUI cooperation is inherited**: the combinators mix in `TorchRunner` (a wrapped trainer's + `loss.backward()` survives an inference-mode executor) and `ProgressReporting` (the injected + progress callback is FORWARDED to whichever branch is running). + +Runnables and the `task`/`role` entry-point markers: [runnable.md](runnable.md). diff --git a/examples/dataset_split.yaml b/examples/dataset_split.yaml deleted file mode 100644 index bac2838..0000000 --- a/examples/dataset_split.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# SampleFlux DatasetSplit / RangeSource / ConcatSource Example -# -# DatasetSplit is a `category="source"` that partitions an indexable source into -# reproducible train/val/test views. `split` is the closed Literal["train","val","test"] -# (sampleflux.SplitName). Reload with `confluid.load(...)` and iterate the Fluxes. - -hf_train: !class:sampleflux.sources.HuggingFaceSource() - path: mnist - split: train - input_feature: image - target_feature: label - -# === Property API (preferred): ONE DatasetSplit exposes .train / .val / .test === -# Configure the split once (80/10/10 here) and reference its cached views by -# attribute. All three `!ref:my_split.` resolve to the SAME materialized -# DatasetSplit, so `hf_train` is loaded exactly once and the partition is shared. -my_split: !class:sampleflux.sources.DatasetSplit() - source: !ref:hf_train - val_fraction: 0.1 - test_fraction: 0.1 - seed: 42 - -train_set: !class:sampleflux.core.Flux() - source: !ref:my_split.train - -val_set: !class:sampleflux.core.Flux() - source: !ref:my_split.val - -test_set: !class:sampleflux.core.Flux() - source: !ref:my_split.test - -# === RangeSource: a contiguous [start:stop) slice over any indexable source === -first_1000: !class:sampleflux.core.Flux() - source: !class:sampleflux.sources.RangeSource() - source: !ref:hf_train - stop: 1000 - -# === ConcatSource: join several indexable sources into one (then optionally split) === -hf_test: !class:sampleflux.sources.HuggingFaceSource() - path: mnist - split: test - input_feature: image - target_feature: label - -# Concatenate train + test into one source; ConcatSource is indexable, so it can -# itself feed a DatasetSplit (e.g. to re-partition the combined pool). -combined: !class:sampleflux.sources.ConcatSource() - sources: - - !ref:hf_train - - !ref:hf_test - -combined_pool: !class:sampleflux.core.Flux() - source: !ref:combined diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py new file mode 100644 index 0000000..0db7864 --- /dev/null +++ b/examples/storage_roundtrip.py @@ -0,0 +1,92 @@ +"""Storage round-trip: every sink has a matching source, one codec serializes everything. + +The storage tour of the record model (`typedrecord-v1` — the key-group layout): + +1. Write the SAME records through all three sink/source pairs — `HDF5Sink`↔`HDF5Source`, + `ZarrGroupSink`↔`ZarrGroupSource`, `DirectorySink`↔`DirectorySource` — and read them back + IDENTICAL: typed values keep their type AND their metadata attrs (`Image.layout`, + `Label.classes`), and a plain scalar entry (`snr_db`) rides the same layout via the + ``"plain"`` codec tag. +2. Query a store's metadata WITHOUT loading a single array: `MetadataFilterSource` over the + HDF5 store filters on the scalar entry (`snr_db.value > 10`) through the + `SupportsMetadataScan` protocol — attrs only, payloads untouched until iteration. + +Everything is written to a temp directory — examples leave no artifacts behind. + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import tempfile +from pathlib import Path + +import numpy as np + +from sampleflux import Image, Label, Record +from sampleflux.storage.directory import DirectorySink, DirectorySource +from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source +from sampleflux.storage.query import MetadataFilterSource +from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource + + +def make_records(n: int = 4) -> list: + rng = np.random.default_rng(0) + return [ + { + "image": Image(rng.random((8, 10, 3)).astype(np.float32)), # typed: knows its layout + "class": Label(i % 2, classes=["noise", "drone"]), # typed: carries its vocabulary + "snr_db": float(5 * i), # plain scalar — just another key + } + for i in range(n) + ] + + +def assert_roundtrip(original: Record, restored: Record) -> None: + assert set(restored) == set(original) + assert isinstance(restored["image"], Image) and restored["image"].layout == "HWC" + assert np.array_equal(np.asarray(restored["image"]), np.asarray(original["image"])) + assert isinstance(restored["class"], Label) + assert restored["class"].value == original["class"].value + assert restored["class"].classes == original["class"].classes + assert restored["snr_db"] == original["snr_db"] # the "plain" codec tag + + +def main() -> None: + records = make_records() + + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + pairs = [ + (HDF5Sink(path=work / "store.h5"), HDF5Source(path=work / "store.h5")), + (ZarrGroupSink(path=work / "store.zarr"), ZarrGroupSource(path=work / "store.zarr")), + (DirectorySink(path=work / "store_dir"), DirectorySource(path=work / "store_dir")), + ] + + # 1. Every sink has a matching source — write, read back, byte-identical typed records. + for sink, source in pairs: + with sink: + for record in records: + sink.write(record) + sink.flush() + with source: + restored = list(source) + assert len(restored) == len(records) + for original, back in zip(records, restored): + assert_roundtrip(original, back) + print( + f" {type(sink).__name__:14} -> {type(source).__name__:16} round-trip OK " + f"({len(restored)} records; Image+attrs, Label+vocab, plain snr_db)" + ) + + # 2. Metadata-only querying: filter the HDF5 store on the plain scalar WITHOUT + # loading arrays (the SupportsMetadataScan protocol reads attrs only; a plain + # scalar is addressable as .value). + filtered = MetadataFilterSource(source=HDF5Source(path=work / "store.h5"), where="snr_db.value > 10") + hits = list(filtered) + assert [r["snr_db"] for r in hits] == [15.0] + print(f" MetadataFilterSource(where='snr_db.value > 10') -> {len(hits)} record (snr_db=15.0)") + + print("OK") + + +if __name__ == "__main__": + main() diff --git a/examples/workflow_pipeline.py b/examples/workflow_pipeline.py new file mode 100644 index 0000000..a33a6e1 --- /dev/null +++ b/examples/workflow_pipeline.py @@ -0,0 +1,140 @@ +"""A resume-safe train → evaluate workflow — ONE Confluid document of runnables. + +The compelling case for the workflow combinators: a pipeline you can re-run after a crash +(or a second `sampleflux run`) that SKIPS the work whose artifact already exists and +carries on — *memoise and continue*, expressed declaratively: + +1. ``Sequence`` drives the stages in order (the workflow itself). +2. ``Conditional`` + ``PathExists`` guard the expensive stage: train ONLY when the + checkpoint is missing. On a cache hit the branch is not just skipped — wired + ``!lazy:``, it is **never even constructed** (no model, no dataset materialised). +3. ``Switch`` picks the report format off a plain config value — one key a CLI override + can flip (``--select text``) without touching the workflow shape. + +This script runs the SAME document twice and proves both guarantees: the second pass +evaluates again but does not retrain, and the trainer class records exactly ONE +construction across both passes. + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import tempfile +from pathlib import Path + +import confluid +from confluid import configurable +from confluid.fluid import Fluid + +# --------------------------------------------------------------------------------------- +# Three tiny stub runnables (a runnable = any object with a no-arg run()). Real pipelines +# put a trainer / DatasetProcessor here; stubs keep the example self-contained and fast. +# --------------------------------------------------------------------------------------- + + +@configurable +class TrainModel: + """Stub trainer: 'trains' by writing the checkpoint artifact. + + Args: + ckpt: Path the checkpoint artifact is written to. + """ + + #: Instrumentation for THIS example: counts constructions, to prove the ``!lazy:`` + #: guarantee (an unchosen branch is never built). Not a pattern for real runnables. + builds = 0 + + def __init__(self, ckpt: str = "") -> None: + type(self).builds += 1 + self.ckpt = ckpt + + def run(self) -> None: + Path(self.ckpt).write_text("weights") + print(f" [train] wrote {Path(self.ckpt).name}") + + +@configurable +class Evaluate: + """Stub evaluator: reads the checkpoint, writes a report in the given format. + + Args: + ckpt: Checkpoint artifact to 'evaluate'. + report: Path the report is written to. + fmt: Report format tag written into the file. + """ + + def __init__(self, ckpt: str = "", report: str = "", fmt: str = "text") -> None: + self.ckpt = ckpt + self.report = report + self.fmt = fmt + + def run(self) -> None: + weights = Path(self.ckpt).read_text() + Path(self.report).write_text(f"[{self.fmt}] accuracy of {weights!r}: 0.93") + print(f" [evaluate] wrote {Path(self.report).name} ({self.fmt})") + + +def workflow_yaml(work: Path) -> str: + """The whole pipeline — stages, the cache guard, AND the format switch — as ONE document.""" + return f""" +runnable: !class:sampleflux.workflow.Sequence + steps: + # Stage 1 — the expensive stage, guarded: train ONLY when the checkpoint is missing. + # On a cache hit the !lazy: branch is never even BUILT (no model materialised). + - !class:sampleflux.workflow.Conditional + condition: !class:sampleflux.workflow.PathExists + path: {work / "model.ckpt"} + if_true: null # cache hit -> skip, Sequence continues + if_false: !lazy:TrainModel # @configurable classes resolve by registered NAME + ckpt: {work / "model.ckpt"} + + # Stage 2 — always runs; the report FORMAT is a Switch on a plain config value + # (override from a CLI with --select text — the workflow shape never changes). + - !class:sampleflux.workflow.Switch + select: json + cases: + json: !lazy:Evaluate + ckpt: {work / "model.ckpt"} + report: {work / "report.json"} + fmt: json + text: !lazy:Evaluate + ckpt: {work / "model.ckpt"} + report: {work / "report.txt"} + fmt: text +""" + + +def run_document(path: Path) -> None: + """What ``sampleflux run `` does: bind the top-level ``runnable:`` and run it.""" + loaded = confluid.load(str(path)) + runnable = loaded["runnable"] + if isinstance(runnable, Fluid): + runnable = confluid.flow(runnable) + runnable.run() + + +def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + doc = work / "workflow.yaml" + doc.write_text(workflow_yaml(work)) + + print("--- pass 1: nothing cached -> trains, then evaluates ---") + run_document(doc) + assert (work / "model.ckpt").exists() and (work / "report.json").exists() + assert TrainModel.builds == 1 + + (work / "report.json").unlink() # so pass 2 visibly re-evaluates + + print("--- pass 2: checkpoint exists -> SKIPS training, evaluates again ---") + run_document(doc) + assert (work / "report.json").exists() + + # The two guarantees: no retrain (artifact-guarded), and the unchosen !lazy: + # branch was never CONSTRUCTED on pass 2 — builds stayed at one. + assert TrainModel.builds == 1, f"trainer was rebuilt: {TrainModel.builds}" + print("\ntrainer constructions across both passes:", TrainModel.builds) + print("OK") + + +if __name__ == "__main__": + main() diff --git a/sampleflux/__init__.py b/sampleflux/__init__.py index 29704d8..f9ef837 100644 --- a/sampleflux/__init__.py +++ b/sampleflux/__init__.py @@ -12,7 +12,7 @@ # --- shared infrastructure ----------------------------------------------------------------- from sampleflux.collate import collate, collate_records, get_collate, register_collate, registered_collates from sampleflux.context import Context -from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp +from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp, register_op_family, registered_op_families # --- the record data model + transforms + item codec ---------------------------------------- from sampleflux.dispatch import dispatch, register_kernel, registered_kernels @@ -83,6 +83,8 @@ "JointFlux", "FilterOp", "WrappedOp", + "register_op_family", + "registered_op_families", "FlowGraph", "from_ops", "to_ops", diff --git a/sampleflux/core.py b/sampleflux/core.py index 5ef0688..b8f247a 100644 --- a/sampleflux/core.py +++ b/sampleflux/core.py @@ -21,6 +21,72 @@ def _op_expands(op: Any) -> bool: return bool(getattr(op, "EXPANDS", False)) +#: An op-family MATCHER recognises a library's op objects. Keep it IMPORT-FREE — inspect +#: ``type(op).__mro__`` module names rather than importing the library. +OpMatcher = Callable[[Any], bool] +#: An op-family INVOKER applies one foreign op with its library's native calling +#: convention: ``(record, op) -> Optional[Record]`` (``None`` = drop the record). +OpInvoker = Callable[[Record, Any], Optional[Record]] + +#: The registered op families, in registration order. Dispatch checks LAST-registered +#: first, so a later (more specific) family can shadow an earlier one. +_OP_FAMILIES: List[Tuple[str, OpMatcher, OpInvoker]] = [] + + +def register_op_family(name: str, matcher: OpMatcher, invoker: OpInvoker) -> None: + """Teach the engine to invoke a NEW library's ops natively — the open extension point. + + ``matcher(op) -> bool`` recognises the family's op objects (keep it import-free — + inspect ``type(op).__mro__`` module names); ``invoker(record, op)`` applies one op with + the library's own calling convention and returns the new record (``None`` drops it). + Re-registering a ``name`` REPLACES that family in place; otherwise the family is + appended, and dispatch checks last-registered first (a more specific family shadows an + earlier one — register yours after the built-ins to win an overlap). + + Both callables MUST be module-level functions (picklable by reference): the engine's + spawn-parallel routes ship non-builtin families to worker processes by pickling them. + + Example — kornia augmentations (``nn.Module``s over batched BCHW tensors):: + + def is_kornia(op) -> bool: + return any(c.__module__.startswith("kornia.augmentation") for c in type(op).__mro__) + + def invoke_kornia(record, op): + img = record["image"] # a CHW torch.Tensor (e.g. after ToTensor) + out = op(img.unsqueeze(0)).squeeze(0) # kornia draws once per batch call + return {**record, "image": out} + + register_op_family("kornia", is_kornia, invoke_kornia) + """ + entry = (str(name), matcher, invoker) + for i, (existing, _, _) in enumerate(_OP_FAMILIES): + if existing == name: + _OP_FAMILIES[i] = entry + return + _OP_FAMILIES.append(entry) + + +def registered_op_families() -> Tuple[str, ...]: + """The registered op-family names, in registration/dispatch-precedence order.""" + return tuple(name for name, _, _ in _OP_FAMILIES) + + +def _sync_op_families(families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]]) -> None: + """Merge families shipped from the parent process into this process's registry. + + Spawn workers import this module (built-ins present) but never re-run the user's + registration side effects — the parallel routes therefore pass the parent's + non-builtin entries along and merge them here (idempotent by name). + """ + for name, matcher, invoker in families or []: + register_op_family(name, matcher, invoker) + + +def _extra_op_families() -> List[Tuple[str, OpMatcher, OpInvoker]]: + """The non-builtin registry entries — what a spawn worker cannot rebuild by import alone.""" + return [entry for entry in _OP_FAMILIES if entry[0] not in _BUILTIN_FAMILIES] + + #: The record keys albumentations understands — its OWN target vocabulary. An albumentations #: op receives exactly these keys (the ones present) and nothing else, so extra record #: entries (scalars, domain items) never reach a library that would reject them. @@ -32,11 +98,49 @@ def _is_albumentations(op: Any) -> bool: return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(op).__mro__) +def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: + """albumentations dispatches by KWARG NAME: hand the op exactly its own target keys + present in the record (one call = one joint draw across them); array outputs are + re-wrapped in the incoming value's item type (``with_data``) so ``Image``/``Mask`` + keep their type and metadata. Box-carrying augmentation belongs in albumentations' own + ``A.Compose(..., bbox_params=...)`` — format handling is Compose's job in that library. + """ + kwargs = {k: record[k] for k in _ALB_KEYS if k in record} + if not kwargs: + logger.debug( + f"albumentations op {type(op).__name__} received no known keys " + f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." + ) + return record + out = op(**kwargs) + merged = dict(record) + for key, value in out.items(): + original = record.get(key) + if isinstance(original, NDArrayItem) and not isinstance(value, NDArrayItem): + value = with_data(original, value) + merged[key] = value + return merged + + def _is_torchvision_v2(op: Any) -> bool: """True for a torchvision ``transforms.v2`` transform — by MRO module name (no import here).""" return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(op).__mro__) +def _invoke_torchvision_v2(record: Record, op: Any) -> Optional[Record]: + """torchvision v2 natively walks a dict: params sampled once, tensor/tv_tensor/PIL + leaves transformed, everything else passed through — called as-is.""" + return cast(Record, op(record)) + + +# The built-in families register through the SAME open registry third parties use — +# one mechanism, no privileged code path. Registered at import, so spawn workers +# rebuild them by importing this module. +register_op_family("albumentations", _is_albumentations, _invoke_albumentations) +register_op_family("torchvision_v2", _is_torchvision_v2, _invoke_torchvision_v2) +_BUILTIN_FAMILIES: Tuple[str, ...] = ("albumentations", "torchvision_v2") + + def _apply_op(record: Record, op: Any) -> Optional[Record]: """Apply one op to the record dict — the engine's op-FAMILY dispatch. @@ -44,37 +148,19 @@ def _apply_op(record: Record, op: Any) -> Optional[Record]: :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths; composing ops (``Pipeline`` / ``Parallel`` / ``Enable`` / ``RandomApply`` / the context ops) route their inner ops through here so every op is applied identically. Each op family - is invoked the way its library expects — no wrapper/adapter classes: - - * **albumentations** — dispatches by KWARG NAME: the op receives exactly its own target - keys present in the record (``image``/``mask``/``bboxes``/…), one call = one joint - draw across them. Array outputs are re-wrapped in the incoming value's item type - (``with_data``) so an ``Image``/``Mask`` keeps its type and metadata. Box-carrying - augmentation belongs in albumentations' own ``A.Compose(..., bbox_params=...)`` - (dropped into the ops list bare) — format handling is Compose's job in that library. - * **torchvision v2** — natively walks the dict, samples params once, transforms - tensor/tv_tensor/PIL leaves and passes everything else through: called as-is. - * **anything else** — a native/wiring op ``record -> Optional[Record]`` (``None`` drops - the record — filter semantics). + is invoked the way its library expects — no wrapper/adapter classes: the registered + families (:func:`register_op_family`; built-ins ``albumentations`` / + ``torchvision_v2``) are checked LAST-registered first, and an op matching none of + them is a native/wiring op called ``op(record) -> Optional[Record]`` (``None`` drops + the record — filter semantics). """ - if _is_albumentations(op): - kwargs = {k: record[k] for k in _ALB_KEYS if k in record} - if not kwargs: - logger.debug( - f"albumentations op {type(op).__name__} received no known keys " - f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." - ) - return record - out = op(**kwargs) - merged = dict(record) - for key, value in out.items(): - original = record.get(key) - if isinstance(original, NDArrayItem) and not isinstance(value, NDArrayItem): - value = with_data(original, value) - merged[key] = value - return merged - if _is_torchvision_v2(op): - return cast(Record, op(record)) + for _name, matcher, invoker in reversed(_OP_FAMILIES): + try: + matched = matcher(op) + except Exception: # pragma: no cover - a defensive matcher never breaks dispatch + matched = False + if matched: + return invoker(record, op) return cast(Optional[Record], op(record)) @@ -218,17 +304,24 @@ def _expand(op: Any, sample: Any) -> List[Any]: return [child for child in raw if child is not None] -def _worker_task(sample: Any, ops: List[Any]) -> Optional[Any]: +def _worker_task( + sample: Any, ops: List[Any], families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None +) -> Optional[Any]: """Single-result worker for STRICTLY 1→1 op lists (the ``Parallel`` op's contract). Kept for callers that need exactly one carrier back; expanding ops raise here — route expanding pipelines through :func:`_worker_task_multi`. """ - results = _worker_task_multi(sample, ops, allow_expansion=False) + results = _worker_task_multi(sample, ops, allow_expansion=False, families=families) return results[0] if results else None -def _worker_task_multi(sample: Any, ops: List[Any], allow_expansion: bool = True) -> List[Any]: +def _worker_task_multi( + sample: Any, + ops: List[Any], + allow_expansion: bool = True, + families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, +) -> List[Any]: """Top-level helper for multiprocess workers. Must be at top level for pickling. Runs one source :class:`Sample` through the op list and returns EVERY resulting sample — @@ -240,10 +333,13 @@ def _worker_task_multi(sample: Any, ops: List[Any], allow_expansion: bool = True context ops (``Save``/``Use``/``Apply``/``Capture``/``MergeFields``) can move data between the linear stream and named cells — the executor itself stays a plain ``for op in ops`` loop. Contexts are created inside the worker (spawn-safe: ops pickle, a Context never - crosses a process boundary). + crosses a process boundary). ``families`` carries the parent process's non-builtin op + families into a spawn worker (:func:`_sync_op_families` — matchers/invokers pickle by + reference); in-process callers omit it. """ from collections import deque + _sync_op_families(families) pending: "deque[Tuple[Any, Context, int]]" = deque([(sample, Context(), 0)]) out: List[Any] = [] while pending: @@ -573,8 +669,9 @@ def _iter_parallel(self) -> Iterator[Record]: with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: futures = [] + extra_families = _extra_op_families() # ship third-party op families to the workers for item in source: - futures.append(executor.submit(_worker_task_multi, item, self.ops)) + futures.append(executor.submit(_worker_task_multi, item, self.ops, True, extra_families)) for future in futures: yield from future.result() diff --git a/sampleflux/ops/__init__.py b/sampleflux/ops/__init__.py index d5873d2..d2d7aaa 100644 --- a/sampleflux/ops/__init__.py +++ b/sampleflux/ops/__init__.py @@ -6,7 +6,7 @@ connected_component_bboxes / resolve_expression helpers) - sampleflux.ops.torch: ToTensor (+ to_tensor helper) - sampleflux.ops.image: ConvertToImage (+ value_to_image / normalize_to_uint8 …) - - sampleflux.ops.target: MetadataToTarget, EncodeTarget, DecodeTarget, + - sampleflux.ops.target: EncodeTarget, DecodeTarget, CocoToTorchVisionDetection, MasksToDetectionBoxes - sampleflux.ops.structure: RenameField, DropField, CopyField, SelectFields - sampleflux.ops.parallel: Parallel (worker-pool sub-pipeline) @@ -14,7 +14,7 @@ - sampleflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - sampleflux.ops.configure: ConfigureOp (per-record parameter injection) - sampleflux.ops.formula: FormulaOp (math formula over one record entry) - - sampleflux.ops.sink: SampleSinkOp (adapt a DataSink as a pass-through op) + - sampleflux.ops.sink: RecordSinkOp (adapt a DataSink as a pass-through op) - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-record Context graph plane — the flat-list building blocks a branchy flow: document lowers to) - sampleflux.ops.debug: PrintSampleOp (per-record summary probe) @@ -32,15 +32,9 @@ from sampleflux.ops.numpy import ConnectedComponents, Threshold from sampleflux.ops.parallel import Parallel from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.sink import SampleSinkOp +from sampleflux.ops.sink import RecordSinkOp from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields -from sampleflux.ops.target import ( - CocoToTorchVisionDetection, - DecodeTarget, - EncodeTarget, - MasksToDetectionBoxes, - MetadataToTarget, -) +from sampleflux.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes from sampleflux.ops.torch import ToTensor __all__ = [ @@ -59,13 +53,12 @@ "FormulaOp", "MasksToDetectionBoxes", "MergeFields", - "MetadataToTarget", "Parallel", "PrintSampleOp", "RandomApply", "RenameField", "Save", - "SampleSinkOp", + "RecordSinkOp", "SelectFields", "Threshold", "ToTensor", diff --git a/sampleflux/ops/configure.py b/sampleflux/ops/configure.py index dea06dc..1100cbf 100644 --- a/sampleflux/ops/configure.py +++ b/sampleflux/ops/configure.py @@ -37,7 +37,7 @@ class ConfigureOp: - !class:sampleflux.ops.configure.ConfigureOp ops: - - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "a.max() * 0.5"} + - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} source: image target: !class:sampleflux.ops.numpy.Threshold low_op: ">=" diff --git a/sampleflux/ops/enable.py b/sampleflux/ops/enable.py index 70761e9..f7e96a1 100644 --- a/sampleflux/ops/enable.py +++ b/sampleflux/ops/enable.py @@ -57,19 +57,21 @@ class Enable: Disambiguating multiple wrappers -------------------------------- When two or more ``Enable`` instances live in the same pipeline, give - each a ``name:`` in YAML. That name becomes the preferred identifier - in Confluid's hierarchy (``--help``) and Liquify's override matcher, - so you can toggle them independently: + each a ``name:`` in YAML and use the GENERIC toggle name ``enable`` — + the name scopes the flag, so a semantic attribute name per wrapper is + unnecessary. ``name`` becomes the preferred identifier in Confluid's + hierarchy (``--help``) and Liquify's override matcher, so you can + toggle them independently: .. code-block:: yaml - !class:sampleflux.ops.enable.Enable name: overlay # dotted-override key - visualize: false # same attr name is fine — name scopes it + enable: false # generic toggle — the name scopes it ops: [render-with-overlays, save-to ./debug_png] - !class:sampleflux.ops.enable.Enable name: labelstudio - visualize: false + enable: false ops: [render-clean, save-to ./ls_png] CLI: @@ -77,10 +79,11 @@ class Enable: .. code-block:: bash # Targeted — only the overlay chain fires. - sampleflux run pipeline.yaml --overlay.visualize true + sampleflux run pipeline.yaml --overlay.enable true + sampleflux run pipeline.yaml --overlay.enable+ # polarity shorthand → True - # Broadcast — every Fluid with a `visualize` kwarg flips. - sampleflux run pipeline.yaml --visualize true + # Broadcast — every Fluid with an `enable` kwarg flips. + sampleflux run pipeline.yaml --enable true ``name`` is a plain string on the instance; Confluid's post-construction paradigm setattr's it automatically from YAML with no ctor change. @@ -93,6 +96,11 @@ class Enable: dunders) may be set on the wrapper — that's the toggle. ``RuntimeError`` is raised on first call if zero or multiple are present. + * RESERVED names: the toggle may be ANY boolean attribute name EXCEPT the + class's own members — ``ops``, ``enabled``, ``flag_name`` (read-only + introspection properties; a YAML kwarg with one of those names raises + ``AttributeError`` at configure time). Use ``enable`` as the generic + toggle name; ``enabled`` (the property) then READS whatever toggle is set. Args: ops: Non-empty list of ops (native or bare library transforms) gated by the toggle. @@ -150,7 +158,7 @@ def __call__(self, record: Record) -> Optional[Record]: return current def close(self) -> None: - """Propagate close to inner ops that own resources (e.g. SampleSinkOp).""" + """Propagate close to inner ops that own resources (e.g. RecordSinkOp).""" for op in self.ops: close_fn = getattr(op, "close", None) if callable(close_fn): diff --git a/sampleflux/ops/formula.py b/sampleflux/ops/formula.py index 0f39bcb..cb7857a 100644 --- a/sampleflux/ops/formula.py +++ b/sampleflux/ops/formula.py @@ -12,6 +12,7 @@ import math as _math from typing import Any, Dict +import numpy as _np from confluid import configurable from sampleflux.items import Record, item_data, with_data @@ -20,6 +21,12 @@ # node's namespace. The bound variable shadows same-named constants (e.g. ``e``). _FORMULA_NAMESPACE: Dict[str, Any] = {k: getattr(_math, k) for k in dir(_math) if not k.startswith("_")} _FORMULA_NAMESPACE.update({"abs": abs, "min": min, "max": max, "round": round, "pow": pow}) +# Array reducers, FUNCTION style (``amax(a) * 0.5``) — pre-bound numpy callables whose +# internal lazy imports resolve via numpy's own globals. The ATTRIBUTE form (``a.max()``) +# is NOT guaranteed under the sandbox: numpy's C reductions lazy-import through the +# CALLING frame, whose ``__builtins__`` is empty here (KeyError: '__import__') unless some +# earlier code already warmed that import in this process. Teach the function form. +_FORMULA_NAMESPACE.update({"amax": _np.max, "amin": _np.min, "mean": _np.mean, "std": _np.std, "median": _np.median}) @configurable(category="op", group="compose") @@ -27,7 +34,7 @@ class FormulaOp: """Replace the ``field``-keyed record value with ``formula`` evaluated over it. Args: - formula: Expression over ``var`` (e.g. ``"a * 0.2"``); ``math.*`` + ``abs``/``min``/``max``/``round`` allowed. + formula: Expression over ``var`` — math.*, abs/min/max/round/pow + reducers amax/amin/mean/std/median. field: Record key whose value the formula reads and replaces; required at call time. var: Variable name the incoming value binds to. Defaults to ``a``. """ diff --git a/sampleflux/ops/parallel.py b/sampleflux/ops/parallel.py index 1a11ec7..9ca67fb 100644 --- a/sampleflux/ops/parallel.py +++ b/sampleflux/ops/parallel.py @@ -72,12 +72,15 @@ def stream(self, samples: Iterable[Optional[Record]]) -> Iterator[Optional[Recor ctx = multiprocessing.get_context("spawn") limit = max(2 * self.workers, self.workers + 1) + from sampleflux.core import _extra_op_families + with concurrent.futures.ProcessPoolExecutor(max_workers=self.workers, mp_context=ctx) as executor: pending: "deque[concurrent.futures.Future[Optional[Record]]]" = deque() + extra_families = _extra_op_families() # ship third-party op families to the workers for s in samples: if s is None: continue - pending.append(executor.submit(_worker_task, s, self.ops)) + pending.append(executor.submit(_worker_task, s, self.ops, extra_families)) if len(pending) >= limit: yield pending.popleft().result() while pending: diff --git a/sampleflux/ops/sink.py b/sampleflux/ops/sink.py index 17e71e8..d0cc454 100644 --- a/sampleflux/ops/sink.py +++ b/sampleflux/ops/sink.py @@ -1,4 +1,4 @@ -"""``SampleSinkOp`` — adapt a :class:`~sampleflux.storage.base.DataSink` as a pass-through op. +"""``RecordSinkOp`` — adapt a :class:`~sampleflux.storage.base.DataSink` as a pass-through op. Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, a domain package's JSON sinks …) slot into a record-based op chain: on first call it opens the @@ -18,7 +18,7 @@ @configurable(category="op", group="sink") -class SampleSinkOp: +class RecordSinkOp: """Adapter: wrap a :class:`sampleflux.storage.base.DataSink` as a pass-through op. Sinks implement the ``open()`` / ``write(record)`` / ``close()`` protocol and @@ -34,7 +34,7 @@ class SampleSinkOp: YAML:: - - !class:sampleflux.ops.sink.SampleSinkOp + - !class:sampleflux.ops.sink.RecordSinkOp sink: !class:sampleflux.storage.hdf5.HDF5Sink path: ./records.h5 @@ -49,7 +49,7 @@ def __init__(self, sink: Any = None) -> None: def __call__(self, record: Record) -> Record: if self.sink is None: - raise ValueError("SampleSinkOp requires a non-None 'sink'.") + raise ValueError("RecordSinkOp requires a non-None 'sink'.") if not self._opened: opener = getattr(self.sink, "open", None) if callable(opener): @@ -68,4 +68,4 @@ def close(self) -> None: self._opened = False -__all__ = ["SampleSinkOp"] +__all__ = ["RecordSinkOp"] diff --git a/sampleflux/ops/target.py b/sampleflux/ops/target.py index b4c30cd..5496780 100644 --- a/sampleflux/ops/target.py +++ b/sampleflux/ops/target.py @@ -1,6 +1,5 @@ """Target-shaping transforms over plain-dict records. -* :class:`MetadataToTarget` promotes a field / attr value into a target ``Label``. * :class:`EncodeTarget` / :class:`DecodeTarget` map a class-name ``Label`` to a class-id ``Label`` and back through an explicit lookup ``mapping`` — the declarative analogue of scikit-learn's ``LabelEncoder``. The mapping is pinned in config, NOT fitted, so @@ -135,63 +134,6 @@ def masks_to_detection( return {"boxes": boxes_t, "labels": labels_t} -@configurable(category="op", group="structure") -class MetadataToTarget(Transform): - """Promote a field / attr value into a target ``Label``. - - Reads a value from a SOURCE field (``field``; blank picks the first ``Label``, else the - first field) — either the field's natural value (a ``Label``'s ``.value``, otherwise the - item's array payload) or, when ``key`` is set, the named ATTRIBUTE of the source item — - and writes a fresh :class:`~sampleflux.Label` under ``output``. - - In a typical classification pipeline the source emits the label directly as a - ``Label`` field, so this op is usually a NO-OP-ish re-home; it - exists for the case where a label rode as another item's attribute (``key=``). - - Args: - field: Source field to read; blank (default) picks the first ``Label`` field, else the first field. - key: Optional attribute name to read off the source item; blank (default) reads the item's natural value. - output: Key the target ``Label`` is written to (added if new). - """ - - handles = (Label,) - consumes = (Label,) - produces = (Label,) - - def __init__(self, field: str = "", key: str = "", output: str = "target") -> None: - super().__init__() - self.field = str(field) - self.key = str(key) - self.output = str(output) - - def _find_source(self, record: Record) -> str: - """Resolve the KEY of the source field (``self.field``, else first ``Label``, else first field).""" - if self.field: - if self.field not in record: - raise ValueError(f"MetadataToTarget: field {self.field!r} not in record (keys: {list(record)})") - return self.field - for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): - return key - for key in record: - return key - raise ValueError("MetadataToTarget: record is empty — no source field to read") - - def __call__(self, record: Record) -> Record: - key = self._find_source(record) - item = record[key] - if self.key: - if not hasattr(item, self.key): - raise AttributeError( - f"MetadataToTarget: field {key!r} ({type(item).__name__}) has no attribute {self.key!r}" - ) - value = getattr(item, self.key) - elif isinstance(item, Label): - value = item.value - else: - value = item_data(item) - return {**record, self.output: Label(value)} - - @configurable(category="op", group="structure") class EncodeTarget(Transform): """A class-NAME ``Label`` → a class-ID ``Label``. @@ -449,7 +391,6 @@ def __call__(self, record: Record) -> Record: __all__ = [ - "MetadataToTarget", "EncodeTarget", "DecodeTarget", "CocoToTorchVisionDetection", diff --git a/sampleflux/runnable.py b/sampleflux/runnable.py index b333518..40de75a 100644 --- a/sampleflux/runnable.py +++ b/sampleflux/runnable.py @@ -28,7 +28,8 @@ method with the ``task`` value it runs and a ``role`` label (``"trainer"`` / ``"evaluator"`` / ``"predictor"``). A discovery consumer (a config generator, a visual editor) reads these via :func:`runnable_entrypoints` to learn that one class both -trains and evaluates, instead of assuming a separate class per role. +trains and evaluates, instead of assuming a separate class per role. Straightforward +worked example (the class + the exact introspector outputs): ``docs/runnable.md``. """ from typing import Callable, Dict, List, Optional @@ -107,6 +108,31 @@ def entrypoint(task: str, role: str = "runnable", primary: bool = False) -> Call method so a discovery consumer (a config generator, a visual editor) can learn — from ONE class — which capabilities it exposes, instead of assuming a separate class per role. + Example — one class, four capabilities (full walkthrough in ``docs/runnable.md``):: + + class Classifier(TorchRunner, ProgressReporting): + def run(self) -> None: + {"fit": self.fit, "evaluate": self.evaluate, + "test": self.test, "predict": self.predict}[self.task]() + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> None: ... + + @entrypoint("evaluate", role="evaluator") # validation-split metrics + def evaluate(self) -> None: ... + + @entrypoint("test", role="evaluator", primary=True) # held-out metrics = the default evaluator + def test(self) -> None: ... + + @entrypoint("predict", role="predictor", primary=True) + def predict(self) -> None: ... + + entrypoint_tasks(Classifier, "evaluator") # -> ["test", "evaluate"] (primary first) + entrypoint_tasks(Classifier, "trainer") # -> ["fit"] + + A consumer asked for "an evaluator config" takes ``entrypoint_tasks(cls, "evaluator")[0]`` + and pins ``task: test`` in the config it emits. + Args: task: The ``task`` value the runnable's ``run()`` dispatches to for this method (e.g. ``"fit"`` / ``"test"``). diff --git a/sampleflux/workflow.py b/sampleflux/workflow.py index 16df38d..5e9fd7f 100644 --- a/sampleflux/workflow.py +++ b/sampleflux/workflow.py @@ -3,12 +3,12 @@ A *runnable* is any object exposing a no-arg ``run(self)`` (a trainer, an evaluator, a :class:`~sampleflux.processing.DatasetProcessor`). This module adds Confluid-``@configurable`` *combinators* that HOLD other runnables and orchestrate -them — the runnable-level analogue of the higher-order ops (``Parallel`` / -``TransformChain`` / ``Enable``): +them — the runnable-level analogue of the composing ops (``Pipeline`` / +``Parallel`` / ``Enable``): * :class:`Sequence` — run a list of runnables in order (the workflow itself). * :class:`Conditional` — run one of two runnables depending on a condition. -* :class:`Switch` — run one of several runnables keyed by a selector value. +* :class:`Switch` — run one of several runnables keyed by a select value. Conditions are themselves Confluid-``@configurable`` *predicates* — a no-arg ``__call__(self) -> bool`` (:class:`PathExists` / :class:`Not` / :class:`AllOf` @@ -33,6 +33,8 @@ model / dataset is materialised). This is the *memoise-and-continue* answer to "don't recompute, move on": the next ``steps:`` entry runs regardless, because ``Sequence`` drives them in order — no execution-blocking, no dead branches. +Runnable proof (the resume-safe train→evaluate pipeline, run twice, both +guarantees asserted): ``examples/workflow_pipeline.py``; usage: ``docs/workflow.md``. All combinators are zero-arg constructible and do NO functional work in ``__init__`` (the workspace lazy-construction convention); branches and @@ -158,24 +160,24 @@ def run(self) -> None: @configurable class Switch(TorchRunner, ProgressReporting): - """Run one of several runnables keyed by a selector's value. + """Run one of several runnables keyed by a select's value. Args: - selector: A no-arg callable / predicate / deferred value producing the + select: A no-arg callable / predicate / deferred value producing the case KEY (coerced to ``str``). ``None`` (or a ``None`` result) selects ``default``. cases: Mapping of key -> runnable. The runnable whose key matches the - selector runs; an unmatched key falls back to ``default``. + select runs; an unmatched key falls back to ``default``. default: Runnable to run when no case matches. ``None`` = no-op. """ def __init__( self, - selector: Any = None, + select: Any = None, cases: Optional[Dict[str, Any]] = None, default: Any = None, ) -> None: - self.selector = selector + self.select = select self.cases: Dict[str, Any] = dict(cases) if cases else {} self.default = default @@ -190,7 +192,7 @@ def run(self) -> None: _run(branch, self._progress_callback) def _select(self) -> Optional[str]: - selector = _resolve(self.selector) + selector = _resolve(self.select) if selector is None: return None value = selector() if callable(selector) else selector diff --git a/tests/test_categories.py b/tests/test_categories.py index ecb6683..2c1e95b 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -20,15 +20,9 @@ from sampleflux.ops.numpy import ConnectedComponents, Threshold from sampleflux.ops.parallel import Parallel from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.sink import SampleSinkOp +from sampleflux.ops.sink import RecordSinkOp from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields -from sampleflux.ops.target import ( - CocoToTorchVisionDetection, - DecodeTarget, - EncodeTarget, - MasksToDetectionBoxes, - MetadataToTarget, -) +from sampleflux.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes from sampleflux.ops.torch import ToTensor from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource from sampleflux.storage.directory import DirectorySink @@ -65,8 +59,7 @@ def test_op_classes_tagged() -> None: Pipeline, Parallel, RandomApply, - SampleSinkOp, - MetadataToTarget, + RecordSinkOp, EncodeTarget, DecodeTarget, CocoToTorchVisionDetection, @@ -107,7 +100,6 @@ def test_op_group_tags() -> None: assert ConvertToImage.__confluid_group__ == "image" assert SelectFields.__confluid_group__ == "structure" assert PrintSampleOp.__confluid_group__ == "debug" - assert MetadataToTarget.__confluid_group__ == "structure" assert EncodeTarget.__confluid_group__ == "structure" assert DecodeTarget.__confluid_group__ == "structure" assert CocoToTorchVisionDetection.__confluid_group__ == "structure" @@ -120,7 +112,7 @@ def test_op_group_tags() -> None: assert RandomApply.__confluid_group__ == "compose" assert ConfigureOp.__confluid_group__ == "compose" assert FormulaOp.__confluid_group__ == "compose" - assert SampleSinkOp.__confluid_group__ == "sink" + assert RecordSinkOp.__confluid_group__ == "sink" def test_categories_enumerable_via_registry() -> None: @@ -138,15 +130,14 @@ def test_categories_enumerable_via_registry() -> None: "ConvertToImage", "Enable", "Pipeline", - "SampleSinkOp", - "MetadataToTarget", + "RecordSinkOp", "EncodeTarget", "DecodeTarget", "CocoToTorchVisionDetection", "MasksToDetectionBoxes", } <= registry.list_classes(category="op") assert {"HDF5Sink", "ZarrGroupSink", "ZarrBatchSink", "DirectorySink"} <= registry.list_classes(category="sink") - assert "SampleSinkOp" not in registry.list_classes(category="sink") + assert "RecordSinkOp" not in registry.list_classes(category="sink") def test_groups_enumerable_via_registry() -> None: @@ -157,9 +148,8 @@ def test_groups_enumerable_via_registry() -> None: assert {"Parallel", "Enable", "Pipeline", "RandomApply", "ConfigureOp", "FormulaOp"} <= registry.list_classes( group="compose" ) - assert {"SampleSinkOp"} <= registry.list_classes(group="sink") + assert {"RecordSinkOp"} <= registry.list_classes(group="sink") assert { - "MetadataToTarget", "EncodeTarget", "DecodeTarget", "CocoToTorchVisionDetection", diff --git a/tests/test_enable.py b/tests/test_enable.py new file mode 100644 index 0000000..b97b6fe --- /dev/null +++ b/tests/test_enable.py @@ -0,0 +1,69 @@ +"""``Enable`` — the one-flag op-list toggle. + +Pins the toggle-attribute contract: ANY boolean attribute set post-construction is the +toggle and its NAME is the CLI flag; ``enable`` is the documented generic name for the +named-wrapper pattern (``--overlay.enable``); the class's own members (``ops`` / +``enabled`` / ``flag_name``) are RESERVED — read-only properties reject a same-named +YAML kwarg loudly at configure time. +""" + +import pytest + +from sampleflux.ops.enable import Enable + + +def _tag(record): + return {**record, "seen": True} + + +class TestEnableToggle: + def test_enable_named_toggle_off_passes_through(self) -> None: + op = Enable(ops=[_tag]) + op.enable = False # what `enable: false` in YAML does (post-construction setattr) + assert op({"x": 1}) == {"x": 1} + assert op.enabled is False + assert op.flag_name == "enable" + + def test_enable_named_toggle_on_fires_ops(self) -> None: + op = Enable(ops=[_tag]) + op.enable = True + assert op({"x": 1}) == {"x": 1, "seen": True} + assert op.enabled is True + + def test_named_wrappers_toggle_independently(self) -> None: + # Two wrappers, same generic `enable` attr — the instance `name` scopes the CLI flag + # (--overlay.enable vs --labelstudio.enable); here we simulate the post-config state. + overlay, labelstudio = Enable(ops=[_tag]), Enable(ops=[_tag]) + overlay.name, labelstudio.name = "overlay", "labelstudio" + overlay.enable, labelstudio.enable = True, False + assert overlay({"x": 1}) == {"x": 1, "seen": True} + assert labelstudio({"x": 1}) == {"x": 1} + + def test_semantic_toggle_name_still_works(self) -> None: + op = Enable(ops=[_tag]) + op.visualize = True + assert op.flag_name == "visualize" + assert op({"x": 1}) == {"x": 1, "seen": True} + + def test_reserved_names_raise_on_set(self) -> None: + # `enabled` / `flag_name` are read-only introspection properties — a YAML kwarg + # with one of those names fails loudly instead of silently shadowing the API. + for reserved in ("enabled", "flag_name"): + with pytest.raises(AttributeError): + setattr(Enable(ops=[_tag]), reserved, False) + + def test_zero_toggles_raises(self) -> None: + with pytest.raises(RuntimeError, match="exactly one boolean toggle"): + Enable(ops=[_tag])({"x": 1}) + + def test_multiple_toggles_raises(self) -> None: + op = Enable(ops=[_tag]) + op.enable, op.visualize = True, False + with pytest.raises(RuntimeError, match="exactly one boolean toggle"): + op({"x": 1}) + + def test_empty_ops_raises(self) -> None: + op = Enable() + op.enable = True + with pytest.raises(ValueError, match="non-empty 'ops'"): + op({"x": 1}) diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index d32e163..7415d88 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -29,7 +29,6 @@ DecodeTarget, EncodeTarget, MasksToDetectionBoxes, - MetadataToTarget, ) from sampleflux.ops.torch import ToTensor from sampleflux.sources import HuggingFaceSource @@ -46,7 +45,6 @@ ConnectedComponents, ConvertToImage, ToTensor, - MetadataToTarget, EncodeTarget, DecodeTarget, CocoToTorchVisionDetection, diff --git a/tests/test_op_families.py b/tests/test_op_families.py index dc81589..9dd64a6 100644 --- a/tests/test_op_families.py +++ b/tests/test_op_families.py @@ -269,3 +269,110 @@ def test_flux_filter_helper(self) -> None: def test_unset_predicate_raises_lazily(self) -> None: with pytest.raises(ValueError, match="predicate"): FilterOp()({"i": 0}) + + +# --------------------------------------------------------------------------- # +# The open op-family registry (register_op_family) — third-party libraries +# --------------------------------------------------------------------------- # +class FakeLibScale: + """Stands in for a foreign library's op type — deliberately NOT record-callable, + so a test passing proves dispatch went through the registered invoker.""" + + def __init__(self, factor: float = 2.0) -> None: + self.factor = factor + + +def is_fakelib(op: object) -> bool: + """Module-level matcher (pickles by reference for the spawn test).""" + return isinstance(op, FakeLibScale) + + +def invoke_fakelib(record: Record, op: FakeLibScale) -> Record: + """Module-level invoker — the fake library's calling convention.""" + return {**record, "gain_db": record["gain_db"] * op.factor} + + +def invoke_fakelib_override(record: Record, op: FakeLibScale) -> Record: + """A second invoker for the shadowing / replacement tests.""" + return {**record, "gain_db": -999.0} + + +@pytest.fixture() +def family_registry(): + """Snapshot/restore the global registry so registrations never leak between tests.""" + from sampleflux import core + + snapshot = list(core._OP_FAMILIES) + yield + core._OP_FAMILIES[:] = snapshot + + +class TestOpFamilyRegistry: + def test_builtins_are_registered_through_the_same_registry(self) -> None: + from sampleflux import registered_op_families + + assert registered_op_families()[:2] == ("albumentations", "torchvision_v2") + + def test_registered_family_dispatches_via_invoker(self, family_registry) -> None: + from sampleflux import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + out = _apply_op(_base_record(), FakeLibScale(factor=3.0)) + assert out is not None and out["gain_db"] == -9.0 # -3.0 * 3 — via the invoker, op never called + assert isinstance(out["image"], Image) # rest of the record untouched + + def test_registered_family_runs_in_flux_ops_list(self, family_registry) -> None: + from sampleflux import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + out = list(Flux(source=[_base_record()], ops=[FakeLibScale(factor=2.0), lambda r: {**r, "tag": 1}])) + assert out[0]["gain_db"] == -6.0 and out[0]["tag"] == 1 # mixes with native ops in ONE list + + def test_last_registered_family_wins_overlap(self, family_registry) -> None: + from sampleflux import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + register_op_family("fakelib_specific", is_fakelib, invoke_fakelib_override) # same matcher, later + out = _apply_op(_base_record(), FakeLibScale()) + assert out is not None and out["gain_db"] == -999.0 + + def test_reregistering_name_replaces_in_place(self, family_registry) -> None: + from sampleflux import register_op_family, registered_op_families + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + n = len(registered_op_families()) + register_op_family("fakelib", is_fakelib, invoke_fakelib_override) + assert len(registered_op_families()) == n # replaced, not duplicated + out = _apply_op(_base_record(), FakeLibScale()) + assert out is not None and out["gain_db"] == -999.0 + + def test_unmatched_op_falls_back_to_native_call(self, family_registry) -> None: + out = _apply_op(_base_record(), lambda r: {**r, "native": True}) + assert out is not None and out["native"] is True + + def test_spawn_parallel_ships_family_to_workers(self, family_registry) -> None: + from sampleflux import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + flux = Flux(source=spawn_records(), ops=[FakeLibScale(factor=2.0)]).parallel(2) + results = list(flux) + assert len(results) == 4 + assert all(r["gain_db"] == -6.0 for r in results) # invoker ran INSIDE the workers + + +class TestFormulaReducers: + def test_array_reducers_are_function_style(self) -> None: + # amax/amin/mean/std/median are pre-bound numpy callables in the sandbox namespace. + from sampleflux.ops.formula import FormulaOp + + rec = {"image": Image(np.arange(16, dtype=np.float32).reshape(4, 4) / 15.0)} + out = FormulaOp(formula="amax(a) * 0.5", field="image")(rec) + assert float(np.asarray(out["image"])) == pytest.approx(0.5) + + def test_attribute_reduction_is_not_part_of_the_contract(self) -> None: + # a.max() depends on numpy's lazy-import cache (KeyError '__import__' in a cold + # process): the FUNCTION form is the sanctioned spelling. We only pin that the + # function form never regresses; the attribute form is deliberately unpinned. + from sampleflux.ops.formula import _FORMULA_NAMESPACE + + assert {"amax", "amin", "mean", "std", "median"} <= set(_FORMULA_NAMESPACE) diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py index 08a4dc1..153401e 100644 --- a/tests/test_typed_collate.py +++ b/tests/test_typed_collate.py @@ -6,7 +6,7 @@ import pytest import torch -from sampleflux import Image, Label, Mask, Record, collate, collate_records, get_collate, register_item +from sampleflux import Image, Label, Mask, Record, Regions, collate, collate_records, get_collate, register_item @register_item @@ -79,3 +79,55 @@ def test_non_dict_items_raise(self) -> None: def test_unknown_key_raises_with_known_keys(self) -> None: with pytest.raises(KeyError, match="no collate registered"): get_collate("nope") + + +# --------------------------------------------------------------------------- # +# The collate REGISTRY: a task collate opts out of the generic folding rules +# (the docs/record-model.md detection example — variable-N boxes cannot stack). +# --------------------------------------------------------------------------- # +def _ragged_detection_records(): + return [ + { + "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), + "target": Regions(boxes=[[0, 0, 2, 2]], labels=[1]), + "pack": "a", + }, + { + "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), + "target": Regions(boxes=[[0, 0, 1, 1], [1, 1, 3, 3], [0, 2, 2, 4]], labels=[0, 1, 0]), + "pack": "b", + }, + ] + + +def test_generic_collate_leaves_ragged_regions_as_lists() -> None: + # Rule 2: the generic fold can only give per-record lists for a wrapper item — + # exactly why detection registers its own collate. + batch = collate_records(_ragged_detection_records()) + assert isinstance(batch["target"], Regions) + assert [len(b) for b in batch["target"].boxes] == [1, 3] + + +def test_registered_task_collate_produces_its_own_batch_contract() -> None: + import torch + + from sampleflux import collate, get_collate, register_collate + + @register_collate("_test_detection") + def detection_collate(items): + images = torch.stack([torch.as_tensor(np.asarray(r["image"])).permute(2, 0, 1) for r in items]) + targets = [ + { + "boxes": torch.as_tensor(r["target"].boxes, dtype=torch.float32).reshape(-1, 4), + "labels": torch.as_tensor(r["target"].labels, dtype=torch.int64), + } + for r in items + ] + metadata = [{k: v for k, v in r.items() if k not in ("image", "target")} for r in items] + return {"images": images, "targets": targets, "metadata": metadata} + + assert get_collate("_test_detection") is detection_collate + batch = collate(_ragged_detection_records(), key="_test_detection") + assert tuple(batch["images"].shape) == (2, 3, 4, 4) + assert [tuple(t["boxes"].shape) for t in batch["targets"]] == [(1, 4), (3, 4)] # raggedness preserved + assert batch["metadata"] == [{"pack": "a"}, {"pack": "b"}] diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index 669f4ae..1a813d4 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -218,3 +218,65 @@ def test_compose_ops_route_records(self) -> None: flux = Flux(source=[_seed(1.0)], ops=[Pipeline(transforms=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])]) (out,) = list(flux) assert np.allclose(np.asarray(out["image"]), 4.0) + + +# --------------------------------------------------------------------------- # +# YAML flow documents end-to-end (docs/graph.md's exact spellings) — closes the +# gap where bind: was only ever exercised through Python-built flow dicts. +# --------------------------------------------------------------------------- # +class TestFlowYaml: + def _doc(self) -> str: + return """ +flow: + spec: {} + masked: !class:sampleflux.ops.numpy.Threshold {low_level: 0.5, from: spec} + thresh: !class:sampleflux.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} + gated: + op: !class:sampleflux.ops.numpy.Threshold {output: gated_mask} + from: spec + bind: + low_level: thresh[image] + out: {from: gated, merge_from: [masked]} +outputs: out +""" + + def _record(self): + return {"image": Image(np.arange(16, dtype=np.float32).reshape(4, 4) / 15.0)} + + def test_yaml_bind_via_plain_mapping_step(self, tmp_path) -> None: + # Scalar reserved keys ride in the marker mapping; the nested bind: mapping MUST use + # the plain-mapping (op:) step form — a nested mapping under a !class: marker is + # consumed by confluid as addressed configuration and never reaches parse_flow. + path = tmp_path / "graph.yaml" + path.write_text(self._doc()) + out = list(FlowGraph.from_yaml(str(path), source=[self._record()]))[0] + assert set(out) == {"image", "mask", "gated_mask"} + assert int(np.asarray(out["mask"]).sum()) == 8 # fixed 0.5 threshold + assert int(np.asarray(out["gated_mask"]).sum()) == 6 # per-record amax(a)*0.6 bind + + def test_yaml_bind_parity_with_lowered_flux(self, tmp_path) -> None: + path = tmp_path / "graph.yaml" + path.write_text(self._doc()) + a = list(FlowGraph.from_yaml(str(path), source=[self._record()]))[0] + b = list(Flux.from_flow_yaml(str(path), source=[self._record()]))[0] + assert np.array_equal(np.asarray(a["gated_mask"]), np.asarray(b["gated_mask"])) + assert np.array_equal(np.asarray(a["mask"]), np.asarray(b["mask"])) + + def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path) -> None: + # Pin the confluid behavior that makes the op:-form MANDATORY for bind — if this + # ever starts surviving in marker kwargs, the doc rule can be relaxed. + path = tmp_path / "graph.yaml" + path.write_text( + """ +flow: + spec: {} + gated: !class:sampleflux.ops.numpy.Threshold + from: spec + bind: + low_level: spec[image] +""" + ) + import confluid + + marker = confluid.resolve(str(path))["flow"]["gated"] + assert "bind" not in marker.kwargs # consumed as addressed configuration diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py index 24d7067..d90e41c 100644 --- a/tests/test_typed_target_ops.py +++ b/tests/test_typed_target_ops.py @@ -4,7 +4,6 @@ encoded TARGET ``Label`` on plain record dicts: * :class:`sampleflux.ops.torch.ToTensor` — array-bearing key → a LIVE CHW-float ``torch.Tensor`` (a plain record value); -* :class:`sampleflux.ops.target.MetadataToTarget` — a key / attr value → a target ``Label``; * :class:`sampleflux.ops.target.EncodeTarget` / ``DecodeTarget`` — class-name ↔ class-id ``Label``. Each op REUSES its shared conversion helper, so the op output is pinned identical to the @@ -18,7 +17,7 @@ from sampleflux import Image, Label, Mask, collate_records, item_data from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.target import DecodeTarget, EncodeTarget, MetadataToTarget +from sampleflux.ops.target import DecodeTarget, EncodeTarget from sampleflux.ops.torch import ToTensor, to_tensor _MAP = {"cat": 0, "dog": 1, "fox": 2} @@ -96,38 +95,6 @@ def test_record_collate_stacks_payloads(self) -> None: assert np.asarray(batch["image"]).shape == (2, 3, 4, 5) -# --------------------------------------------------------------------------- # -# MetadataToTarget -# --------------------------------------------------------------------------- # -class TestMetadataToTarget: - def test_promotes_label_value_to_target(self) -> None: - out = MetadataToTarget(field="class", output="target")({"class": Label("cat")}) - assert isinstance(out["target"], Label) - assert out["target"].value == "cat" - - def test_default_picks_first_label(self) -> None: - rec = {"image": Image(_hwc_uint8()), "y": Label("dog")} - out = MetadataToTarget()(rec) - assert out["target"].value == "dog" - - def test_read_named_attribute(self) -> None: - # Read a carried attribute off an item (metadata lives ON the value that owns it). - out = MetadataToTarget(field="y", key="classes", output="vocab")({"y": Label("cat", classes=["cat", "dog"])}) - assert out["vocab"].value == ["cat", "dog"] - - def test_missing_attribute_raises(self) -> None: - with pytest.raises(AttributeError, match="no attribute 'nope'"): - MetadataToTarget(field="y", key="nope")({"y": Label("cat")}) - - def test_missing_field_raises(self) -> None: - with pytest.raises(ValueError, match="field 'nope' not in record"): - MetadataToTarget(field="nope")({"y": Label("cat")}) - - def test_empty_record_raises(self) -> None: - with pytest.raises(ValueError, match="record is empty"): - MetadataToTarget()({}) - - # --------------------------------------------------------------------------- # # EncodeTarget / DecodeTarget # --------------------------------------------------------------------------- # @@ -222,7 +189,6 @@ def test_convert_then_tensor_chain() -> None: # --------------------------------------------------------------------------- # def test_zero_arg_constructible() -> None: assert ToTensor().output == "" - assert MetadataToTarget().output == "target" assert EncodeTarget().mapping == {} assert DecodeTarget().mapping == {} @@ -231,7 +197,6 @@ def test_zero_arg_constructible() -> None: ("name", "cls", "group"), [ ("ToTensor", ToTensor, "torch"), - ("MetadataToTarget", MetadataToTarget, "structure"), ("EncodeTarget", EncodeTarget, "structure"), ("DecodeTarget", DecodeTarget, "structure"), ], diff --git a/tests/test_workflow.py b/tests/test_workflow.py index cd15162..97d8e4e 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -112,12 +112,12 @@ def test_conditional_accepts_predicate_condition() -> None: # Switch # --------------------------------------------------------------------------- # def test_switch_selects_matching_case() -> None: - Switch(selector=lambda: "b", cases={"a": _RunStep("a"), "b": _RunStep("b")}).run() + Switch(select=lambda: "b", cases={"a": _RunStep("a"), "b": _RunStep("b")}).run() assert _RUN_LOG == ["b"] def test_switch_falls_back_to_default_on_miss() -> None: - Switch(selector=lambda: "z", cases={"a": _RunStep("a")}, default=_RunStep("d")).run() + Switch(select=lambda: "z", cases={"a": _RunStep("a")}, default=_RunStep("d")).run() assert _RUN_LOG == ["d"] @@ -127,12 +127,12 @@ def test_switch_none_selector_uses_default() -> None: def test_switch_no_match_no_default_is_noop() -> None: - Switch(selector=lambda: "z", cases={"a": _RunStep("a")}).run() + Switch(select=lambda: "z", cases={"a": _RunStep("a")}).run() assert _RUN_LOG == [] def test_switch_coerces_non_string_key() -> None: - Switch(selector=lambda: 2, cases={"2": _RunStep("two")}).run() + Switch(select=lambda: 2, cases={"2": _RunStep("two")}).run() assert _RUN_LOG == ["two"] From 8ac08f48ee4e7e417f8ceea9360788d29c20b56c Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 26 Jul 2026 15:19:21 +0200 Subject: [PATCH 042/102] docs: project-owned backlog + current-state section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts the 2026-07-26 workspace doc split: this project now owns its own TASKS.md (open items only — completed work is git history, not a [x] graveyard) and its AGENTS.md opens with a compacted 'Current state' section moved out of the workspace-root file. Stale claims found during the move were corrected against the source. --- AGENTS.md | 4 ++++ TASKS.md | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 TASKS.md diff --git a/AGENTS.md b/AGENTS.md index 5dcab0a..781995a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,9 @@ # SampleFlux Mandates +## Current state + +Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Flux`/`JointFlux`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`sampleflux run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. + - **The Runnable Protocol Lives Here (`sampleflux.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** sampleflux owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `sampleflux.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `sampleflux.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `sampleflux.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `flux` validated in `run()`). `sampleflux.cli`: the `sampleflux run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `sampleflux-processing`/`sampleflux-workflow` + the `sampleflux` console script + `liquifai.apps`. - **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`sampleflux.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. diff --git a/TASKS.md b/TASKS.md new file mode 100644 index 0000000..1f3404c --- /dev/null +++ b/TASKS.md @@ -0,0 +1,21 @@ +# sampleflux — backlog + +Open work for this project. Cross-cutting / multi-project initiatives live in the +workspace root `TASKS.md`. Completed items are not archived here — git history is the record. + +- [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of sampleflux verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor +- [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `sampleflux.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in sampleflux; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low +- [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (sampleflux, waivefront, marainer, …) were not. @low +- [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in sampleflux, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor +- [ ] GPU-aware batch processing engine @performance +- [ ] S3 storage backend support @feature +- [ ] **SampleFlux Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance +- [ ] **Typed-bag PoC → torch-`Tensor`-subclass item base:** array-backed items are `np.ndarray` subclasses only; add a `TensorItem` base (torch `__torch_function__` attr-preservation) so torch payloads can be array items instead of riding wrapper `Signal`s. @low @ml +- [ ] **Typed-bag PoC → confluid-native item-type discovery:** item types register in a local `register_item` registry (an `np.ndarray` subclass fights confluid's `__init__` validation wrap); make item types `@configurable(category="itemtype")` + entry-pointed so navigaitor/FluxStudio enumerate them as socket types. @low @tooling +- [ ] **Record model → inverse-transform path:** design an inverse/back-projection hook on `sampleflux.transform.Transform` (the old `Transform.decode` stub was dropped in the record conversion) for visualization / inference back-projection (e.g. Spectrogram→Signal, region px→signal coords). @low @ml +- [ ] **S3 storage backend:** extend the `DataSource`/`DataSink` protocols (HDF5/Zarr/Directory exist) with cloud-native S3 support. @feature +- [ ] **Stratified splits** — per-class balanced train/val/test on `DatasetSplit` so small classes don't fall entirely into one partition. @feature +- [ ] **K-fold cross-validation** — a `KFoldSplit` companion yielding fold views; a single config loops over folds. @feature +- [ ] **Grouped splits** — honour a `group_by` metadata key (patient id, source file) so samples from one group never leak across train/val. @feature +- [ ] **Pre-computed split manifests** — export the train/val index lists + seed as Confluid artifacts for reproducibility and dataset cards. @low @feature +- [ ] **Auto-split when `val_set` is missing** — implicit fraction split if only `train_set` is wired; deliberately deferred in favour of explicit YAML, revisit for ergonomics. @low From 854d98ba8b7ba5b753fd102c83e7a9448ad39c33 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 27 Jul 2026 12:23:03 +0200 Subject: [PATCH 043/102] refactor!: rename sampleflux -> recordstream and Flux -> Stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: import name, distribution name, GitHub repo, console script, every entry point, and the on-disk store attribute all change. No aliases. The package was named for a data model it no longer has: the 2026-07-25 migration made the carrier a record (`items.Record = Dict[str, Any]`), leaving "sampleflux" describing nothing in the tree. The vocabulary is now one word per concept — a `Stream` of `Record`s — which also closes a long-standing split where the GUI wire type said SAMPLE while the Python alias said Record. - package/dist `sampleflux` -> `recordstream`; repo Gearlux/sampleflux renamed - `Flux` -> `Stream`, `JointFlux` -> `JointStream`, `JointFlux.fluxes` -> `JointStream.streams`, `DatasetProcessor.flux` -> `.stream` - console script + runner `sampleflux run` -> `recordstream run`; all `sampleflux-*` entry points -> `recordstream-*` - `PrintSampleOp` -> `PrintRecordOp`, `sample_to_image` -> `record_to_image`, `per_sample` -> `per_record`, `_iter_samples` -> `_iter_records`, storage `sample_dir` -> `record_dir`, Zarr group prefix `record_NNNNNN` - storage root attr `sampleflux_format` -> `recordstream_format`; the tag value stays `typedrecord-v1`, so a pre-rename store hits the existing "re-generate with a current sink" error rather than a silent misread "sample" is deliberately preserved where it means something else: a stochastic draw (`Transform` still samples its params once per record) and the test fixtures that mean "example" (`sample_func`, `SampleClass`). 421 tests pass; examples record_pipeline / workflow_pipeline / storage_roundtrip and the cat_exploration notebook all execute. --- .coveragerc | 2 +- .github/workflows/ci.yml | 8 +- AGENTS.md | 66 ++++--- CLAUDE.md | 41 ++++- GEMINI.md | 41 ++++- Jenkinsfile | 38 ++-- Jenkinsfile.local | 42 ++--- README.md | 32 ++-- TASKS.md | 18 +- docs/architecture.md | 114 ++++++------ docs/augmentation.md | 22 +-- docs/configure.md | 14 +- docs/graph.md | 42 ++--- docs/image.md | 8 +- docs/kinds.md | 28 +-- docs/projection.md | 12 +- docs/record-model.md | 72 ++++---- docs/runnable.md | 10 +- docs/sources.md | 28 +-- docs/storage.md | 28 +-- docs/workflow.md | 12 +- examples/cache_pipeline.py | 6 +- examples/discovery_demo.py | 4 +- examples/record_pipeline.py | 6 +- examples/storage_roundtrip.py | 10 +- examples/workflow_pipeline.py | 12 +- pyproject.toml | 84 ++++----- {sampleflux => recordstream}/__init__.py | 48 ++--- {sampleflux => recordstream}/cli.py | 18 +- {sampleflux => recordstream}/collate.py | 10 +- {sampleflux => recordstream}/context.py | 34 ++-- {sampleflux => recordstream}/core.py | 170 +++++++++--------- {sampleflux => recordstream}/discovery.py | 4 +- {sampleflux => recordstream}/dispatch.py | 2 +- {sampleflux => recordstream}/flow.py | 100 +++++------ {sampleflux => recordstream}/io.py | 8 +- {sampleflux => recordstream}/items.py | 6 +- {sampleflux => recordstream}/labels.py | 16 +- recordstream/ops/__init__.py | 66 +++++++ {sampleflux => recordstream}/ops/configure.py | 12 +- {sampleflux => recordstream}/ops/context.py | 22 +-- {sampleflux => recordstream}/ops/debug.py | 8 +- {sampleflux => recordstream}/ops/enable.py | 30 ++-- {sampleflux => recordstream}/ops/formula.py | 2 +- {sampleflux => recordstream}/ops/image.py | 30 ++-- {sampleflux => recordstream}/ops/numpy.py | 18 +- {sampleflux => recordstream}/ops/parallel.py | 20 +-- .../ops/random_apply.py | 8 +- {sampleflux => recordstream}/ops/sink.py | 16 +- {sampleflux => recordstream}/ops/structure.py | 2 +- {sampleflux => recordstream}/ops/target.py | 28 +-- {sampleflux => recordstream}/ops/torch.py | 8 +- {sampleflux => recordstream}/processing.py | 84 ++++----- {sampleflux => recordstream}/projection.py | 12 +- {sampleflux => recordstream}/py.typed | 0 {sampleflux => recordstream}/runnable.py | 2 +- {sampleflux => recordstream}/sources.py | 40 ++--- {sampleflux => recordstream}/storage/base.py | 10 +- {sampleflux => recordstream}/storage/cache.py | 0 .../storage/directory.py | 36 ++-- {sampleflux => recordstream}/storage/hdf5.py | 24 +-- {sampleflux => recordstream}/storage/query.py | 16 +- {sampleflux => recordstream}/storage/zarr.py | 26 +-- {sampleflux => recordstream}/transform.py | 18 +- {sampleflux => recordstream}/workflow.py | 16 +- sampleflux/ops/__init__.py | 66 ------- tests/_fixtures.py | 4 +- tests/test_cache.py | 2 +- tests/test_categories.py | 50 +++--- tests/test_cli_run.py | 4 +- tests/test_discovery.py | 10 +- tests/test_dispatch.py | 4 +- tests/test_enable.py | 2 +- tests/test_entrypoint.py | 2 +- tests/test_io.py | 6 +- tests/test_items.py | 4 +- tests/test_labels.py | 10 +- tests/test_lazy_construction.py | 22 +-- tests/test_node_docs.py | 43 ++--- tests/test_op_families.py | 78 ++++---- tests/test_parallel.py | 6 +- tests/test_pipeline.py | 6 +- tests/test_runnable.py | 2 +- tests/test_structure_ops.py | 4 +- tests/test_transform.py | 4 +- tests/test_typed_collate.py | 4 +- tests/test_typed_detection_target_ops.py | 12 +- tests/test_typed_flow.py | 46 ++--- tests/test_typed_generic_ops.py | 14 +- tests/test_typed_storage.py | 20 +-- tests/test_typed_target_ops.py | 17 +- tests/test_workflow.py | 10 +- 92 files changed, 1159 insertions(+), 1063 deletions(-) mode change 120000 => 100644 CLAUDE.md mode change 120000 => 100644 GEMINI.md rename {sampleflux => recordstream}/__init__.py (61%) rename {sampleflux => recordstream}/cli.py (71%) rename {sampleflux => recordstream}/collate.py (93%) rename {sampleflux => recordstream}/context.py (76%) rename {sampleflux => recordstream}/core.py (83%) rename {sampleflux => recordstream}/discovery.py (98%) rename {sampleflux => recordstream}/dispatch.py (97%) rename {sampleflux => recordstream}/flow.py (91%) rename {sampleflux => recordstream}/io.py (94%) rename {sampleflux => recordstream}/items.py (97%) rename {sampleflux => recordstream}/labels.py (89%) create mode 100644 recordstream/ops/__init__.py rename {sampleflux => recordstream}/ops/configure.py (91%) rename {sampleflux => recordstream}/ops/context.py (94%) rename {sampleflux => recordstream}/ops/debug.py (96%) rename {sampleflux => recordstream}/ops/enable.py (86%) rename {sampleflux => recordstream}/ops/formula.py (98%) rename {sampleflux => recordstream}/ops/image.py (97%) rename {sampleflux => recordstream}/ops/numpy.py (94%) rename {sampleflux => recordstream}/ops/parallel.py (84%) rename {sampleflux => recordstream}/ops/random_apply.py (93%) rename {sampleflux => recordstream}/ops/sink.py (79%) rename {sampleflux => recordstream}/ops/structure.py (99%) rename {sampleflux => recordstream}/ops/target.py (94%) rename {sampleflux => recordstream}/ops/torch.py (93%) rename {sampleflux => recordstream}/processing.py (63%) rename {sampleflux => recordstream}/projection.py (91%) rename {sampleflux => recordstream}/py.typed (100%) rename {sampleflux => recordstream}/runnable.py (99%) rename {sampleflux => recordstream}/sources.py (94%) rename {sampleflux => recordstream}/storage/base.py (93%) rename {sampleflux => recordstream}/storage/cache.py (100%) rename {sampleflux => recordstream}/storage/directory.py (84%) rename {sampleflux => recordstream}/storage/hdf5.py (90%) rename {sampleflux => recordstream}/storage/query.py (95%) rename {sampleflux => recordstream}/storage/zarr.py (91%) rename {sampleflux => recordstream}/transform.py (92%) rename {sampleflux => recordstream}/workflow.py (94%) delete mode 100644 sampleflux/ops/__init__.py diff --git a/.coveragerc b/.coveragerc index f034c22..b346d37 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -source = sampleflux +source = recordstream omit = */tests/* */examples/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 175e50d..b97774e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,10 @@ # ========================================================================= # AUTO-GENERATED FILE — DO NOT EDIT BY HAND -# Generated by: aisland jenkins scaffold --project sampleflux +# Generated by: aisland jenkins scaffold --project recordstream # Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -# To regenerate: aisland jenkins scaffold --project sampleflux --force +# To regenerate: aisland jenkins scaffold --project recordstream --force # ========================================================================= -name: Sampleflux CI +name: Recordstream CI on: push: @@ -64,7 +64,7 @@ jobs: - name: Run Tests run: | if [ -d tests ] && find tests -name '*.py' | grep -q .; then - pytest tests --junitxml=test-report.xml --cov=sampleflux --cov-report=xml --cov-report=term + pytest tests --junitxml=test-report.xml --cov=recordstream --cov-report=xml --cov-report=term else echo "No tests found. Skipping." fi diff --git a/AGENTS.md b/AGENTS.md index 781995a..31fa280 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,37 +1,49 @@ -# SampleFlux Mandates +# RecordStream Mandates ## Current state -Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Flux`/`JointFlux`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`sampleflux run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. +> **Renamed 2026-07-26 — `sampleflux` → `recordstream`, `Flux` → `Stream`.** The package was named +> for a data model it no longer has: the 2026-07-25 migration made the carrier a **record**, so the +> vocabulary is now one word per concept — a **`Stream`** of **`Record`**s. Import name, distribution +> name, GitHub repo, console script (`recordstream run`), every `recordstream-*` entry point, the +> `RECORDSTREAM_*` StreamStudio socket types, and the on-disk root attr (`recordstream_format`, +> value still `typedrecord-v1`) all moved together; `JointFlux.fluxes` is `JointStream.streams`. +> **No back-compat aliases** — a pre-rename config, saved canvas, or store must be re-pointed +> (a store missing `recordstream_format` raises the usual re-generate error). The word *sample* is +> now reserved for its OTHER meanings and was deliberately NOT renamed: a discrete-time signal +> sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still +> *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). -- **The Runnable Protocol Lives Here (`sampleflux.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** sampleflux owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `sampleflux.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `sampleflux.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `sampleflux.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `flux` validated in `run()`). `sampleflux.cli`: the `sampleflux run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `sampleflux-processing`/`sampleflux-workflow` + the `sampleflux` console script + `liquifai.apps`. -- **SampleFlux Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). -- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`sampleflux.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. -- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `sampleflux.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The RECORD Is THE Data Model (2026-07-25):** A sample is a **PLAIN `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `sampleflux.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`sampleflux.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `sampleflux.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from sampleflux import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). sampleflux ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `sampleflux/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `sampleflux.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Flux._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). -- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-sample flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. + +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). +- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. +- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `sampleflux.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`sampleflux.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(sample, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with sampleflux.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `sampleflux-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. -- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`sampleflux.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Flux`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Flux's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `sampleflux-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Flux.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY sampleflux `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. -- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`sampleflux.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. -- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `sampleflux.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). -- **Collation Is a Pluggable Registry (`sampleflux.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. -- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_sample` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Flux.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Flux._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(flux)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. +- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `recordstream.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`recordstream.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(record, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with recordstream.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `recordstream-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. +- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`recordstream.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Stream`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Stream's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `recordstream-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Stream.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. -- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into FluxStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `sampleflux/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `sampleflux_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `sample_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `sampleflux/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `sampleflux.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`sampleflux-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. -- **Metadata Is QUERYABLE Without Array Loads (`sampleflux.storage.query`, 2026-07-17):** `sampleflux.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `sampleflux-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; sampleflux keeps ZERO knowledge of it. -- **Key Projection (`sampleflux.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Flux.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Flux` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Flux` look classification-capable to duck-typed consumers. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`sampleflux.labels`):** `EncodeTarget` / `DecodeTarget` (`sampleflux.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing sampleflux never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a sampleflux dependency for this. -- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The sampleflux buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Flux` / `JointFlux` / `FlowGraph` (a `Flux` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in FluxStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Flux.source` (single `SAMPLEFLUX_SOURCE` input) + `Flux.ops` (dynamic `op_N` `SAMPLEFLUX_OP` inputs), `JointFlux.fluxes` (dynamic `source_N` `SAMPLEFLUX_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from FluxStudio as uncategorised (nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. FluxStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `SAMPLEFLUX_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. FluxStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; FluxStudio nests the palette as `Taidal/SampleFlux/Op/`). The sampleflux groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `sampleflux.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`sampleflux.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; FluxStudio renders `ops` as `op_N` sockets and `target` as ONE `SAMPLEFLUX_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`sampleflux.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintSampleOp` = `sampleflux.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`sampleflux.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `sample_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing FluxStudio's in-canvas viewer nodes (`fluxstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `fluxstudio.nodes.SampleExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in sampleflux (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `sample_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for FluxStudio's *Draw Text to Image* node (`fluxstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. - Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Flux` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from FluxStudio). +- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. +- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing recordstream never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a recordstream dependency for this. +- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. + Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). - **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) ## Testing & Validation diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..49da027 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,40 @@ +# RecordStream Mandates + +## Current state + +Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. + +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). +- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. +- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +- **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. +- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `recordstream.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`recordstream.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(record, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with recordstream.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `recordstream-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. +- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`recordstream.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Stream`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Stream's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `recordstream-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Stream.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. +- **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. +- **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. +- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. +- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing recordstream never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a recordstream dependency for this. +- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. + Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). +- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) + +## Testing & Validation +- **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. +- **Multiprocess Safety:** Parallel pipelines MUST use the `spawn` context. Verify pickle-safety of all operations. +- **Line Length:** 120 characters (Black, isort, flake8). diff --git a/GEMINI.md b/GEMINI.md deleted file mode 120000 index 47dc3e3..0000000 --- a/GEMINI.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..49da027 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,40 @@ +# RecordStream Mandates + +## Current state + +Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. + +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). +- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. +- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +- **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. +- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `recordstream.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`recordstream.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(record, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with recordstream.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `recordstream-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. +- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`recordstream.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Stream`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Stream's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `recordstream-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Stream.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. +- **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. +- **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. +- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. +- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing recordstream never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a recordstream dependency for this. +- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. +- **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. + Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). +- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) + +## Testing & Validation +- **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. +- **Multiprocess Safety:** Parallel pipelines MUST use the `spawn` context. Verify pickle-safety of all operations. +- **Line Length:** 120 characters (Black, isort, flake8). diff --git a/Jenkinsfile b/Jenkinsfile index 63be8de..e38150d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,8 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project sampleflux +// Generated by: aisland jenkins scaffold --project recordstream // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project sampleflux --force +// To regenerate: aisland jenkins scaffold --project recordstream --force // ========================================================================= pipeline { agent any @@ -51,7 +51,7 @@ pipeline { steps { script { sh "rm -f black-diff.txt black-checkstyle.xml" - def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/black --check --diff ${targets} 2>&1 | tee black-diff.txt'", returnStatus: true) @@ -87,8 +87,8 @@ with open('black-checkstyle.xml', 'w') as f: script { if (fileExists('black-checkstyle.xml')) { recordIssues( - id: 'black-sampleflux', - name: 'Black Formatting (Sampleflux)', + id: 'black-recordstream', + name: 'Black Formatting (Recordstream)', tools: [checkStyle(pattern: 'black-checkstyle.xml')] ) } @@ -100,7 +100,7 @@ with open('black-checkstyle.xml', 'w') as f: steps { script { sh "rm -f isort-diff.txt isort-checkstyle.xml" - def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/isort --check-only --diff ${targets} 2>&1 | tee isort-diff.txt'", returnStatus: true) @@ -136,8 +136,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('isort-checkstyle.xml')) { recordIssues( - id: 'isort-sampleflux', - name: 'Isort Import Order (Sampleflux)', + id: 'isort-recordstream', + name: 'Isort Import Order (Recordstream)', tools: [checkStyle(pattern: 'isort-checkstyle.xml')] ) } @@ -149,7 +149,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { sh "rm -f flake8.txt" - def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { sh "${VENV_BIN}/flake8 ${targets} --tee --output-file=flake8.txt" } else { @@ -162,8 +162,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('flake8.txt') && readFile('flake8.txt').trim()) { recordIssues( - id: 'flake8-sampleflux', - name: 'Flake8 (Sampleflux)', + id: 'flake8-recordstream', + name: 'Flake8 (Recordstream)', tools: [flake8(pattern: 'flake8.txt')] ) } @@ -192,8 +192,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('mypy.txt') && readFile('mypy.txt').trim()) { recordIssues( - id: 'mypy-sampleflux', - name: 'Mypy (Sampleflux)', + id: 'mypy-recordstream', + name: 'Mypy (Recordstream)', tools: [myPy(pattern: 'mypy.txt')] ) } @@ -208,7 +208,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { if (fileExists('tests') && sh(script: "find tests -name '*.py' | grep -q .", returnStatus: true) == 0) { - sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=sampleflux --cov-report=xml:coverage.xml --cov-report=term" + sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=recordstream --cov-report=xml:coverage.xml --cov-report=term" } else { echo "No tests found in 'tests' directory. Skipping." } @@ -222,8 +222,8 @@ with open('isort-checkstyle.xml', 'w') as f: } if (fileExists('coverage.xml')) { recordCoverage( - id: 'coverage-sampleflux', - name: 'Sampleflux Coverage', + id: 'coverage-recordstream', + name: 'Recordstream Coverage', tools: [[parser: 'COBERTURA', pattern: 'coverage.xml']] ) } @@ -298,13 +298,13 @@ with open('isort-checkstyle.xml', 'w') as f: post { always { - echo 'Sampleflux Pipeline Complete.' + echo 'Recordstream Pipeline Complete.' } success { - echo 'Sampleflux is healthy.' + echo 'Recordstream is healthy.' } failure { - echo 'Sampleflux build failed. Please check linting or test failures.' + echo 'Recordstream build failed. Please check linting or test failures.' } } } diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 5bf079d..8d4af5c 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -1,14 +1,14 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project sampleflux +// Generated by: aisland jenkins scaffold --project recordstream // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project sampleflux --force +// To regenerate: aisland jenkins scaffold --project recordstream --force // ========================================================================= pipeline { agent { node { label 'built-in' - customWorkspace "${env.WORKSPACE_ROOT}/sampleflux" + customWorkspace "${env.WORKSPACE_ROOT}/recordstream" } } @@ -60,7 +60,7 @@ pipeline { steps { script { sh "rm -f black-diff.txt black-checkstyle.xml" - def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/black --check --diff ${targets} 2>&1 | tee black-diff.txt'", returnStatus: true) @@ -96,8 +96,8 @@ with open('black-checkstyle.xml', 'w') as f: script { if (fileExists('black-checkstyle.xml')) { recordIssues( - id: 'black-sampleflux', - name: 'Black Formatting (Sampleflux)', + id: 'black-recordstream', + name: 'Black Formatting (Recordstream)', tools: [checkStyle(pattern: 'black-checkstyle.xml')] ) } @@ -109,7 +109,7 @@ with open('black-checkstyle.xml', 'w') as f: steps { script { sh "rm -f isort-diff.txt isort-checkstyle.xml" - def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/isort --check-only --diff ${targets} 2>&1 | tee isort-diff.txt'", returnStatus: true) @@ -145,8 +145,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('isort-checkstyle.xml')) { recordIssues( - id: 'isort-sampleflux', - name: 'Isort Import Order (Sampleflux)', + id: 'isort-recordstream', + name: 'Isort Import Order (Recordstream)', tools: [checkStyle(pattern: 'isort-checkstyle.xml')] ) } @@ -158,7 +158,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { sh "rm -f flake8.txt" - def targets = sh(script: "for d in sampleflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { sh "${VENV_BIN}/flake8 ${targets} --tee --output-file=flake8.txt" } else { @@ -171,8 +171,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('flake8.txt') && readFile('flake8.txt').trim()) { recordIssues( - id: 'flake8-sampleflux', - name: 'Flake8 (Sampleflux)', + id: 'flake8-recordstream', + name: 'Flake8 (Recordstream)', tools: [flake8(pattern: 'flake8.txt')] ) } @@ -187,7 +187,7 @@ with open('isort-checkstyle.xml', 'w') as f: // Run from workspace root so root mypy.ini is picked up and cross-project imports resolve. // tee streams errors to the Jenkins console; pipefail ensures mypy's exit code (not tee's) propagates // so the stage fails visibly when types break. - def exitCode = sh(script: "bash -c 'set -o pipefail; cd ${env.WORKSPACE_ROOT} && ${VENV_BIN}/mypy sampleflux 2>&1 | tee ${env.WORKSPACE_ROOT}/sampleflux/mypy.txt'", returnStatus: true) + def exitCode = sh(script: "bash -c 'set -o pipefail; cd ${env.WORKSPACE_ROOT} && ${VENV_BIN}/mypy recordstream 2>&1 | tee ${env.WORKSPACE_ROOT}/recordstream/mypy.txt'", returnStatus: true) if (exitCode != 0) { if (params.REPORT_ALL_WARNINGS) { unstable("Mypy found type errors. See console output above and Mypy report.") @@ -202,8 +202,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('mypy.txt') && readFile('mypy.txt').trim()) { recordIssues( - id: 'mypy-sampleflux', - name: 'Mypy (Sampleflux)', + id: 'mypy-recordstream', + name: 'Mypy (Recordstream)', tools: [myPy(pattern: 'mypy.txt')] ) } @@ -218,7 +218,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { if (fileExists('tests') && sh(script: "find tests -name '*.py' | grep -q .", returnStatus: true) == 0) { - sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=sampleflux --cov-report=xml:coverage.xml --cov-report=term" + sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=recordstream --cov-report=xml:coverage.xml --cov-report=term" } else { echo "No tests found in 'tests' directory. Skipping." } @@ -232,8 +232,8 @@ with open('isort-checkstyle.xml', 'w') as f: } if (fileExists('coverage.xml')) { recordCoverage( - id: 'coverage-sampleflux', - name: 'Sampleflux Coverage', + id: 'coverage-recordstream', + name: 'Recordstream Coverage', tools: [[parser: 'COBERTURA', pattern: 'coverage.xml']] ) } @@ -308,13 +308,13 @@ with open('isort-checkstyle.xml', 'w') as f: post { always { - echo 'Sampleflux Pipeline Complete.' + echo 'Recordstream Pipeline Complete.' } success { - echo 'Sampleflux is healthy.' + echo 'Recordstream is healthy.' } failure { - echo 'Sampleflux build failed. Please check linting or test failures.' + echo 'Recordstream build failed. Please check linting or test failures.' } } } diff --git a/README.md b/README.md index e141b7f..6b9eae9 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ -# SampleFlux +# RecordStream -**SampleFlux** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. +**RecordStream** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. -Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `SampleFlux`. +Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `RecordStream`. ## 🚀 Key Features -- **A sample is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. +- **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. - **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. - **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. -- **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-record `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Flux` engine. +- **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-record `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Stream` engine. - **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. -- **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-samplefluxstoragequery) — filter stored datasets without loading a single array. +- **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-recordstreamstoragequery) — filter stored datasets without loading a single array. - **Passive Introspection:** ops declare the value types they [handle / consume / produce](docs/record-model.md) and are discoverable by category for visual editors and schema generators. - **100% Reproducibility:** Entire pipelines are serializable via **Confluid** manifests. @@ -22,7 +22,7 @@ One pipeline mixing a **bare albumentations Compose** (image + mask + boxes move ```python import albumentations as A import numpy as np -from sampleflux import Flux, Image, Label, Mask, as_transform +from recordstream import Stream, Image, Label, Mask, as_transform records = [ { @@ -36,7 +36,7 @@ records = [ for rng in (np.random.default_rng(i) for i in range(100)) ] -flux = Flux( +stream = Stream( source=records, ops=[ A.Compose( # bare albumentations — as-is @@ -48,7 +48,7 @@ flux = Flux( ], ).parallel(workers=4) -for record in flux: +for record in stream: print(record["image"].shape, record["class"].value) # image+mask+boxes flipped together ``` @@ -60,7 +60,7 @@ ops: p: 0.5 - !class:albumentations.GaussNoise p: 1.0 - - !class:sampleflux.ops.numpy.Threshold + - !class:recordstream.ops.numpy.Threshold low_level: 0.5 ``` @@ -70,37 +70,37 @@ ops: |---|---| | [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | | [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`), 1→N expanding ops | -| [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Flux.from_ops_yaml` | +| [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Stream.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | | [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap` | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | -| [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `sampleflux run`), the `@entrypoint` task/role markers with a worked example, `TorchRunner` / `ProgressReporting` | +| [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers with a worked example, `TorchRunner` / `ProgressReporting` | | [docs/workflow.md](docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | | [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | ## 🧭 Scope: a modality-neutral engine -SampleFlux deliberately contains **no domain-specific code** — every op, source and sink in this package is meaningful for any modality (arrays, tensors, images, generic metadata). Domain packages build on it and keep their own vocabulary: +RecordStream deliberately contains **no domain-specific code** — every op, source and sink in this package is meaningful for any modality (arrays, tensors, images, generic metadata). Domain packages build on it and keep their own vocabulary: - Signal/waveform items and ops (spectrograms, FFT windows, recording formats) live in the domain package, which registers its item types into the same registries. - Task-specific trainers, collates and models live in their consuming projects. ## 🌐 Ecosystem Integration -SampleFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: +RecordStream is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: - **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability (see [docs/sources.md](docs/sources.md)). - **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node — including bare library transforms — every run reproducible. -- **PyTorch**: `Flux` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#batching--collate_records--the-collate-registry-samplefluxcollate) (`collate_records` is the default). +- **PyTorch**: `Stream` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#batching--collate_records--the-collate-registry-recordstreamcollate) (`collate_records` is the default). - **Augmentation libraries**: [albumentations](https://albumentations.ai) and torchvision `transforms.v2` transforms run **as-is** in any ops list — the engine speaks each library's native convention (kwarg vocabulary vs dict walk), so there is nothing to wrap (see [docs/augmentation.md](docs/augmentation.md)). ## 🔧 Installation ```bash -pip install git+https://github.com/Gearlux/sampleflux.git@main +pip install git+https://github.com/Gearlux/recordstream.git@main ``` ## 📄 License diff --git a/TASKS.md b/TASKS.md index 1f3404c..6518ae5 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1,21 +1,21 @@ -# sampleflux — backlog +# recordstream — backlog Open work for this project. Cross-cutting / multi-project initiatives live in the workspace root `TASKS.md`. Completed items are not archived here — git history is the record. -- [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of sampleflux verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor -- [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `sampleflux.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in sampleflux; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low -- [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (sampleflux, waivefront, marainer, …) were not. @low -- [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in sampleflux, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor +- [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor +- [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `recordstream.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in recordstream; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low +- [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, marainer, …) were not. @low +- [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in recordstream, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor - [ ] GPU-aware batch processing engine @performance - [ ] S3 storage backend support @feature -- [ ] **SampleFlux Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance +- [ ] **RecordStream Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance - [ ] **Typed-bag PoC → torch-`Tensor`-subclass item base:** array-backed items are `np.ndarray` subclasses only; add a `TensorItem` base (torch `__torch_function__` attr-preservation) so torch payloads can be array items instead of riding wrapper `Signal`s. @low @ml -- [ ] **Typed-bag PoC → confluid-native item-type discovery:** item types register in a local `register_item` registry (an `np.ndarray` subclass fights confluid's `__init__` validation wrap); make item types `@configurable(category="itemtype")` + entry-pointed so navigaitor/FluxStudio enumerate them as socket types. @low @tooling -- [ ] **Record model → inverse-transform path:** design an inverse/back-projection hook on `sampleflux.transform.Transform` (the old `Transform.decode` stub was dropped in the record conversion) for visualization / inference back-projection (e.g. Spectrogram→Signal, region px→signal coords). @low @ml +- [ ] **Typed-bag PoC → confluid-native item-type discovery:** item types register in a local `register_item` registry (an `np.ndarray` subclass fights confluid's `__init__` validation wrap); make item types `@configurable(category="itemtype")` + entry-pointed so navigaitor/StreamStudio enumerate them as socket types. @low @tooling +- [ ] **Record model → inverse-transform path:** design an inverse/back-projection hook on `recordstream.transform.Transform` (the old `Transform.decode` stub was dropped in the record conversion) for visualization / inference back-projection (e.g. Spectrogram→Signal, region px→signal coords). @low @ml - [ ] **S3 storage backend:** extend the `DataSource`/`DataSink` protocols (HDF5/Zarr/Directory exist) with cloud-native S3 support. @feature - [ ] **Stratified splits** — per-class balanced train/val/test on `DatasetSplit` so small classes don't fall entirely into one partition. @feature - [ ] **K-fold cross-validation** — a `KFoldSplit` companion yielding fold views; a single config loops over folds. @feature -- [ ] **Grouped splits** — honour a `group_by` metadata key (patient id, source file) so samples from one group never leak across train/val. @feature +- [ ] **Grouped splits** — honour a `group_by` metadata key (patient id, source file) so records from one group never leak across train/val. @feature - [ ] **Pre-computed split manifests** — export the train/val index lists + seed as Confluid artifacts for reproducibility and dataset cards. @low @feature - [ ] **Auto-split when `val_set` is missing** — implicit fraction split if only `train_set` is wired; deliberately deferred in favour of explicit YAML, revisit for ergonomics. @low diff --git a/docs/architecture.md b/docs/architecture.md index a9e27b1..5c1f635 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ -# SampleFlux architecture +# RecordStream architecture -The *why* behind sampleflux's module boundaries and mechanisms. The user-facing documentation +The *why* behind recordstream's module boundaries and mechanisms. The user-facing documentation ([README](../README.md), the per-topic `docs/*.md`) shows **how to use** each surface; this document records **why the surface is shaped the way it is** — so a reader who asks "why does this module exist?" finds the answer here instead of reverse-engineering it from git history. @@ -17,15 +17,15 @@ Maintenance rules: | Layer | Modules | What it is | Where the *why* lives | |---|---|---|---| -| Data model | `items.py`, `io.py` | A sample is a plain `dict` of typed values; one codec serializes any value | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Data model | `items.py`, `io.py` | A record is a plain `dict` of typed values; one codec serializes any value | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | | Native ops | `transform.py`, `dispatch.py`, `ops/*` | Type-dispatched `Transform`s (kernels, `field=`) + structural/compose/context ops | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | | Library interop | `core._apply_op`, `register_op_family` | External libraries run as-is via the op-family dispatch — no adapters | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | -| Engines | `core.py` (`Flux`/`JointFlux`), `flow.py` (`FlowGraph`) | One op-application chokepoint, four routes; a named-step graph engine with pinned lowering parity | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17), [§5](#5-the-engines-own-callable-wrappers-live-in-corepy-2026-07-20) | -| Graph wiring | `context.py`, `ops/context.py` | Fan-out/fan-in/cross-branch values on the plain sequential engine | [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17) | -| Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17) | +| Engines | `core.py` (`Stream`/`JointStream`), `flow.py` (`FlowGraph`) | One op-application chokepoint, four routes; a named-step graph engine with pinned lowering parity | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17), [§5](#5-the-engines-own-callable-wrappers-live-in-corepy-2026-07-20) | +| Graph wiring | `context.py`, `ops/context.py` | Fan-out/fan-in/cross-branch values on the plain sequential engine | [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17) | +| Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17) | | Storage & query | `storage/*` | The `typedrecord-v1` key-group layout over the codec; metadata scans without array loads | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) (contracts) + [storage.md](storage.md) | -| Introspection & serialization | `discovery.py` | Callable↔string identity + registration-free module scans | [§4](#4-callablestring-serialization--passive-introspection-samplefluxdiscovery-2026-07-20) | -| Runnables & workflows | `runnable.py`, `workflow.py`, `processing.py`, `cli.py` | `run()` objects, entry-point markers, combinators, the one `sampleflux run` runner | no record yet — [runnable.md](runnable.md), [workflow.md](workflow.md) | +| Introspection & serialization | `discovery.py` | Callable↔string identity + registration-free module scans | [§4](#4-callablestring-serialization--passive-introspection-recordstreamdiscovery-2026-07-20) | +| Runnables & workflows | `runnable.py`, `workflow.py`, `processing.py`, `cli.py` | `run()` objects, entry-point markers, combinators, the one `recordstream run` runner | no record yet — [runnable.md](runnable.md), [workflow.md](workflow.md) | --- @@ -33,9 +33,9 @@ Maintenance rules: ### Context -The rejected alternative was a bespoke sample container: typed items (that part was right) -wrapped in a `Sample` class with per-key role tags, plus an adapter registry that wrapped every -external library transform in an adapter object before it could touch a sample (two adapter +The rejected alternative was a bespoke record container: typed items (that part was right) +wrapped in a `Record` class with per-key role tags, plus an adapter registry that wrapped every +external library transform in an adapter object before it could touch a record (two adapter classes, a coercion registry, and ~170 generated per-transform wrapper ops — all maintenance surface). The container was the friction point: role tags duplicated what key names already say (`"mask"` *is* the mask), and dict-native libraries — torchvision `transforms.v2` walks dicts, @@ -47,7 +47,7 @@ families), "where does augmentation come from?" had three answers. Collapse to ONE carrier and ONE op engine: -- **A sample is a plain `dict`** — `sampleflux.items.Record = Dict[str, Any]` — of **typed +- **A record is a plain `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **typed values** (`Image`/`Mask`/`Regions`/`Label`, base `NDArrayItem`; open registry `register_item`; uniform payload accessors `item_data`/`with_data`). No container class, no roles, no `primary()`: **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"class"`), and a @@ -55,16 +55,16 @@ Collapse to ONE carrier and ONE op engine: `Label.classes`) or more dict keys (`"samplerate": 30.72e6`). Items are deliberately NOT confluid-`@configurable`: an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap — they live in their own registry. -- **Native ops are type-dispatched `Transform`s** (`sampleflux/transform.py`): +- **Native ops are type-dispatched `Transform`s** (`recordstream/transform.py`): `get_params(record)` draws shared parameters ONCE per record, per-type kernels - (`@MyOp.kernel(ItemType)`, MRO-aware registry in `sampleflux/dispatch.py`) apply to every + (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream/dispatch.py`) apply to every handled value, `field=` pins one key. The second sanctioned shape — type-CHANGING ops (`Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, `ConnectedComponents`: `Mask`→`Regions`, the target ops) — overrides `__call__`, resolves its source by an explicit `field=` or the first value of the natural type, and raises a `ValueError` naming the record's keys on every miss. - **External libraries run AS-IS through the engine's op-family dispatch** - (`sampleflux.core._apply_op`): an albumentations op receives exactly its own kwarg vocabulary + (`recordstream.core._apply_op`): an albumentations op receives exactly its own kwarg vocabulary (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record; one call = one joint draw; array outputs re-wrapped in the incoming `NDArrayItem` type so `Image`/`Mask` survive); a torchvision-v2 op is called on the dict as-is; everything else is `op(record)` with @@ -74,7 +74,7 @@ Collapse to ONE carrier and ONE op engine: detection is by MRO module name — no eager imports, no adapters, no generated wrappers. Box-carrying augmentation is the library's own `A.Compose(..., bbox_params=...)`; seeding is the libraries' own mechanisms. -- **`Pipeline(transforms=[...])`** (`sampleflux/transform.py`) is THE sequential composer; every +- **`Pipeline(transforms=[...])`** (`recordstream/transform.py`) is THE sequential composer; every composing op routes inner ops through `_apply_op`, so bare library transforms nest anywhere a native op does. - **Tensors are plain values.** `ToTensor` writes a LIVE CHW-float `torch.Tensor` under its key — @@ -83,7 +83,7 @@ Collapse to ONE carrier and ONE op engine: a tensor (`NDArrayItem.__new__` runs `np.asarray`); a typed tensor ITEM base is a tracked follow-up (root `TASKS.md`). - **Storage is the record key-group layout** (`typedrecord-v1`): everything serializes through - the `sampleflux/io.py` codec; plain values ride the `"plain"` tag; NO backward compatibility + the `recordstream/io.py` codec; plain values ride the `"plain"` tag; NO backward compatibility with the pre-record layout (an old/untagged store raises via `storage/base.py::require_record_format` — an explicit decision: re-generate, never accrete legacy readers). @@ -111,13 +111,13 @@ Collapse to ONE carrier and ONE op engine: ### Example -One `Flux` ops list mixing both worlds, no wrappers: +One `Stream` ops list mixing both worlds, no wrappers: ```python import albumentations as A -from sampleflux import Flux, Image, as_transform +from recordstream import Stream, Image, as_transform -flux = Flux(source=records, ops=[ +stream = Stream(source=records, ops=[ A.Compose([A.HorizontalFlip(p=0.5)], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"])), A.GaussNoise(p=1.0), # bare library op — as-is @@ -131,7 +131,7 @@ The same shape in YAML: ops: - !class:albumentations.HorizontalFlip p: 0.5 - - !class:sampleflux.ops.numpy.Threshold + - !class:recordstream.ops.numpy.Threshold low_level: 0.5 ``` @@ -151,13 +151,13 @@ ops: --- -## 2. Batching is two-stage; collation is a pluggable registry (`sampleflux.collate`, 2026-07-17) +## 2. Batching is two-stage; collation is a pluggable registry (`recordstream.collate`, 2026-07-17) ### Context Turning N pipeline items into one batched carrier has two distinct halves: -1. **Grouping** — the engine yields groups of N items (`Flux.batch` / `FlowGraph.batch` yield +1. **Grouping** — the engine yields groups of N items (`Stream.batch` / `FlowGraph.batch` yield `list`s, and a torch `DataLoader` hands its `collate_fn` a list). 2. **Stacking** — a *collate function* turns one group into one batched carrier. @@ -167,10 +167,10 @@ and divergent batched-metadata conventions emerged between them. ### Decision -`sampleflux/collate.py` is a **pluggable registry of collate functions keyed by representation**: +`recordstream/collate.py` is a **pluggable registry of collate functions keyed by representation**: `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`, where an omitted key uses the default **`"record"`** collate (`collate_records`) — N plain record dicts into ONE -batched record: per key, typed values encode through the `sampleflux/io.py` codec, payloads stack +batched record: per key, typed values encode through the `recordstream/io.py` codec, payloads stack (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into one batched item of the same type), and a `"plain"`-tagged value batches as the plain list. Batches must be key-homogeneous — a mismatch @@ -188,7 +188,7 @@ directly remains the normal path. ### Consequences -- The engine stays task-agnostic: sampleflux stacks by key + item type, never +- The engine stays task-agnostic: recordstream stacks by key + item type, never classification/detection/…. - Item metadata batches deterministically: per-record attrs become lists on the ONE batched item (`batch["image"].layout == ["HWC", "HWC", ...]`), plain values become plain lists — there is @@ -204,15 +204,15 @@ directly remains the normal path. ```python from torch.utils.data import DataLoader -from sampleflux import Flux, collate, collate_records, get_collate, register_collate +from recordstream import Stream, collate, collate_records, get_collate, register_collate -flux = Flux(source=my_source, ops=[...]) +stream = Stream(source=my_source, ops=[...]) -batch = collate([flux[0], flux[1]]) # the "record" default +batch = collate([stream[0], stream[1]]) # the "record" default batch["image"].shape # stacked payloads, one batched Image batch["image"].layout # per-record attrs -> a list -loader = DataLoader(flux, batch_size=8, collate_fn=collate_records) +loader = DataLoader(stream, batch_size=8, collate_fn=collate_records) # A task alias registers additively (runs when the defining module is imported). @@ -221,7 +221,7 @@ def yolo_collate(items): ... # stack to the task's own batch layout -loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("yolo")) +loader = DataLoader(stream, batch_size=8, collate_fn=get_collate("yolo")) ``` ### What you may change (and where it's documented) @@ -231,18 +231,18 @@ loader = DataLoader(flux, batch_size=8, collate_fn=get_collate("yolo")) [kinds.md](kinds.md); the detection walkthrough: [record-model.md](record-model.md). - **Changing the default collate's semantics** (how `"record"` stacks, the attrs-become-lists convention) is an architectural change: every batch consumer depends on it. Update this record - and the sampleflux `AGENTS.md` metadata mandate together. + and the recordstream `AGENTS.md` metadata mandate together. --- -## 3. The per-record Context is an ambient wiring plane (`sampleflux.context`, 2026-07-17) +## 3. The per-record Context is an ambient wiring plane (`recordstream.context`, 2026-07-17) ### Context Graph-shaped pipelines — fan-out, fan-in, cross-branch values — need somewhere to hold a value between the op that produces it and the op that consumes it. The obvious candidate, extra keys on the record itself, was rejected: the record is the carrier that **persists** — it flows into -sinks, crosses process boundaries, and is the sample's serialized identity — while wiring data is +sinks, crosses process boundaries, and is the record's serialized identity — while wiring data is transient scaffolding that should be gone by the end of a well-formed graph. Three constraints shaped the mechanism: ops keep the plain `__call__(record)` signature (no threading a context parameter through every op), the executor stays a bare `for op in ops` loop (graphs run on the @@ -250,10 +250,10 @@ parameter through every op), the executor stays a bare `for op in ops` loop (gra ### Decision -`sampleflux/context.py` is a **per-record named-cell store activated ambiently**: the engine +`recordstream/context.py` is a **per-record named-cell store activated ambiently**: the engine creates one fresh `Context` per source item and activates it around the op loop via a `contextvars.ContextVar`; the six wiring ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields` -in `sampleflux.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature +in `recordstream.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature change anywhere. Deliberate semantics: cells are stored **by reference** and copy-on-read is the *reading* op's decision (`Use` deep-copies unless `drop` frees the cell = move); a missing cell on read or delete **raises loudly** with the live-cell list (a liveness bug must never pass @@ -284,11 +284,11 @@ directly, held to the context-op semantics by the pinned flow⇄ops execution-pa ### Example ```python -from sampleflux import Flux -from sampleflux.ops.context import MergeFields, Save +from recordstream import Stream +from recordstream.ops.context import MergeFields, Save # Fan-out/fan-in on the PLAIN sequential engine: snapshot → mutate the stream → merge back. -flux = Flux( +stream = Stream( source=my_source, ops=[ Save(name="clean"), # snapshot into a cell @@ -298,7 +298,7 @@ flux = Flux( ) # The same op list outside an engine needs the Context an engine would have created: -from sampleflux.context import Context, activate, require +from recordstream.context import Context, activate, require with activate(Context()): for op in ops: @@ -318,12 +318,12 @@ with activate(Context()): the record itself, not in a cell. - **Changing cell semantics** (by-reference storage, loud missing-cell errors, the `Parallel` boundary rule, `copy()` shallowness) is an architectural change: the flow⇄ops parity suite and - the pinned context invariants define the contract. Update this record and the sampleflux + the pinned context invariants define the contract. Update this record and the recordstream `AGENTS.md` context mandate together. --- -## 4. Callable↔string serialization + passive introspection (`sampleflux.discovery`, 2026-07-20) +## 4. Callable↔string serialization + passive introspection (`recordstream.discovery`, 2026-07-20) ### Context @@ -339,7 +339,7 @@ callable out of a plain `.py` script or `__main__`, or walk a module to introspe ### Decision -`sampleflux/discovery.py` is one small stdlib-only module with **two halves**: +`recordstream/discovery.py` is one small stdlib-only module with **two halves**: - **Serialization** — `get_callable_path(fn)` → an importable `"module:qualname"` string (resolving `__main__` to the script filename so the path survives process boundaries) and @@ -377,13 +377,13 @@ name/category*. ```python import numpy as np -from sampleflux.discovery import get_callable_path, resolve_callable, scan_module +from recordstream.discovery import get_callable_path, resolve_callable, scan_module path = get_callable_path(np.sqrt) # "numpy:sqrt" — YAML/pickle-safe identity fn = resolve_callable(path) # back to the live callable fn is resolve_callable(fn) # an already-callable argument passes through -schemas = scan_module("sampleflux.ops.numpy") # one JSON schema per op defined there +schemas = scan_module("recordstream.ops.numpy") # one JSON schema per op defined there ``` ### What you may change (and where it's documented) @@ -400,20 +400,20 @@ schemas = scan_module("sampleflux.ops.numpy") # one JSON schema per op defined ### Context -Three classes sit in `core.py` next to the `Flux` engine that look, at first glance, like they -belong elsewhere: `FilterOp` and `WrappedOp` (op-shaped, so why not `ops/`?) and `JointFlux` +Three classes sit in `core.py` next to the `Stream` engine that look, at first glance, like they +belong elsewhere: `FilterOp` and `WrappedOp` (op-shaped, so why not `ops/`?) and `JointStream` (a second engine in the engine module). ### Decision They stay in `core.py` because of **who constructs them and which way imports flow**. All three -are the construction targets of `Flux`'s own fluent API — `.filter(pred)` appends a `FilterOp`, -`.map(fn)` appends a `WrappedOp`, `Flux.joint([...])` wraps a `JointFlux` — so the engine itself +are the construction targets of `Stream`'s own fluent API — `.filter(pred)` appends a `FilterOp`, +`.map(fn)` appends a `WrappedOp`, `Stream.joint([...])` wraps a `JointStream` — so the engine itself instantiates them. And `core.py` is the *bottom* of the op-facing layer: every composing op in `ops/` imports `core._apply_op` (the op-family dispatch chokepoint); moving `FilterOp`/`WrappedOp` -into `ops/` would make `core` import from `ops` and close an import cycle. `JointFlux` is -`Flux`'s iteration-only fan-in sibling (`category="engine"`), 20 lines that exist to be -`Flux.joint`'s return value — a module of its own would be structure for structure's sake +into `ops/` would make `core` import from `ops` and close an import cycle. `JointStream` is +`Stream`'s iteration-only fan-in sibling (`category="engine"`), 20 lines that exist to be +`Stream.joint`'s return value — a module of its own would be structure for structure's sake (`FlowGraph` earns its separate module by size and its own document grammar). `FilterOp`/`WrappedOp` carry **no discovery category** on purpose: they wrap a *raw Python @@ -426,19 +426,19 @@ off visual canvases. - `ops/` stays a pure consumer of `core` — the layering is one-directional. - `WrappedOp` is a package-root export (the public "lift a plain function" surface, and its stored-string `f` is the reference use of the discovery serialization half); `FilterOp` is not - root-exported (normally reached via `Flux.filter`; importable as `sampleflux.core.FilterOp`). -- `JointFlux` is YAML-addressable (`!class:sampleflux.core.JointFlux()`) and canvas-composable as + root-exported (normally reached via `Stream.filter`; importable as `recordstream.core.FilterOp`). +- `JointStream` is YAML-addressable (`!class:recordstream.core.JointStream()`) and canvas-composable as an engine node; its indexable counterpart for raw sources is `ConcatSource`. ### Example ```python -flux = ( - Flux(source=src) +stream = ( + Stream(source=src) .map(np.sqrt, key="image") # appends WrappedOp(f="numpy:sqrt", key="image") .filter(lambda r: float(r["image"].max()) > 0) # appends FilterOp(p=...) ) -both = Flux.joint([flux_a, flux_b]) # Flux(source=JointFlux([flux_a, flux_b])) +both = Stream.joint([stream_a, stream_b]) # Stream(source=JointStream([stream_a, stream_b])) ``` ### What you may change (and where it's documented) diff --git a/docs/augmentation.md b/docs/augmentation.md index 2f2ccb0..be1fc2f 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -1,27 +1,27 @@ # Augmentation — well-known libraries run AS-IS -SampleFlux does not reimplement augmentations, and it does not wrap them either. A bare +RecordStream does not reimplement augmentations, and it does not wrap them either. A bare [albumentations](https://albumentations.ai) transform or a bare torchvision `transforms.v2` -transform drops **as-is** into any ops list — `Flux(ops=[...])`, a `Pipeline`, a `flow:` step, +transform drops **as-is** into any ops list — `Stream(ops=[...])`, a `Pipeline`, a `flow:` step, inside `RandomApply` / `Enable` — and the engine's op-family dispatch -(`sampleflux.core._apply_op`) invokes it the way its own library expects. There are no adapter +(`recordstream.core._apply_op`) invokes it the way its own library expects. There are no adapter classes and no generated per-transform op families. ```python import albumentations as A from torchvision.transforms import v2 -from sampleflux import Flux, Pipeline +from recordstream import Stream, Pipeline -flux = Flux(source=records, ops=[ +stream = Stream(source=records, ops=[ A.HorizontalFlip(p=0.5), # bare albumentations A.GaussNoise(p=1.0), # bare albumentations - my_native_op, # native sampleflux op — same list + my_native_op, # native recordstream op — same list ]) Pipeline([v2.ToImage(), v2.RandomCrop(8)])(record) # bare torchvision v2 ``` -Torchvision is optional (`pip install "sampleflux[vision]"`); albumentations is a core +Torchvision is optional (`pip install "recordstream[vision]"`); albumentations is a core dependency. The family check is by MRO module name — neither library is imported until you actually put one of its transforms in a pipeline. @@ -44,11 +44,11 @@ actually put one of its transforms in a pipeline. Key names carry meaning: albumentations sees only its own vocabulary, so a value augments only if it rides one of those keys. If your pipeline produced the value under another name, route it with -`RenameField` (`sampleflux.ops.structure`) before the library op: +`RenameField` (`recordstream.ops.structure`) before the library op: ```yaml ops: - - !class:sampleflux.ops.structure.RenameField {src: spec_view, dst: image} + - !class:recordstream.ops.structure.RenameField {src: spec_view, dst: image} - !class:albumentations.GaussNoise p: 1.0 ``` @@ -77,7 +77,7 @@ engine adds nothing on top. The detection-target ops (`CocoToTorchVisionDetectio ## YAML — bare library transforms are ordinary `!class:` nodes No library-specific serialization format — a transform is a Confluid `!class:` node like any op, -in mapping form or call form. `Flux` flows deferred markers at route entry, and composing ops +in mapping form or call form. `Stream` flows deferred markers at route entry, and composing ops (`Pipeline` / `Enable` / `RandomApply`) flow theirs lazily: ```yaml @@ -85,7 +85,7 @@ ops: - !class:albumentations.HorizontalFlip p: 0.5 - !class:albumentations.GaussNoise {p: 1.0} - - !class:sampleflux.ops.numpy.Threshold + - !class:recordstream.ops.numpy.Threshold low_level: 0.5 ``` diff --git a/docs/configure.md b/docs/configure.md index 3f27d96..d31da7d 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -3,7 +3,7 @@ Some op parameters are only known *per record*. Two mechanisms cover this: - **`ConfigureOp(ops, target, param, source)`** — runs the `ops` compute-chain on the record as a SIDE branch (its transformations are discarded — the original record continues); the `source`-keyed entry of the chain's final record becomes the VALUE (payload-unwrapped via `item_data`), which is set as the `param` attribute of `target` — post-construction configuration, the confluid paradigm — and then `target` is applied to the original record. Use it when the value is *derived from the record itself* (e.g. a threshold from the record's own max) — the whole derivation reads as one node/YAML block. -- **`Capture` + `Apply`** (`sampleflux.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. +- **`Capture` + `Apply`** (`recordstream.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. Concretely — a producer that draws a random gain per record and publishes what it ACTUALLY drew as a confluid `@output` (apply `@output` UNDER `@property`), and a consumer whose `level` gets set @@ -14,7 +14,7 @@ original image, which proves the LIVE draw — not a recomputation — reached t ```python # mypackage/ops.py from confluid import configurable, output -from sampleflux import Record, item_data, with_data +from recordstream import Record, item_data, with_data import numpy as np @configurable(category="op", random=True) @@ -60,12 +60,12 @@ class CompensateOp: ```yaml ops: # AugmentOp draws a random gain each call; capture the LIVE @output into a cell. - - !class:sampleflux.ops.context.Capture + - !class:recordstream.ops.context.Capture op: !class:mypackage.ops.AugmentOp {} # or the registered short name: !class:AugmentOp {} output: applied_level name: __captured_level # …then inject the captured value into the consumer's parameter, per record. - - !class:sampleflux.ops.context.Apply + - !class:recordstream.ops.context.Apply op: !class:mypackage.ops.CompensateOp {} param: level source: __captured_level @@ -81,11 +81,11 @@ A self-contained `ConfigureOp` example — derive a per-record threshold from th ```yaml ops: - - !class:sampleflux.ops.configure.ConfigureOp + - !class:recordstream.ops.configure.ConfigureOp ops: - - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} + - !class:recordstream.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} source: image - target: !class:sampleflux.ops.numpy.Threshold + target: !class:recordstream.ops.numpy.Threshold low_op: ">=" param: low_level ``` diff --git a/docs/graph.md b/docs/graph.md index a1655c6..fa27050 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -1,16 +1,16 @@ # Graph pipelines — flow documents, the FlowGraph engine and Context ops -## Flow documents & the FlowGraph engine (`sampleflux.flow`) +## Flow documents & the FlowGraph engine (`recordstream.flow`) The **readable authoring form** of a graph pipeline is a `flow:` document — named steps where a step's name is how later steps reference its result: ```yaml flow: spec: !class:mypkg.MakeSpectrogram {} # input: the source record (writes key `image`) - masked: !class:sampleflux.ops.numpy.Threshold {low_level: 0.5, from: spec} # 2nd reader of `spec` = fan-out - thresh: !class:sampleflux.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} + masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} # 2nd reader of `spec` = fan-out + thresh: !class:recordstream.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} gated: # a step with bind: uses the plain-mapping form (op: + reserved keys) - op: !class:sampleflux.ops.numpy.Threshold {output: gated_mask} + op: !class:recordstream.ops.numpy.Threshold {output: gated_mask} from: spec bind: low_level: thresh[image] # per-record param := the `image` entry of thresh's result @@ -36,20 +36,20 @@ A plain-mapping step with no op (`out: {from: a, merge_from: [b]}`) is a pure fa Two engines, one contract — **bidirectional conversion with execution parity**: ```python -from sampleflux import Flux, FlowGraph, to_ops, from_ops +from recordstream import Stream, FlowGraph, to_ops, from_ops graph = FlowGraph.from_yaml("graph.yaml", source=src) # native named-step engine -flux = Flux.from_flow_yaml("graph.yaml", source=src) # same graph, LOWERED to the +stream = Stream.from_flow_yaml("graph.yaml", source=src) # same graph, LOWERED to the # flat context-ops list (serial) ops = to_ops(graph.steps, graph.output_step) # flow -> flat ops flow2 = from_ops(ops) # flat ops -> flow (lifting) ``` -`FlowGraph` is a `torch.utils.data.Dataset` like `Flux` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Flux's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. +`FlowGraph` is a `torch.utils.data.Dataset` like `Stream` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Stream's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. ## Graph pipelines on a flat op list (Context ops) -A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Flux` engine** via six *context ops* (`sampleflux.ops.context`). The engine creates one per-record **`Context`** (a named-cell store, `sampleflux.context`) around each record's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never mutates the record's entries — a linear run's record stays byte-identical whether or not context threading exists. +A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Stream` engine** via six *context ops* (`recordstream.ops.context`). The engine creates one per-record **`Context`** (a named-cell store, `recordstream.context`) around each record's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never mutates the record's entries — a linear run's record stays byte-identical whether or not context threading exists. | Op | Semantics | |---|---| @@ -62,13 +62,13 @@ A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a ```yaml ops: - - !class:sampleflux.ops.context.Save(name=fork) # fork the stream + - !class:recordstream.ops.context.Save(name=fork) # fork the stream - !class:albumentations.GaussNoise {p: 1.0} # branch A rides the stream - - !class:sampleflux.ops.context.Save(name=branch_a) - - !class:sampleflux.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork - - !class:sampleflux.ops.numpy.Threshold + - !class:recordstream.ops.context.Save(name=branch_a) + - !class:recordstream.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork + - !class:recordstream.ops.numpy.Threshold low_level: 0.5 - - !class:sampleflux.ops.context.MergeFields # fan-in + - !class:recordstream.ops.context.MergeFields # fan-in sources: [branch_a] keys: [image] drop: [branch_a] @@ -77,26 +77,26 @@ ops: A straight sequence needs none of this — a bare `ops:` list stays exactly as before. Outside an engine (a hand-rolled loop), activate a Context explicitly: ```python -from sampleflux.context import Context, activate +from recordstream.context import Context, activate with activate(Context()): for op in ops: record = op(record) ``` -Cells hold whole records (from `Save`) or raw values (from `Capture`); `Apply` reads a record cell's `key`-named entry (whole record when `key` is blank), `MergeFields` unions each cell's entries. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). On a deliberate key collision at the fan-in, rename on the producing branch first (`RenameField`, `sampleflux.ops.structure`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-record store instead of extra record keys (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#3-the-per-record-context-is-an-ambient-wiring-plane-samplefluxcontext-2026-07-17). +Cells hold whole records (from `Save`) or raw values (from `Capture`); `Apply` reads a record cell's `key`-named entry (whole record when `key` is blank), `MergeFields` unions each cell's entries. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). On a deliberate key collision at the fan-in, rename on the producing branch first (`RenameField`, `recordstream.ops.structure`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-record store instead of extra record keys (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17). -> **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the value under its own key with `CopyField` (`sampleflux.ops.structure`); the snapshot then rides the record as a real entry. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. +> **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the value under its own key with `CopyField` (`recordstream.ops.structure`); the snapshot then rides the record as a real entry. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. -## Reattach an ops-only YAML (`Flux.from_ops_yaml`) +## Reattach an ops-only YAML (`Stream.from_ops_yaml`) A `{ops: [!class:…()]}` document — e.g. one exported by an external pipeline-authoring tool — can be attached to any source: ```python -from sampleflux import Flux -from sampleflux.sources import HuggingFaceSource +from recordstream import Stream +from recordstream.sources import HuggingFaceSource -flux = Flux.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) +stream = Stream.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) ``` -The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: `Flux` also flows any still-deferred marker in place at engine-route entry (the same lazy-flow convention the composing ops use), which is what lets a bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. The manual equivalent is `Flux(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. +The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: `Stream` also flows any still-deferred marker in place at engine-route entry (the same lazy-flow convention the composing ops use), which is what lets a bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. The manual equivalent is `Stream(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. diff --git a/docs/image.md b/docs/image.md index b5d1a8e..14bc484 100644 --- a/docs/image.md +++ b/docs/image.md @@ -1,9 +1,9 @@ -# Image conversion (`sampleflux.ops.image`) +# Image conversion (`recordstream.ops.image`) The single, modality-agnostic "any value → image" layer — generic so every consuming project (spectrogram previews, dataset browsers, GUI viewers) reuses one implementation. Domain-specific rendering (overlays, signal plots) stays in the consuming package. ```python -from sampleflux.ops.image import ConvertToImage, value_to_image +from recordstream.ops.image import ConvertToImage, value_to_image # Op: an array-bearing record value (2-D map / CHW tensor / PIL / bool mask) -> an Image item. op = ConvertToImage( @@ -21,13 +21,13 @@ rgb = value_to_image(some_value, colormap="magma", max_size=512) # normalize_to_uint8: the standalone min-max value -> uint8 quantization step # (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; # set them to pin a fixed scale across records (out-of-range values clamp). -from sampleflux.ops.image import normalize_to_uint8 +from recordstream.ops.image import normalize_to_uint8 u8 = normalize_to_uint8(arr) # auto per-array min/max u8 = normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # fixed dB window across a dataset ``` -`sample_to_image(record, ...)` renders a record's first array-bearing (2-D / 3-D) value the same way — the ad-hoc whole-record preview for viewer tooling. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). +`record_to_image(record, ...)` renders a record's first array-bearing (2-D / 3-D) value the same way — the ad-hoc whole-record preview for viewer tooling. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). ## Introspection helpers diff --git a/docs/kinds.md b/docs/kinds.md index 43241a3..7e589eb 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -1,11 +1,11 @@ -# Ops, batching & expanding ops (`sampleflux.transform` / `sampleflux.collate`) +# Ops, batching & expanding ops (`recordstream.transform` / `recordstream.collate`) ## What an op processes — dispatch on value type -A **sample** is a plain record dict of typed values (`Image`, `Mask`, `Regions`, `Label`, … — see [record-model.md](record-model.md)). A native op is a `Transform`: it declares which value TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per record (`get_params`), then applies the matching kernel to every value whose type it handles, passing untouched values through: +A **record** is a plain dict of typed values (`Image`, `Mask`, `Regions`, `Label`, … — see [record-model.md](record-model.md)). A native op is a `Transform`: it declares which value TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per record (`get_params`), then applies the matching kernel to every value whose type it handles, passing untouched values through: ```python -from sampleflux import Record, Transform, Image +from recordstream import Record, Transform, Image class Recenter(Transform): handles = (Image,) # which value types this op touches @@ -29,7 +29,7 @@ Bare library transforms (torchvision `transforms.v2` walking the dict natively, ```python import albumentations as A -from sampleflux import Pipeline +from recordstream import Pipeline out = Pipeline([ A.HorizontalFlip(p=1.0), # image + mask + bboxes together (one library draw) @@ -39,29 +39,29 @@ out = Pipeline([ # record["class"] (a Label) is untouched: no kernel handles it, no library key names it. ``` -## Batching — `collate_records` & the collate registry (`sampleflux.collate`) +## Batching — `collate_records` & the collate registry (`recordstream.collate`) Ops are per-record; batching is a separate stage. **`collate_records`** (the registry's `"record"` default) stacks N record dicts into ONE batched record: per key, typed payloads stack (torch → stacked tensor, numpy → stacked array, else a list) and each item's declared attrs become per-record lists, decoded back into one batched item of the same type; plain values batch as plain lists. Batches must carry the same keys — a mismatch raises. ```python -from sampleflux import collate_records +from recordstream import collate_records from torch.utils.data import DataLoader -batch = collate_records(list(flux)) # ONE batched record: payloads stacked per key -loader = DataLoader(flux, collate_fn=collate_records) +batch = collate_records(list(stream)) # ONE batched record: payloads stacked per key +loader = DataLoader(stream, collate_fn=collate_records) ``` Collation is a pluggable registry keyed by name, so a task can register its own convention additively: ```python -from sampleflux import register_collate, get_collate +from recordstream import register_collate, get_collate @register_collate("yolo") # task aliases are additive def yolo_collate(items): ... -loader = DataLoader(flux, collate_fn=get_collate("yolo")) +loader = DataLoader(stream, collate_fn=get_collate("yolo")) ``` -The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#2-batching-is-two-stage-collation-is-a-pluggable-registry-samplefluxcollate-2026-07-17). +The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17). ## 1→N expanding ops (iterable-only pipelines) @@ -70,8 +70,8 @@ An op may return **several** carriers — a windowing op splitting one capture i ```python from typing import Iterator from confluid import configurable -from sampleflux import Record -from sampleflux.items import item_data, with_data +from recordstream import Record +from recordstream.items import item_data, with_data @configurable(category="op") class SlidingWindow: @@ -85,4 +85,4 @@ class SlidingWindow: Expansion is flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. -A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(flux)` / `flux[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(flux)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Flux` engine. +A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(stream)` / `stream[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(stream)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Stream` engine. diff --git a/docs/projection.md b/docs/projection.md index 558a522..e72fd81 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -1,11 +1,11 @@ -# Key projection, class counting & label maps (`sampleflux.projection` / `sampleflux.labels`) +# Key projection, class counting & label maps (`recordstream.projection` / `recordstream.labels`) ## Key projection -Walking a source for a single record key (the classic case: counting classes from the label key) shouldn't pay to build the values you don't need. `sampleflux.projection` adds an opt-in protocol plus lazy helpers, all **key-addressed** — any subset of record keys: +Walking a source for a single record key (the classic case: counting classes from the label key) shouldn't pay to build the values you don't need. `recordstream.projection` adds an opt-in protocol plus lazy helpers, all **key-addressed** — any subset of record keys: ```python -from sampleflux import project, iter_key, num_classes +from recordstream import project, iter_key, num_classes # A source MAY implement SupportsProjection (`project(keys)`) to skip building # unrequested values — e.g. an image dataset reads only the label column for a @@ -18,21 +18,21 @@ labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, n = num_classes(my_source, key="class") # max(class_id) + 1 — always walks ``` -Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Flux.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Flux` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. +Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Stream.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Stream` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. ## `LabelMap` — fittable name↔id encoding When a dataset's label is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTarget` / `DecodeTarget` ops need — the *fittable* companion to those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: ```python -from sampleflux import LabelMap, Flux, iter_key +from recordstream import LabelMap, Stream, iter_key lm = LabelMap.fit(iter_key(train_source, "class")) # {"bird": 0, "cat": 1, "dog": 2} lm.num_classes # 3 lm.label_names # ["bird", "cat", "dog"] (id -> name) lm.save("class_names.json") # {"class_names": [...], "num_classes": N} -encoded = Flux(source=train_source, ops=[lm.encode_op()]) # "class" Labels now carry int ids +encoded = Stream(source=train_source, ops=[lm.encode_op()]) # "class" Labels now carry int ids # Later, at eval time — reload the SAME ordering instead of refitting: lm2 = LabelMap.load("class_names.json") diff --git a/docs/record-model.md b/docs/record-model.md index 740566b..c1d9fdd 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -1,7 +1,7 @@ -# The record model — THE sampleflux data model +# The record model — THE recordstream data model -A sample is a **plain `dict`** of **typed values**. Import the whole surface from the PACKAGE TOP -LEVEL (`from sampleflux import Record, Image, Mask, Regions, Label, Transform, Pipeline, +A record is a **plain `dict`** of **typed values**. Import the whole surface from the PACKAGE TOP +LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). The design rationale is recorded in [architecture.md](architecture.md#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25). @@ -13,7 +13,7 @@ region boxes, a signal's samplerate, an image's layout, a label's class names flat `metadata` dict keyed by string, it is disconnected from the value it describes. And if the carrier is a bespoke container class, every external library needs an adapter before it can touch it. -The record model fixes both. **A sample is a plain dict, values are typed, and metadata lives on the +The record model fixes both. **A record is a plain dict, values are typed, and metadata lives on the value it describes** — an `Image` carries its `layout`, a `Label` its `classes`. **Key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the same convention as every torch batch dict and albumentations' keyword vocabulary), so libraries that already understand dicts or @@ -31,18 +31,18 @@ record = { ``` There is deliberately **no container class** — `Record` is a type alias (`Dict[str, Any]` in -`sampleflux.items`), ops receive and return ordinary dicts, and `None` means "drop this record" +`recordstream.items`), ops receive and return ordinary dicts, and `None` means "drop this record" (filter semantics). ## The pieces ### Items — typed values that own their metadata -sampleflux is **modality-neutral**, so its core ships only generic items — images, masks, boxes, +recordstream is **modality-neutral**, so its core ships only generic items — images, masks, boxes, labels. (Domain items — a signal, a spectrogram — live in the domain package; see below.) ```python -from sampleflux import Image, Mask, Regions, Label +from recordstream import Image, Mask, Regions, Label Image(rgb_hwc, layout="HWC") # an image knows its layout ("HWC" default / "CHW") Mask(seg_hw) # a mask shares its image's frame @@ -57,7 +57,7 @@ wrappers (a bounding-box set is not an array). A uniform payload accessor hides kernels: ```python -from sampleflux import item_data, with_data +from recordstream import item_data, with_data item_data(Image(arr)) # -> the plain ndarray with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved ``` @@ -68,16 +68,16 @@ decorator, no core edit). ### Ops — type dispatch with once-per-record parameters -A `Transform` (`sampleflux.transform`) samples its parameters ONCE per record +A `Transform` (`recordstream.transform`) samples its parameters ONCE per record (`get_params(record)`), then applies a per-type **kernel** to every value whose type it handles -(`@MyOp.kernel(ItemType)`, resolved MRO-aware by `sampleflux.dispatch`). Values it does not handle +(`@MyOp.kernel(ItemType)`, resolved MRO-aware by `recordstream.dispatch`). Values it does not handle pass through. Because the parameters are sampled once and shared, one op moves every handled value with the SAME decision — the torchvision-v2 model. Targeting is by TYPE; the `field=` constructor parameter pins an op to one named key when a record holds several values of a handled type. ```python import numpy as np -from sampleflux import Image, Record, Transform +from recordstream import Image, Record, Transform class Brighten(Transform): handles = (Image,) @@ -226,13 +226,13 @@ Rules of use: refuse to run without an input, validate lazily in `__call__` with a clear error — the same lazy-validation convention every op follows. -**sampleflux ships no native augmentation ops** — geometric/photometric augmentation comes from +**recordstream ships no native augmentation ops** — geometric/photometric augmentation comes from torchvision `transforms.v2` / albumentations run as-is (next section); native ops exist only where no library covers them. ### Mixing libraries — as-is, no adapters -The engine's single op-application chokepoint, `sampleflux.core._apply_op(record, op)`, dispatches +The engine's single op-application chokepoint, `recordstream.core._apply_op(record, op)`, dispatches on the op's FAMILY (by MRO module name, no eager import) and invokes each family the way its own library expects: @@ -245,12 +245,12 @@ library expects: silently. - **everything else** — `op(record)`; `None` drops the record. -So bare library transforms sit in one list with native ops — in `Flux(ops=[...])`, in a `Pipeline`, +So bare library transforms sit in one list with native ops — in `Stream(ops=[...])`, in a `Pipeline`, in a `flow:` step: ```python import albumentations as A -from sampleflux import Pipeline +from recordstream import Pipeline Pipeline([ A.Compose( # box-carrying augmentation: the library's own Compose @@ -279,7 +279,7 @@ Runnable end-to-end: [`examples/record_pipeline.py`](../examples/record_pipeline ### `Pipeline` — the sequential composer -`Pipeline(transforms=[...])` (`sampleflux.transform`, `@configurable(category="op", +`Pipeline(transforms=[...])` (`recordstream.transform`, `@configurable(category="op", group="compose")`) wraps an ordered op list so it appears as one named block in a config and one node on a visual canvas: zero-arg/lazy (config-deferred markers flow on first call), entries applied through `_apply_op` (so bare library transforms nest exactly as in a bare ops list), `None` @@ -291,7 +291,7 @@ resources. ### A custom op from a plain function ```python -from sampleflux import as_transform, Image +from recordstream import as_transform, Image brighten = as_transform(lambda d: d + 0.1, handles=(Image,), field="image") ``` @@ -299,7 +299,7 @@ brighten = as_transform(lambda d: d + 0.1, handles=(Image,), field="image") ```python from dataclasses import dataclass, field -from sampleflux import register_item +from recordstream import register_item from mypkg.transforms import MyGeoTransform # any Transform subclass @register_item @@ -317,7 +317,7 @@ subclass transform inherits its base's kernels until it overrides them. ### Domain items live in the domain package -The same mechanism, applied across packages: because sampleflux is modality-neutral, a signal-domain +The same mechanism, applied across packages: because recordstream is modality-neutral, a signal-domain package defines its own items (a signal, a spectrogram) and its own type-changing ops, registers them with `register_item`, and they become first-class record values — dispatchable, collatable, storable — with no core edit. @@ -331,7 +331,7 @@ calling convention. Every engine route (sequential, spawn-parallel, streamed, ra every composing op picks it up at once, because they all funnel through `_apply_op`: ```python -from sampleflux import register_op_family +from recordstream import register_op_family def is_kornia(op) -> bool: # Keep the matcher IMPORT-FREE: inspect MRO module names, never import the library. @@ -345,8 +345,8 @@ def invoke_kornia(record, op): register_op_family("kornia", is_kornia, invoke_kornia) -# From here on, bare kornia ops sit in ANY ops list — Flux, Pipeline, RandomApply, flow steps: -flux = Flux(source=records, ops=[ToTensor(field="image"), K.RandomHorizontalFlip(p=1.0)]) +# From here on, bare kornia ops sit in ANY ops list — Stream, Pipeline, RandomApply, flow steps: +stream = Stream(source=records, ops=[ToTensor(field="image"), K.RandomHorizontalFlip(p=1.0)]) ``` The rules: dispatch checks families **last-registered first**, so a more specific family (say a @@ -360,21 +360,21 @@ privileged code path. When a library's convention needs per-op configuration ins read, per-op state), write a normal `Transform` op that wraps it explicitly — the registry is for AS-IS drop-in. -## Engines — Flux and FlowGraph carry the record +## Engines — Stream and FlowGraph carry the record -Every carrier is a plain record dict, and every route applies ops through `_apply_op` — sequential, -spawn-parallel, streamed, and random-access (`__getitem__`) alike, in `Flux` and in `FlowGraph`. +Every carrier is a plain dict, and every route applies ops through `_apply_op` — sequential, +spawn-parallel, streamed, and random-access (`__getitem__`) alike, in `Stream` and in `FlowGraph`. Composing ops (`Pipeline`, `RandomApply`, `Enable`, `Parallel`, `ConfigureOp`, the context ops `Apply`/`Capture`) route their inner ops through the same chokepoint, so a bare library transform nests anywhere a native op does. ```python -Flux(source=my_source, ops=[A.GaussNoise(p=1.0), Brighten()]).to_sink(HDF5Sink(path="out.h5")) +Stream(source=my_source, ops=[A.GaussNoise(p=1.0), Brighten()]).to_sink(HDF5Sink(path="out.h5")) ``` -`Flux.map(func, key=None)` lifts a plain function over one record entry (`key=None` hands it the +`Stream.map(func, key=None)` lifts a plain function over one record entry (`key=None` hands it the whole dict — internally a `WrappedOp`, which stores the callable as its importable path so it -pickles across `spawn` workers); `Flux.project(keys)` yields partial records restricted to the +pickles across `spawn` workers); `Stream.project(keys)` yields partial records restricted to the requested keys (see [projection.md](projection.md)). ### Graph fan-in (`merge_from`) and entry binds (`step[key]`) @@ -386,8 +386,8 @@ produce, `SelectFields` the new key(s), merge: ```yaml flow: start: {} - masked: {op: !class:sampleflux.ops.numpy.Threshold(low_level=0.5), from: start} - mask_only: {op: !class:sampleflux.ops.structure.SelectFields(keys: [mask]), from: masked} + masked: {op: !class:recordstream.ops.numpy.Threshold(low_level=0.5), from: start} + mask_only: {op: !class:recordstream.ops.structure.SelectFields(keys: [mask]), from: masked} boosted: {op: !class:mypkg.Boost(), from: start} out: {from: boosted, merge_from: [mask_only]} ``` @@ -405,10 +405,10 @@ schema: per record, one group per KEY carrying the value's registered type name the payload as a `data` dataset, and its attrs (scalars natively — queryable; arrays as sub-datasets under `attrs/`; structured values JSON-tagged so tuples survive). A plain (non-item) value rides the `"plain"` type tag — an array payload as `data`, a scalar under the `value` attr. Key order is -preserved in `__field_order__`; the store is stamped `sampleflux_format = "typedrecord-v1"`. +preserved in `__field_order__`; the store is stamped `recordstream_format = "typedrecord-v1"`. Backends never inspect item internals — everything serializes through the item codec -(`sampleflux/io.py`: `encode_item` / `decode_item` / `encode_record` / `decode_record`), so an +(`recordstream/io.py`: `encode_item` / `decode_item` / `encode_record` / `decode_record`), so an externally-registered item type round-trips with zero storage edits; `register_io(MyItem, encode=..., decode=...)` overrides the default structural codec when needed. Decoding requires the item type to be registered (imported) in the reading process — the same @@ -417,7 +417,7 @@ contract as Confluid's `!class:`. ```python sink = HDF5Sink(path="out.h5", overwrite=True) with sink: - for record in flux: + for record in stream: sink.write(record) back = list(HDF5Source(path="out.h5")) # exact records: keys, types, order, tuple attrs ``` @@ -441,11 +441,11 @@ fast = MetadataFilterSource(source=HDF5Source(path="out.h5"), where="signal.samp Array-valued attrs appear as shape/dtype stubs (presence/shape testable, never loaded). A key named like a Python keyword (e.g. `class`) can't be addressed in an expression — use the programmatic `predicate` or a non-keyword key name. Live records expose the same nested shape via -`sampleflux.storage.query.record_metadata(record)`. See [storage.md](storage.md). +`recordstream.storage.query.record_metadata(record)`. See [storage.md](storage.md). ## Batching — `collate_records` and the collate registry -A torch `DataLoader` (or `Flux.batch`) hands a collate function a LIST of N records and expects +A torch `DataLoader` (or `Stream.batch`) hands a collate function a LIST of N records and expects ONE object back. `collate_records` — the registry's `"record"` default — folds per key with three rules (all records must share the same key set; a mismatch raises): @@ -474,7 +474,7 @@ images + RAGGED per-record target dicts), so the task package registers a collat exactly that: ```python -from sampleflux import Image, Regions, collate, register_collate +from recordstream import Image, Regions, collate, register_collate @register_collate("detection") def detection_collate(items): diff --git a/docs/runnable.md b/docs/runnable.md index ca0af5e..adbcbeb 100644 --- a/docs/runnable.md +++ b/docs/runnable.md @@ -1,7 +1,7 @@ # Runnables and entry points A **runnable** is any object exposing a no-arg `run()` — a trainer, an evaluator, a dataset -processor, a workflow. It is the unit `sampleflux run` executes: +processor, a workflow. It is the unit `recordstream run` executes: ```yaml # config.yaml — the ONE runner shape for every kind of run @@ -11,7 +11,7 @@ runnable: !class:mypkg.Classifier ``` ```bash -python -m sampleflux.cli run config.yaml # builds `runnable:`, calls .run() +python -m recordstream.cli run config.yaml # builds `runnable:`, calls .run() ``` ## The problem entry points solve @@ -25,7 +25,7 @@ declares exactly that, per method. ## A straightforward example ```python -from sampleflux import ProgressReporting, TorchRunner, entrypoint +from recordstream import ProgressReporting, TorchRunner, entrypoint class Classifier(TorchRunner, ProgressReporting): """One class, four capabilities — run() dispatches off the ``task`` knob.""" @@ -61,7 +61,7 @@ secondary, validation-split variant). Real output for the class above (these are executed facts, not sketches): ```python ->>> from sampleflux import runnable_entrypoints, entrypoint_tasks +>>> from recordstream import runnable_entrypoints, entrypoint_tasks >>> runnable_entrypoints(Classifier) {'fit': {'task': 'fit', 'role': 'trainer', 'primary': True}, 'evaluate': {'task': 'evaluate', 'role': 'evaluator', 'primary': False}, @@ -83,7 +83,7 @@ wins) and reads the marker off the raw function object, so property getters neve A config generator asked for "an evaluator config for `Classifier`" calls `entrypoint_tasks(Classifier, "evaluator")[0]` → `"test"` and pins `task: test` in the YAML -it emits — one `sampleflux run` then dispatches correctly with no human editing. The same +it emits — one `recordstream run` then dispatches correctly with no human editing. The same walk over every discovered class tells a visual editor which classes to offer in a "trainer" picker versus an "evaluator" picker, even when both answers are the same class. diff --git a/docs/sources.md b/docs/sources.md index 61d4297..9d2132a 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -1,4 +1,4 @@ -# Sources — HuggingFace, splits, ranges, concatenation (`sampleflux.sources`) +# Sources — HuggingFace, splits, ranges, concatenation (`recordstream.sources`) ## Hugging Face datasets @@ -7,7 +7,7 @@ - **`metadata_features` (which extra columns become record entries):** the sentinel **`"*"`** (or `["*"]`, the default) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load; an explicit list keeps exactly those columns; `None` / `[]` keep none. ```yaml -hf_train: !class:sampleflux.sources.HuggingFaceSource() +hf_train: !class:recordstream.sources.HuggingFaceSource() path: mnist input_feature: image target_feature: label @@ -23,7 +23,7 @@ hf_train: !class:sampleflux.sources.HuggingFaceSource() **Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: ```python -from sampleflux import DatasetSplit +from recordstream import DatasetSplit split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% ``` @@ -31,27 +31,27 @@ split.train # ≈80% — the remainder split.val # ≈10% split.te The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: ```yaml -hf_train: !class:sampleflux.sources.HuggingFaceSource() +hf_train: !class:recordstream.sources.HuggingFaceSource() path: mnist split: train -my_split: !class:sampleflux.sources.DatasetSplit() +my_split: !class:recordstream.sources.DatasetSplit() source: !ref:hf_train val_fraction: 0.1 test_fraction: 0.1 seed: 42 -train_set: !class:sampleflux.core.Flux() { source: !ref:my_split.train } -val_set: !class:sampleflux.core.Flux() { source: !ref:my_split.val } -test_set: !class:sampleflux.core.Flux() { source: !ref:my_split.test } +train_set: !class:recordstream.core.Stream() { source: !ref:my_split.train } +val_set: !class:recordstream.core.Stream() { source: !ref:my_split.val } +test_set: !class:recordstream.core.Stream() { source: !ref:my_split.test } ``` Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). -**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `sampleflux.SplitName`. +**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `recordstream.SplitName`. ```yaml -val_set: !class:sampleflux.sources.DatasetSplit() +val_set: !class:recordstream.sources.DatasetSplit() source: !ref:hf_train split: val val_fraction: 0.1 @@ -63,21 +63,21 @@ val_set: !class:sampleflux.sources.DatasetSplit() - **`RangeSource(source, start, stop)`** — a contiguous index slice `[start:stop)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. ```yaml - first_half: !class:sampleflux.sources.RangeSource() + first_half: !class:recordstream.sources.RangeSource() source: !ref:hf_train start: 0 stop: 5000 ``` -- **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointFlux`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. +- **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointStream`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. ```yaml - combined: !class:sampleflux.sources.ConcatSource() + combined: !class:recordstream.sources.ConcatSource() sources: - !ref:train_main - !ref:extra_shard ``` -**HuggingFace native slicing** (alternative, no SampleFlux split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. +**HuggingFace native slicing** (alternative, no RecordStream split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. > **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. diff --git a/docs/storage.md b/docs/storage.md index 6cc0933..de6f648 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,17 +1,17 @@ -# Storage — sinks, sources and queryable metadata (`sampleflux.storage`) +# Storage — sinks, sources and queryable metadata (`recordstream.storage`) > Runnable tour: [`examples/storage_roundtrip.py`](../examples/storage_roundtrip.py) — the same > records through all three sink/source pairs (typed values + a plain scalar, byte-identical > round-trips) plus a `MetadataFilterSource` query that never loads an array. -SampleFlux makes it easy to move data between different formats: +RecordStream makes it easy to move data between different formats: ```python -from sampleflux.storage.hdf5 import HDF5Source -from sampleflux.storage.zarr import ZarrGroupSink +from recordstream.storage.hdf5 import HDF5Source +from recordstream.storage.zarr import ZarrGroupSink # Stream from HDF5 to Zarr in parallel -Flux.from_source(HDF5Source("input.h5")) \ +Stream.from_source(HDF5Source("input.h5")) \ .parallel(workers=8) \ .map(heavy_op, key="image") \ .to_sink(ZarrGroupSink("output.zarr")) @@ -29,19 +29,19 @@ Every sink has a source that reads its layout back into record dicts of typed va | Directory (one dir / record) | `DirectorySink` | `DirectorySource` | all keys + item types + attrs | ```python -from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource +from recordstream.storage.zarr import ZarrGroupSink, ZarrGroupSource -Flux(records).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) +Stream(records).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) for record in ZarrGroupSource("ds.zarr"): # exact keys, item types and attrs reconstructed ... ``` All four backends share ONE logical schema — the **record key-group layout**, stamped -`sampleflux_format = "typedrecord-v1"`: per record, one group per KEY carrying the value's +`recordstream_format = "typedrecord-v1"`: per record, one group per KEY carrying the value's registered type name, the payload as a `data` dataset, and its attrs (scalars natively; structured values JSON-tagged so tuples survive); a plain (non-item) value rides the `"plain"` type tag — an array payload as `data`, a scalar under the `value` attr. Everything serializes through the item -codec (`sampleflux/io.py`), so an externally-registered item type round-trips with zero storage +codec (`recordstream/io.py`), so an externally-registered item type round-trips with zero storage edits (see [record-model.md](record-model.md#storage--the-record-key-group-layout)). > **No backward compatibility.** A store whose format tag is missing or pre-record @@ -62,16 +62,16 @@ its own sub-dataset under `attrs/` instead — a large array never overflows the is a first-class `Mask` value under its own key with its own payload. ```python -from sampleflux import Image, Mask, item_data +from recordstream import Image, Mask, item_data record = {"image": Image(data), "mask": Mask(mask_2d)} -Flux([record]).to_sink(HDF5Sink("ds.h5", overwrite=True)) +Stream([record]).to_sink(HDF5Sink("ds.h5", overwrite=True)) loaded = next(iter(HDF5Source("ds.h5"))) item_data(loaded["mask"]) # the full mask array, byte-exact (not a truncated repr) loaded["image"].layout # item attrs round-trip too ``` -## Queryable metadata (`sampleflux.storage.query`) +## Queryable metadata (`recordstream.storage.query`) Filter stored records by metadata predicates *without loading arrays*: sources implementing the `SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs / `.zattrs` / sidecar JSON — @@ -80,11 +80,11 @@ structural protocol without importing this module), so **record-layout HDF5/Zarr queryable with no extra index**: ```python -from sampleflux.storage.query import MetadataFilterSource +from recordstream.storage.query import MetadataFilterSource view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="signal.samplerate > 1e6") len(view) # matches counted from a metadata-only scan -flux = Flux(source=view, ops=[...]) # arrays load ONLY for matching records +stream = Stream(source=view, ops=[...]) # arrays load ONLY for matching records ``` The scans yield the nested `{key: {attr: value}}` shape, and a `where` expression addresses it as diff --git a/docs/workflow.md b/docs/workflow.md index 5473b15..2e34cfc 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -1,9 +1,9 @@ # Workflows — composing runnables -The workflow combinators (`sampleflux.workflow`) are the RUNNABLE-level analogue of the +The workflow combinators (`recordstream.workflow`) are the RUNNABLE-level analogue of the composing ops: they HOLD other runnables and orchestrate them, so a multi-stage pipeline (prepare → train → evaluate) is ONE Confluid document executed by the same -`sampleflux run workflow.yaml` as any single runnable. +`recordstream run workflow.yaml` as any single runnable. | Combinator | Runs | |---|---| @@ -21,12 +21,12 @@ Re-run the SAME document after a crash (or just again tomorrow) and it skips the artifact already exists — *memoise and continue*: ```yaml -runnable: !class:sampleflux.workflow.Sequence +runnable: !class:recordstream.workflow.Sequence steps: # Train ONLY when the checkpoint is missing. On a cache hit the !lazy: branch is # not just skipped — it is never even BUILT (no model / dataset materialised). - - !class:sampleflux.workflow.Conditional - condition: !class:sampleflux.workflow.PathExists + - !class:recordstream.workflow.Conditional + condition: !class:recordstream.workflow.PathExists path: $MODEL_ROOT/run1/model.ckpt if_true: null # cache hit -> skip, Sequence continues if_false: !lazy:TrainModel # @configurable classes resolve by registered NAME @@ -34,7 +34,7 @@ runnable: !class:sampleflux.workflow.Sequence # Always runs; the report FORMAT is a Switch on a plain config value — one key a # CLI override can flip (--select text) without touching the workflow shape. - - !class:sampleflux.workflow.Switch + - !class:recordstream.workflow.Switch select: json cases: json: !lazy:Evaluate { report: $MODEL_ROOT/run1/report.json, fmt: json } diff --git a/examples/cache_pipeline.py b/examples/cache_pipeline.py index 5f5c8ed..2becf28 100644 --- a/examples/cache_pipeline.py +++ b/examples/cache_pipeline.py @@ -6,7 +6,7 @@ 3. The cache enforces an LRU budget — the oldest entry is evicted when a new one would push total usage past ``max_bytes``. -Runs end-to-end with no external data and no SampleFlux pipeline. +Runs end-to-end with no external data and no RecordStream pipeline. """ import tempfile @@ -14,11 +14,11 @@ from pathlib import Path from typing import Callable -from sampleflux.storage.cache import CacheBudgetExceeded, DiskCache +from recordstream.storage.cache import CacheBudgetExceeded, DiskCache def main() -> None: - with tempfile.TemporaryDirectory(prefix="sampleflux-cache-demo-") as tmp: + with tempfile.TemporaryDirectory(prefix="recordstream-cache-demo-") as tmp: cache = DiskCache(Path(tmp), max_bytes=300) print(f"Cache root: {cache.root} (max_bytes={cache.max_bytes})") diff --git a/examples/discovery_demo.py b/examples/discovery_demo.py index 81ea845..476cf40 100644 --- a/examples/discovery_demo.py +++ b/examples/discovery_demo.py @@ -5,11 +5,11 @@ import json -from sampleflux.discovery import scan_module +from recordstream.discovery import scan_module def main() -> None: - module = "sampleflux.ops.numpy" + module = "recordstream.ops.numpy" print(f"--- Scanning Module: {module} ---") schemas = scan_module(module) diff --git a/examples/record_pipeline.py b/examples/record_pipeline.py index ff4ff26..8970ec6 100644 --- a/examples/record_pipeline.py +++ b/examples/record_pipeline.py @@ -2,11 +2,11 @@ Demonstrates the modality-neutral core of the engine: -1. a sample is a PLAIN ``dict`` of TYPED values, each owning its metadata — an ``Image`` +1. a record is a PLAIN ``dict`` of TYPED values, each owning its metadata — an ``Image`` carries its layout, a ``Label`` its classes; scalar side values are just more keys; 2. the HEADLINE — ONE pipeline mixing a BARE albumentations transform (invoked natively by the engine's op-family dispatch: it receives exactly its own ``image``/``mask``/``bboxes`` - keys, one call = one joint draw) with native ops. sampleflux ships NO augmentation of its + keys, one call = one joint draw) with native ops. recordstream ships NO augmentation of its own and NO adapter classes — the libraries run as-is; 3. cross-key consistency — ONE ``A.Compose`` draw moves image, mask and bboxes together, the Label untouched; @@ -23,7 +23,7 @@ import torch from torchvision.transforms import v2 -from sampleflux import Image, Label, Mask, Pipeline, Record, Transform, as_transform +from recordstream import Image, Label, Mask, Pipeline, Record, Transform, as_transform def make_record(rng: np.random.Generator) -> Record: diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py index 0db7864..3e6e326 100644 --- a/examples/storage_roundtrip.py +++ b/examples/storage_roundtrip.py @@ -21,11 +21,11 @@ import numpy as np -from sampleflux import Image, Label, Record -from sampleflux.storage.directory import DirectorySink, DirectorySource -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.query import MetadataFilterSource -from sampleflux.storage.zarr import ZarrGroupSink, ZarrGroupSource +from recordstream import Image, Label, Record +from recordstream.storage.directory import DirectorySink, DirectorySource +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source +from recordstream.storage.query import MetadataFilterSource +from recordstream.storage.zarr import ZarrGroupSink, ZarrGroupSource def make_records(n: int = 4) -> list: diff --git a/examples/workflow_pipeline.py b/examples/workflow_pipeline.py index a33a6e1..6a0f2cc 100644 --- a/examples/workflow_pipeline.py +++ b/examples/workflow_pipeline.py @@ -1,7 +1,7 @@ """A resume-safe train → evaluate workflow — ONE Confluid document of runnables. The compelling case for the workflow combinators: a pipeline you can re-run after a crash -(or a second `sampleflux run`) that SKIPS the work whose artifact already exists and +(or a second `recordstream run`) that SKIPS the work whose artifact already exists and carries on — *memoise and continue*, expressed declaratively: 1. ``Sequence`` drives the stages in order (the workflow itself). @@ -76,12 +76,12 @@ def run(self) -> None: def workflow_yaml(work: Path) -> str: """The whole pipeline — stages, the cache guard, AND the format switch — as ONE document.""" return f""" -runnable: !class:sampleflux.workflow.Sequence +runnable: !class:recordstream.workflow.Sequence steps: # Stage 1 — the expensive stage, guarded: train ONLY when the checkpoint is missing. # On a cache hit the !lazy: branch is never even BUILT (no model materialised). - - !class:sampleflux.workflow.Conditional - condition: !class:sampleflux.workflow.PathExists + - !class:recordstream.workflow.Conditional + condition: !class:recordstream.workflow.PathExists path: {work / "model.ckpt"} if_true: null # cache hit -> skip, Sequence continues if_false: !lazy:TrainModel # @configurable classes resolve by registered NAME @@ -89,7 +89,7 @@ def workflow_yaml(work: Path) -> str: # Stage 2 — always runs; the report FORMAT is a Switch on a plain config value # (override from a CLI with --select text — the workflow shape never changes). - - !class:sampleflux.workflow.Switch + - !class:recordstream.workflow.Switch select: json cases: json: !lazy:Evaluate @@ -104,7 +104,7 @@ def workflow_yaml(work: Path) -> str: def run_document(path: Path) -> None: - """What ``sampleflux run `` does: bind the top-level ``runnable:`` and run it.""" + """What ``recordstream run `` does: bind the top-level ``runnable:`` and run it.""" loaded = confluid.load(str(path)) runnable = loaded["runnable"] if isinstance(runnable, Fluid): diff --git a/pyproject.toml b/pyproject.toml index 6da9aa4..02373aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "sampleflux" +name = "recordstream" version = "0.1.0" description = "Clean, functional data pipelines for ML research and production." authors = [{ name = "Taidal", email = "info@gearlux.ai" }] @@ -18,7 +18,7 @@ dependencies = [ "fsspec", "cloudpathlib", "scikit-learn", # LabelMap.fit() uses sklearn.preprocessing.LabelEncoder (lazy-imported) - # liquifai powers the generic `sampleflux run ` CLI (sampleflux.cli) that + # liquifai powers the generic `recordstream run ` CLI (recordstream.cli) that # runs any Confluid-wired runnable; it also brings `rich` (used by DatasetProcessor's # optional console progress bar). liquifai depends only on confluid/loggair/rich — no cycle. "liquifai>=0.1.0", @@ -46,7 +46,7 @@ notebook = [ vision = [ "scipy", # Bare torchvision transforms.v2 ops run as-is through the engine's op-family dispatch - # (sampleflux.core._apply_op); the extra makes `pip install sampleflux[vision]` the + # (recordstream.core._apply_op); the extra makes `pip install recordstream[vision]` the # documented way to enable them. "torchvision", ] @@ -56,72 +56,72 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" # CONFLUID-CONFIGURABLE DISCOVERY -# Sources / ops live in separate submodules that `sampleflux/__init__.py` +# Sources / ops live in separate submodules that `recordstream/__init__.py` # does not eagerly import. Listing them here makes navigaitor discover # every ``@configurable`` on bootstrap without hardcoding the list. [project.entry-points."confluid.configurables"] -sampleflux = "sampleflux" -sampleflux-core = "sampleflux.core" -sampleflux-sources = "sampleflux.sources" -sampleflux-ops-parallel = "sampleflux.ops.parallel" -sampleflux-ops-enable = "sampleflux.ops.enable" -sampleflux-ops-random-apply = "sampleflux.ops.random_apply" -# ConfigureOp (per-sample parameter injection — the helios Configure pattern); entry-point -# changes need an editable reinstall before FluxStudio/navigaitor discovery sees the module. -sampleflux-ops-configure = "sampleflux.ops.configure" -sampleflux-ops-formula = "sampleflux.ops.formula" +recordstream = "recordstream" +recordstream-core = "recordstream.core" +recordstream-sources = "recordstream.sources" +recordstream-ops-parallel = "recordstream.ops.parallel" +recordstream-ops-enable = "recordstream.ops.enable" +recordstream-ops-random-apply = "recordstream.ops.random_apply" +# ConfigureOp (per-record parameter injection — the helios Configure pattern); entry-point +# changes need an editable reinstall before StreamStudio/navigaitor discovery sees the module. +recordstream-ops-configure = "recordstream.ops.configure" +recordstream-ops-formula = "recordstream.ops.formula" # Context ops (Save/Use/Drop/Apply/Capture/MergeFields) — the graph-plane building blocks lowered from flow: docs -sampleflux-ops-context = "sampleflux.ops.context" +recordstream-ops-context = "recordstream.ops.context" # The FlowGraph engine (flow: named-step documents + the flow<->ops converters) -sampleflux-flow = "sampleflux.flow" +recordstream-flow = "recordstream.flow" # The queryable-metadata scan protocol + MetadataFilterSource view source -sampleflux-storage-query = "sampleflux.storage.query" -sampleflux-ops-sink = "sampleflux.ops.sink" -sampleflux-ops-numpy = "sampleflux.ops.numpy" -sampleflux-ops-torch = "sampleflux.ops.torch" -sampleflux-ops-target = "sampleflux.ops.target" -sampleflux-ops-image = "sampleflux.ops.image" -# PrintSampleOp (log/print a per-sample summary). Entry-point changes need an editable reinstall -# before FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). -sampleflux-ops-debug = "sampleflux.ops.debug" +recordstream-storage-query = "recordstream.storage.query" +recordstream-ops-sink = "recordstream.ops.sink" +recordstream-ops-numpy = "recordstream.ops.numpy" +recordstream-ops-torch = "recordstream.ops.torch" +recordstream-ops-target = "recordstream.ops.target" +recordstream-ops-image = "recordstream.ops.image" +# PrintRecordOp (log/print a per-record summary). Entry-point changes need an editable reinstall +# before StreamStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). +recordstream-ops-debug = "recordstream.ops.debug" # Storage SINKS (HDF5Sink / ZarrGroupSink / ZarrBatchSink / DirectorySink) carry -# category="sink" so FluxStudio surfaces them as DatasetProcessor sink nodes. They live under -# sampleflux.storage.* (NOT re-exported from the package root), and scan_module does not recurse +# category="sink" so StreamStudio surfaces them as DatasetProcessor sink nodes. They live under +# recordstream.storage.* (NOT re-exported from the package root), and scan_module does not recurse # submodules, so each storage module needs its own entry point. The matching SOURCES in these # modules stay uncategorised, so the positive {op,source,engine,sink} allowlist surfaces only the # tagged sinks. (Entry-point changes need an editable reinstall — `aisland setup`, never --reinstall.) -sampleflux-storage-hdf5 = "sampleflux.storage.hdf5" -sampleflux-storage-zarr = "sampleflux.storage.zarr" -sampleflux-storage-directory = "sampleflux.storage.directory" +recordstream-storage-hdf5 = "recordstream.storage.hdf5" +recordstream-storage-zarr = "recordstream.storage.zarr" +recordstream-storage-directory = "recordstream.storage.directory" # The record model's op surface: Transform (type-dispatched record ops) + Pipeline (THE -# compose op) live in sampleflux.transform. Entry-point changes need an editable reinstall -# before FluxStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). -sampleflux-transform = "sampleflux.transform" +# compose op) live in recordstream.transform. Entry-point changes need an editable reinstall +# before StreamStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). +recordstream-transform = "recordstream.transform" # Structure ops (RenameField/DropField/CopyField/SelectFields) — reshape a record's entries. -sampleflux-ops-structure = "sampleflux.ops.structure" +recordstream-ops-structure = "recordstream.ops.structure" # The runnable orchestration layer: DatasetProcessor (generic source→sink runner) and the # workflow combinators (Sequence/Conditional/Switch + PathExists/Not/AllOf/AnyOf predicates). -# Discovered as @configurable runnables; run via `sampleflux run`. -sampleflux-processing = "sampleflux.processing" -sampleflux-workflow = "sampleflux.workflow" +# Discovered as @configurable runnables; run via `recordstream run`. +recordstream-processing = "recordstream.processing" +recordstream-workflow = "recordstream.workflow" -# The `sampleflux` console script — `sampleflux run ` runs any Confluid-wired +# The `recordstream` console script — `recordstream run ` runs any Confluid-wired # runnable (a trainer, an evaluator, a DatasetProcessor, a workflow). [project.scripts] -sampleflux = "sampleflux.cli:main" +recordstream = "recordstream.cli:main" # Liquifai app registration: lets `liquifai-install-completions` discover this CLI via an # instant entry-point metadata read instead of a subprocess probe. Name = the binary name; # value = the LiquifyApp instance. [project.entry-points."liquifai.apps"] -sampleflux = "sampleflux.cli:app" +recordstream = "recordstream.cli:app" [tool.setuptools.packages.find] where = ["."] -include = ["sampleflux*"] +include = ["recordstream*"] [tool.setuptools.package-data] -sampleflux = ["py.typed"] +recordstream = ["py.typed"] [tool.black] line-length = 120 diff --git a/sampleflux/__init__.py b/recordstream/__init__.py similarity index 61% rename from sampleflux/__init__.py rename to recordstream/__init__.py index f9ef837..5adcd28 100644 --- a/sampleflux/__init__.py +++ b/recordstream/__init__.py @@ -1,24 +1,32 @@ """ -SampleFlux: Modular, functional data pipelines. +RecordStream: Modular, functional data pipelines. -The data model is the RECORD: a sample is a plain ``dict`` of typed values (each value +The data model is the RECORD: a record is a plain ``dict`` of typed values (each value owning its metadata — an ``Image`` its layout, a ``Label`` its classes), and ops dispatch on value TYPE (the torchvision-v2 model). Bare albumentations / torchvision ``transforms.v2`` transforms drop into any ops list AS-IS — the engine invokes each op family natively -(``sampleflux.core._apply_op``). Import the whole surface from the package top level -(``from sampleflux import Record, Image, Transform, Pipeline, ...``). +(``recordstream.core._apply_op``). Import the whole surface from the package top level +(``from recordstream import Record, Image, Transform, Pipeline, ...``). """ # --- shared infrastructure ----------------------------------------------------------------- -from sampleflux.collate import collate, collate_records, get_collate, register_collate, registered_collates -from sampleflux.context import Context -from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp, register_op_family, registered_op_families +from recordstream.collate import collate, collate_records, get_collate, register_collate, registered_collates +from recordstream.context import Context +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp, register_op_family, registered_op_families # --- the record data model + transforms + item codec ---------------------------------------- -from sampleflux.dispatch import dispatch, register_kernel, registered_kernels -from sampleflux.flow import FlowGraph, from_ops, to_ops -from sampleflux.io import EncodedField, EncodedItem, decode_item, decode_record, encode_item, encode_record, register_io -from sampleflux.items import ( +from recordstream.dispatch import dispatch, register_kernel, registered_kernels +from recordstream.flow import FlowGraph, from_ops, to_ops +from recordstream.io import ( + EncodedField, + EncodedItem, + decode_item, + decode_record, + encode_item, + encode_record, + register_io, +) +from recordstream.items import ( Image, Label, Mask, @@ -33,10 +41,10 @@ register_item, with_data, ) -from sampleflux.labels import LabelMap -from sampleflux.processing import DatasetProcessor -from sampleflux.projection import SupportsProjection, iter_key, num_classes, project -from sampleflux.runnable import ( +from recordstream.labels import LabelMap +from recordstream.processing import DatasetProcessor +from recordstream.projection import SupportsProjection, iter_key, num_classes, project +from recordstream.runnable import ( ProgressCallback, ProgressReporting, TorchRunner, @@ -44,9 +52,9 @@ entrypoint_tasks, runnable_entrypoints, ) -from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName -from sampleflux.transform import FunctionTransform, Pipeline, Transform, as_transform -from sampleflux.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName +from recordstream.transform import FunctionTransform, Pipeline, Transform, as_transform +from recordstream.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch __all__ = [ # ---- record data model ---- @@ -79,8 +87,8 @@ "decode_record", # ---- shared infrastructure ---- "Context", - "Flux", - "JointFlux", + "Stream", + "JointStream", "FilterOp", "WrappedOp", "register_op_family", diff --git a/sampleflux/cli.py b/recordstream/cli.py similarity index 71% rename from sampleflux/cli.py rename to recordstream/cli.py index 442b25a..56b362a 100644 --- a/sampleflux/cli.py +++ b/recordstream/cli.py @@ -1,21 +1,21 @@ -"""The ``sampleflux`` CLI — a generic runner for any Confluid-wired runnable. +"""The ``recordstream`` CLI — a generic runner for any Confluid-wired runnable. -``sampleflux run `` loads a Confluid YAML that binds a *runnable* +``recordstream run `` loads a Confluid YAML that binds a *runnable* object (anything exposing a no-arg ``run()``) under the top-level ``runnable:`` key, flows it under the active context (so nested ``!ref:`` markers resolve), and calls ``run()``. This is the single entry point that replaces bespoke per-verb CLIs: a training run, an evaluation, a dataset conversion, or a whole -:mod:`~sampleflux.workflow` are all just runnables — the ``!class:`` the YAML roots +:mod:`~recordstream.workflow` are all just runnables — the ``!class:`` the YAML roots on decides what happens. Example:: # convert.yaml - runnable: !class:sampleflux.processing.DatasetProcessor - flux: !class:sampleflux.Flux { source: !class:my.Source(), ops: [...] } - sink: !class:sampleflux.storage.HDF5Sink { path: out.h5 } + runnable: !class:recordstream.processing.DatasetProcessor + stream: !class:recordstream.Stream { source: !class:my.Source(), ops: [...] } + sink: !class:recordstream.storage.HDF5Sink { path: out.h5 } - sampleflux run convert.yaml + recordstream run convert.yaml """ from typing import Any @@ -25,7 +25,7 @@ logger = get_logger(__name__) -app = LiquifyApp(name="sampleflux") +app = LiquifyApp(name="recordstream") @app.script_command(flow_mode="auto") @@ -51,7 +51,7 @@ def run(runnable: Any) -> None: if not callable(run_method): logger.error(f"The injected runnable ({label}) does not implement 'run()'.") return - logger.info(f"sampleflux running: {label}") + logger.info(f"recordstream running: {label}") run_method() diff --git a/sampleflux/collate.py b/recordstream/collate.py similarity index 93% rename from sampleflux/collate.py rename to recordstream/collate.py index 5f15fa4..e8c2697 100644 --- a/sampleflux/collate.py +++ b/recordstream/collate.py @@ -1,6 +1,6 @@ """The pluggable collate registry — batch builders keyed by representation. -Batching in sampleflux is two-stage: the engine groups carriers (``Flux.batch`` / +Batching in recordstream is two-stage: the engine groups carriers (``Stream.batch`` / ``FlowGraph.batch`` yield ``list``\\ s of N items) and a COLLATE function stacks a group into one batched carrier. This registry gives consumer packages ONE addressable home for their task collates — consumers ``register_collate`` their task collates additively, and @@ -21,8 +21,8 @@ from loggair import get_logger -from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item -from sampleflux.items import Record +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record logger = get_logger(__name__) @@ -72,7 +72,7 @@ def collate(items: Sequence[Any], key: Optional[str] = None) -> Any: """Collate ``items`` into one batched carrier. ``key`` picks a registered collate explicitly; omitted, the default ``"record"`` collate - is used (every carrier is a plain record dict). An empty batch raises. + is used (every carrier is a plain dict). An empty batch raises. """ if not items: raise ValueError("collate: cannot collate an empty batch") @@ -106,7 +106,7 @@ def collate_records(items: Sequence[Record]) -> Record: """The record collate: N record dicts → ONE batched record dict. Per key (union of keys is NOT taken — every record must carry the same keys, a - mismatch raises): typed values encode through :func:`~sampleflux.io.encode_item`, + mismatch raises): typed values encode through :func:`~recordstream.io.encode_item`, payloads are stacked via :func:`_stack` (torch → stacked tensor, numpy → stacked array, else a list) and each declared item attr becomes a LIST of per-record values, decoding back into ONE batched item of the same type. A ``"plain"``-tagged value diff --git a/sampleflux/context.py b/recordstream/context.py similarity index 76% rename from sampleflux/context.py rename to recordstream/context.py index 9032f7c..fa53838 100644 --- a/sampleflux/context.py +++ b/recordstream/context.py @@ -1,19 +1,19 @@ -"""Per-sample named-cell store — the graph data plane for graph-shaped pipelines. +"""Per-record named-cell store — the graph data plane for graph-shaped pipelines. -A :class:`Context` holds named **cells** for exactly one sample's trip through the op +A :class:`Context` holds named **cells** for exactly one record's trip through the op list: branch snapshots (a cell holding a record dict), captured -``@output`` values, and per-sample parameters. The context ops in -:mod:`sampleflux.ops.context` (``Save`` / ``Use`` / ``Drop`` / ``Apply`` / ``Capture`` / -``Mix``) move data between the linear sample stream and these cells, which is what lets +``@output`` values, and per-record parameters. The context ops in +:mod:`recordstream.ops.context` (``Save`` / ``Use`` / ``Drop`` / ``Apply`` / ``Capture`` / +``Mix``) move data between the linear record stream and these cells, which is what lets a plain sequential op list execute a fan-out/fan-in graph. -The context ops route graph data through these per-sample Context CELLS and never touch -the sample's own fields — each typed item still owns its own metadata inside the sample. -The Context is the *wiring* plane — engine-created, per sample, empty again by the end of +The context ops route graph data through these per-record Context CELLS and never touch +the record's own fields — each typed item still owns its own metadata inside the record. +The Context is the *wiring* plane — engine-created, per record, empty again by the end of a well-formed graph (every cell freed after its last read). Nothing here is ``@configurable``; a Context never appears in YAML. -The engine (``Flux`` — and ``FlowGraph``, which manages its env directly) creates one +The engine (``Stream`` — and ``FlowGraph``, which manages its env directly) creates one Context per source item and activates it around the op loop via a :class:`contextvars.ContextVar`, so ops reach it inside ``__call__`` with no signature change (:func:`current` / :func:`require`). A hand-rolled loop outside an engine opts in @@ -21,7 +21,7 @@ with activate(Context()): for op in ops: - sample = op(sample) + record = op(record) """ import contextvars @@ -32,7 +32,7 @@ class Context: - """Named-cell store for one sample's trip through a graph-shaped pipeline. + """Named-cell store for one record's trip through a graph-shaped pipeline. Cells are stored and returned **by reference** — copy semantics are the reading op's decision (``Use`` deep-copies unless it drops the cell). @@ -89,11 +89,11 @@ def __repr__(self) -> str: # pragma: no cover - debug aid return f"Context(cells={sorted(self._cells)})" -_CURRENT: contextvars.ContextVar[Optional[Context]] = contextvars.ContextVar("sampleflux_context", default=None) +_CURRENT: contextvars.ContextVar[Optional[Context]] = contextvars.ContextVar("recordstream_context", default=None) def current() -> Optional[Context]: - """The active per-sample :class:`Context`, or ``None`` outside an engine/`activate` block.""" + """The active per-record :class:`Context`, or ``None`` outside an engine/`activate` block.""" return _CURRENT.get() @@ -102,16 +102,16 @@ def require(op_name: str = "context op") -> Context: ctx = _CURRENT.get() if ctx is None: raise RuntimeError( - f"{op_name}: no active Context. Context ops need the per-sample Context the engine " - f"creates — run the pipeline through Flux/FlowGraph, or wrap a manual loop in " - f"`with sampleflux.context.activate(Context()):`." + f"{op_name}: no active Context. Context ops need the per-record Context the engine " + f"creates — run the pipeline through Stream/FlowGraph, or wrap a manual loop in " + f"`with recordstream.context.activate(Context()):`." ) return ctx @contextmanager def activate(ctx: Context) -> Iterator[Context]: - """Activate ``ctx`` as the current per-sample Context for the enclosed block.""" + """Activate ``ctx`` as the current per-record Context for the enclosed block.""" token = _CURRENT.set(ctx) try: yield ctx diff --git a/sampleflux/core.py b/recordstream/core.py similarity index 83% rename from sampleflux/core.py rename to recordstream/core.py index b8f247a..98364e4 100644 --- a/sampleflux/core.py +++ b/recordstream/core.py @@ -10,8 +10,8 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.context import Context, activate -from sampleflux.items import NDArrayItem, Record, item_data, with_data +from recordstream.context import Context, activate +from recordstream.items import NDArrayItem, Record, item_data, with_data logger = get_logger(__name__) @@ -176,23 +176,23 @@ def _describe_deferred_source(source: Any) -> str: def _fluid_source_guidance(source: Any) -> str: - """Build an actionable message when Flux.source is still a Confluid Fluid.""" + """Build an actionable message when Stream.source is still a Confluid Fluid.""" return ( - f"Flux.source is still a deferred Confluid marker: {_describe_deferred_source(source)}. " + f"Stream.source is still a deferred Confluid marker: {_describe_deferred_source(source)}. " "Confluid has not materialized it yet. Fixes: (a) in YAML, write the source as " "`!class:X()` (with parens) instead of `!class:X` so it becomes an Instance and is " "materialized at load time; (b) or call `flow(source)` on the source before handing " - "it to Flux." + "it to Stream." ) def _fluid_op_guidance(op: Any, index: int) -> str: - """Build an actionable message when a Flux op marker cannot be materialized.""" + """Build an actionable message when a Stream op marker cannot be materialized.""" return ( - f"Flux.ops[{index}] is a deferred Confluid marker that could not be materialized: " + f"Stream.ops[{index}] is a deferred Confluid marker that could not be materialized: " f"{_describe_deferred_source(op)}. Fixes: (a) in YAML, write the op as `!class:X()` " "(with parens) so it becomes an Instance and is materialized at load time; (b) or " - "call `flow(op)` on the op before handing it to Flux." + "call `flow(op)` on the op before handing it to Stream." ) @@ -219,7 +219,7 @@ def _check_ops_materialized(ops: List[Any]) -> None: class FilterOp: """Configurable filter operation. - The op form of :meth:`Flux.filter` — a predicate gate over the stream: the record + The op form of :meth:`Stream.filter` — a predicate gate over the stream: the record passes when the predicate returns ``True`` and is dropped otherwise (``__call__`` returns ``None``, which every engine route treats as "skip this record"). @@ -242,9 +242,9 @@ def __call__(self, record: Record) -> Optional[Record]: class WrappedOp: """Configurable transformation wrapper with smart mapping. - The op form of :meth:`Flux.map` — lifts a plain function over one record value. The + The op form of :meth:`Stream.map` — lifts a plain function over one record value. The callable is ALWAYS stored as its importable ``module:function`` path (via - :mod:`sampleflux.discovery`), so the op pickles across ``spawn`` workers and + :mod:`recordstream.discovery`), so the op pickles across ``spawn`` workers and serializes into Confluid YAML verbatim; the live function resolves lazily on first call. @@ -257,7 +257,7 @@ class WrappedOp: """ def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: Optional[Dict[str, Any]] = None): - from sampleflux.discovery import get_callable_path + from recordstream.discovery import get_callable_path # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` # property). EXPLICIT: always store the string path for serialization. @@ -270,7 +270,7 @@ def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: @property def func(self) -> Callable: if self._func_cache is None: - from sampleflux.discovery import resolve_callable + from recordstream.discovery import resolve_callable self._func_cache = resolve_callable(self.f) return self._func_cache @@ -292,44 +292,44 @@ def __call__(self, record: Record) -> Optional[Record]: class _Carried(NamedTuple): """A record travelling the streamed route together with its per-record Context.""" - sample: Any + record: Any ctx: Context -def _expand(op: Any, sample: Any) -> List[Any]: +def _expand(op: Any, record: Any) -> List[Any]: """Run a 1→N EXPANDING op and return its flattened children.""" - raw = op(sample) + raw = op(record) if raw is None: return [] return [child for child in raw if child is not None] def _worker_task( - sample: Any, ops: List[Any], families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None + record: Any, ops: List[Any], families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None ) -> Optional[Any]: """Single-result worker for STRICTLY 1→1 op lists (the ``Parallel`` op's contract). Kept for callers that need exactly one carrier back; expanding ops raise here — route expanding pipelines through :func:`_worker_task_multi`. """ - results = _worker_task_multi(sample, ops, allow_expansion=False, families=families) + results = _worker_task_multi(record, ops, allow_expansion=False, families=families) return results[0] if results else None def _worker_task_multi( - sample: Any, + record: Any, ops: List[Any], allow_expansion: bool = True, families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, ) -> List[Any]: """Top-level helper for multiprocess workers. Must be at top level for pickling. - Runs one source :class:`Sample` through the op list and returns EVERY resulting sample — + Runs one source :class:`Record` through the op list and returns EVERY resulting record — usually one, zero when filtered, several when a 1→N EXPANDING op fired; each expansion - child continues through the REMAINING ops with a shallow copy of the per-sample Context, + child continues through the REMAINING ops with a shallow copy of the per-record Context, depth-first so sibling order matches the nested-loop intuition. - Activates ONE fresh per-sample :class:`~sampleflux.context.Context` around the op loop so + Activates ONE fresh per-record :class:`~recordstream.context.Context` around the op loop so context ops (``Save``/``Use``/``Apply``/``Capture``/``MergeFields``) can move data between the linear stream and named cells — the executor itself stays a plain ``for op in ops`` loop. Contexts are created inside the worker (spawn-safe: ops pickle, a Context never @@ -340,7 +340,7 @@ def _worker_task_multi( from collections import deque _sync_op_families(families) - pending: "deque[Tuple[Any, Context, int]]" = deque([(sample, Context(), 0)]) + pending: "deque[Tuple[Any, Context, int]]" = deque([(record, Context(), 0)]) out: List[Any] = [] while pending: current, ctx, start = pending.popleft() @@ -354,7 +354,7 @@ def _worker_task_multi( if not allow_expansion: raise TypeError( f"op {type(op).__name__!r} is a 1→N expanding op, which this strictly " - "1→1 route cannot carry — run it through the Flux iteration paths." + "1→1 route cannot carry — run it through the Stream iteration paths." ) children = _expand(op, current) if not children: @@ -377,43 +377,43 @@ def _worker_task_multi( @configurable(category="engine") -class JointFlux: +class JointStream: """ - Aggregates multiple Flux streams into a single joint stream. - Each sub-flux maintains its own unique transformation chain. + Aggregates multiple Stream streams into a single joint stream. + Each sub-stream maintains its own unique transformation chain. - The iteration-only fan-in engine behind :meth:`Flux.joint`: each sub-flux applies + The iteration-only fan-in engine behind :meth:`Stream.joint`: each sub-stream applies its OWN op chain, so differently-processed streams concatenate lazily without materialization. For an indexable (random-access) concatenation of raw sources, use ``ConcatSource`` instead. Args: - fluxes: The Flux streams to concatenate; iteration walks them in order and length is their sum. + streams: The Stream streams to concatenate; iteration walks them in order and length is their sum. Defaults to ``None`` ⇒ an empty joint stream (zero-arg construction). """ - def __init__(self, fluxes: Optional[List["Flux"]] = None) -> None: - # Lazy / zero-arg: store config only; no sub-fluxes ⇒ an empty stream. - self.fluxes = fluxes if fluxes is not None else [] + def __init__(self, streams: Optional[List["Stream"]] = None) -> None: + # Lazy / zero-arg: store config only; no sub-streams ⇒ an empty stream. + self.streams = streams if streams is not None else [] def __iter__(self) -> Iterator[Record]: - """Iterate through all sub-fluxes sequentially.""" - for flux in self.fluxes: - yield from flux + """Iterate through all sub-streams sequentially.""" + for stream in self.streams: + yield from stream def __len__(self) -> int: - """Total length is the sum of all sub-fluxes.""" - return sum(len(f) for f in self.fluxes) + """Total length is the sum of all sub-streams.""" + return sum(len(f) for f in self.streams) @configurable(category="engine") -class Flux(torch.utils.data.Dataset[Record]): +class Stream(torch.utils.data.Dataset[Record]): """ - The primary stream engine for SampleFlux. + The primary stream engine for RecordStream. Wraps any iterable or indexed dataset and provides a functional API. Every carrier is a plain record ``dict`` of typed values, and every op is applied - through the op-FAMILY dispatch (:func:`_apply_op`) — so native sampleflux ops, + through the op-FAMILY dispatch (:func:`_apply_op`) — so native recordstream ops, bare albumentations transforms, and bare torchvision ``transforms.v2`` transforms all sit in ONE ``ops`` list as-is. ``source`` is duck-typed (any iterable; the Indexable protocol if ``__getitem__``/``__len__`` are present). @@ -445,17 +445,17 @@ def _guard_live_source(self) -> Any: return self.source @classmethod - def from_source(cls, source: Any) -> "Flux": - """Create a Flux from a DataSource.""" + def from_source(cls, source: Any) -> "Stream": + """Create a Stream from a DataSource.""" return cls(source=source) @classmethod - def joint(cls, fluxes: List["Flux"]) -> "Flux": - """Create a new Flux that aggregates multiple other Flux streams.""" - return cls(source=JointFlux(fluxes)) + def joint(cls, streams: List["Stream"]) -> "Stream": + """Create a new Stream that aggregates multiple other Stream streams.""" + return cls(source=JointStream(streams)) @classmethod - def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": + def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Stream": """Attach an ops-only Confluid YAML (e.g. one exported by a pipeline-authoring tool) to ``source``. ``path`` is the ``{ops: [!class:...()]}`` document produced by an external graph @@ -469,11 +469,11 @@ def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Fl return cls(source=source, ops=ops) @classmethod - def from_flow_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Flux": + def from_flow_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Stream": """Attach a ``{flow: {...}}`` graph document to ``source``, LOWERED to the serial form.""" - from sampleflux.flow import flow_yaml_to_flux + from recordstream.flow import flow_yaml_to_stream - return cast("Flux", flow_yaml_to_flux(path, source=source)) + return cast("Stream", flow_yaml_to_stream(path, source=source)) @property def _expands(self) -> bool: @@ -486,10 +486,10 @@ def _guard_not_expanding(self, operation: str) -> None: type(op).__name__ for op in self.ops if not isinstance(op, _ConfluidFluid) and _op_expands(op) ) raise TypeError( - f"Flux.{operation}: the pipeline contains the 1→N expanding op {culprit!r}, so the " + f"Stream.{operation}: the pipeline contains the 1→N expanding op {culprit!r}, so the " "expanded length/index mapping is unknowable up front — the pipeline is ITERABLE-ONLY. " "Iterate it (or wrap in a torch IterableDataset); for random access, window/expand at " - "the source instead, or materialize with list(flux) first." + "the source instead, or materialize with list(stream) first." ) def __len__(self) -> int: @@ -503,59 +503,59 @@ def __len__(self) -> int: return 0 def __getitem__(self, index: int) -> Any: - """Random access: get the i-th sample with ops applied.""" + """Random access: get the i-th record with ops applied.""" source = self._guard_live_source() if source is None: - raise TypeError("Flux source is None — cannot index. Pass a DataSource / iterable to Flux(source=...).") + raise TypeError("Stream source is None — cannot index. Pass a DataSource / iterable to Stream(source=...).") self._guard_not_expanding("__getitem__") if hasattr(source, "__getitem__"): raw = source[index] elif hasattr(source, "__len__"): if self._indexable_cache is None: - logger.debug(f"Flux: materializing iterable-only source {type(source).__name__} for random access.") + logger.debug(f"Stream: materializing iterable-only source {type(source).__name__} for random access.") self._indexable_cache = list(source) raw = self._indexable_cache[index] else: raise TypeError( - f"Flux source {type(source).__name__} does not support indexing and has no __len__ " + f"Stream source {type(source).__name__} does not support indexing and has no __len__ " "(bare iterator). Map-style DataLoader random access is unsafe on a one-shot " - "iterator; give the source a __len__ (then Flux caches on first access) or wrap " - "it in ``list(...)`` before handing it to Flux." + "iterator; give the source a __len__ (then Stream caches on first access) or wrap " + "it in ``list(...)`` before handing it to Stream." ) _check_ops_materialized(self.ops) - sample: Any = raw + record: Any = raw with activate(Context()): for op in self.ops: - result = _apply_op(sample, op) + result = _apply_op(record, op) if result is None: - raise IndexError(f"Sample {index} filtered out by {op}") - sample = result - return cast(Record, sample) + raise IndexError(f"Record {index} filtered out by {op}") + record = result + return cast(Record, record) def to_sink(self, sink: Any) -> None: - """Write the entire flux to a DataSink.""" - from sampleflux.storage.base import Storage + """Write the entire stream to a DataSink.""" + from recordstream.storage.base import Storage target_sink: Any = sink if isinstance(sink, Storage) else nullcontext() with target_sink: - for sample in self: - sink.write(sample) + for record in self: + sink.write(record) sink.flush() - def parallel(self, workers: int = 4) -> "Flux": + def parallel(self, workers: int = 4) -> "Stream": """Enable multiprocess execution for the pipeline.""" self._workers = workers return self - def batch(self, chunk_size: int) -> "Flux": - """Group samples into chunks (lists of N samples).""" + def batch(self, chunk_size: int) -> "Stream": + """Group records into chunks (lists of N records).""" self._chunk_size = chunk_size return self - def map(self, func: Callable, key: Optional[str] = None, **kwargs: Any) -> "Flux": - """Append a transformation to the flux. + def map(self, func: Callable, key: Optional[str] = None, **kwargs: Any) -> "Stream": + """Append a transformation to the stream. ``key`` names the record entry whose payload ``func`` transforms; ``None`` hands ``func`` the whole record dict. @@ -564,8 +564,8 @@ def map(self, func: Callable, key: Optional[str] = None, **kwargs: Any) -> "Flux self.ops.append(op) return self - def filter(self, predicate: Callable[[Record], bool]) -> "Flux": - """Filter the flux based on a predicate.""" + def filter(self, predicate: Callable[[Record], bool]) -> "Stream": + """Filter the stream based on a predicate.""" self.ops.append(FilterOp(predicate)) return self @@ -583,8 +583,8 @@ def __iter__(self) -> Iterator[Any]: if self._chunk_size > 0: batch = [] - for sample in it: - batch.append(sample) + for record in it: + batch.append(record) if len(batch) == self._chunk_size: yield batch batch = [] @@ -594,7 +594,7 @@ def __iter__(self) -> Iterator[Any]: yield from it def _iter_streamed(self) -> Iterator[Record]: - """Mixed per-sample / stream-level op chain (a stream-level op exposes ``.stream``).""" + """Mixed per-record / stream-level op chain (a stream-level op exposes ``.stream``).""" source = self._guard_live_source() if source is None: return @@ -604,16 +604,16 @@ def to_carried() -> Iterator[Optional[_Carried]]: for item in source: yield _Carried(item, Context()) - def per_sample(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[_Carried]]: + def per_record(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[_Carried]]: expands = _op_expands(op) for c in stream: if c is None: continue with activate(c.ctx): if expands: - children = _expand(op, c.sample) + children = _expand(op, c.record) else: - s = _apply_op(c.sample, op) + s = _apply_op(c.record, op) if expands: for j, child in enumerate(children): yield _Carried(child, c.ctx if j == 0 else c.ctx.copy()) @@ -627,11 +627,11 @@ def strip(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[Re continue if c.ctx.live(): raise RuntimeError( - f"Flux: context cells {c.ctx.live()!r} are still live at the stream-level op " + f"Stream: context cells {c.ctx.live()!r} are still live at the stream-level op " f"{type(op).__name__!r}. Context cells cannot cross a stream-op boundary " f"(e.g. Parallel) — drop them before it, or move the whole graph inside it." ) - yield c.sample + yield c.record def wrap(stream: Iterator[Optional[Record]]) -> Iterator[Optional[_Carried]]: for s in stream: @@ -642,11 +642,11 @@ def wrap(stream: Iterator[Optional[Record]]) -> Iterator[Optional[_Carried]]: if hasattr(op, "stream") and callable(op.stream): carried = wrap(op.stream(strip(carried, op))) else: - carried = per_sample(carried, op) + carried = per_record(carried, op) for c in carried: if c is not None: - yield c.sample + yield c.record def _iter_sequential(self) -> Iterator[Record]: """Standard single-threaded execution.""" @@ -677,13 +677,13 @@ def _iter_parallel(self) -> Iterator[Record]: yield from future.result() def collect(self) -> List[Record]: - """Materialize the full flux into a list.""" + """Materialize the full stream into a list.""" return list(self) def project(self, keys: Collection[str]) -> Iterator[Record]: """Yield pipeline-output records carrying only ``keys`` (the projection primitive). - Implements :class:`sampleflux.projection.SupportsProjection`. Flux must run its op + Implements :class:`recordstream.projection.SupportsProjection`. Stream must run its op chain to produce each record (an op may consume the input), so this is the generic "iterate, then keep only the requested keys" form. Lazy: a generator. """ diff --git a/sampleflux/discovery.py b/recordstream/discovery.py similarity index 98% rename from sampleflux/discovery.py rename to recordstream/discovery.py index 9634e13..5bdc904 100644 --- a/sampleflux/discovery.py +++ b/recordstream/discovery.py @@ -1,4 +1,4 @@ -"""Passive introspection for SampleFlux callables. +"""Passive introspection for RecordStream callables. A round-trip bridge between live Python callables (sources, ops, plain functions) and JSON-serializable schemas, so downstream tools can discover and @@ -16,7 +16,7 @@ their property panels. The serialization half doubles as the workspace's generic string-callable hook -pattern (:class:`~sampleflux.core.WrappedOp` stores its ``f`` this way; consuming +pattern (:class:`~recordstream.core.WrappedOp` stores its ``f`` this way; consuming packages reuse it for their own dotted-path hooks). Curated discovery (MCP form-specs, option pickers) builds on the Confluid registry instead — which registers classes AND builder functions, but only opt-in by name; this module is diff --git a/sampleflux/dispatch.py b/recordstream/dispatch.py similarity index 97% rename from sampleflux/dispatch.py rename to recordstream/dispatch.py index 6da114f..ae33c16 100644 --- a/sampleflux/dispatch.py +++ b/recordstream/dispatch.py @@ -3,7 +3,7 @@ An op does not hard-code how to handle each value type. Instead a kernel is registered per ``(transform class, item type)`` pair, and :func:`dispatch` looks one up — walking the value's MRO so a kernel registered for a base item type also serves its subclasses. This is -the same registry idea as :mod:`sampleflux.collate` (batching keyed by representation), +the same registry idea as :mod:`recordstream.collate` (batching keyed by representation), applied to per-type op behaviour. Registration is open: a downstream package teaches an existing op about a new value diff --git a/sampleflux/flow.py b/recordstream/flow.py similarity index 91% rename from sampleflux/flow.py rename to recordstream/flow.py index cd71a19..4461790 100644 --- a/sampleflux/flow.py +++ b/recordstream/flow.py @@ -3,25 +3,25 @@ A **flow document** is the readable, named-step form of a graph-shaped pipeline: a mapping of ``step-name → op``, where a step's name is also the name later steps use to reference its result. It is the authoring format (humans and graph exporters -write it); the flat context-ops form (:mod:`sampleflux.ops.context`) is the serial -execution format the plain :class:`~sampleflux.core.Flux` engine runs. The two convert +write it); the flat context-ops form (:mod:`recordstream.ops.context`) is the serial +execution format the plain :class:`~recordstream.core.Stream` engine runs. The two convert **bidirectionally**: :func:`to_ops` lowers a flow into a flat op list, :func:`from_ops` lifts a flat op list back — with execution parity in both directions. .. code-block:: yaml flow: - spec: !class:waivefront.SpectrogramOp() # input: the source sample - rescaled: !class:sampleflux.ops.numpy.RescaleOp() # input: previous step + spec: !class:waivefront.SpectrogramOp() # input: the source record + rescaled: !class:recordstream.ops.numpy.RescaleOp() # input: previous step masked: !class:waivefront.SegmentOp() {from: spec} # 2nd reader of spec = fan-out - thresh: !class:sampleflux.ops.formula.FormulaOp(formula="a*0.5") {from: masked} + thresh: !class:recordstream.ops.formula.FormulaOp(formula="a*0.5") {from: masked} out: {from: masked, merge_from: [rescaled]} # typed fan-in (no op) outputs: out Step grammar (the three RESERVED step keys, stripped before the op is built): -- ``from:`` — the step supplying this step's input sample. Omitted = the previous step - (the first step reads the source sample). Must name an EARLIER step: document order is +- ``from:`` — the step supplying this step's input record. Omitted = the previous step + (the first step reads the source record). Must name an EARLIER step: document order is the schedule, so forward references are errors and cycles are inexpressible. - ``merge_from:`` — fan-in: UNION another step's record entries into this step's incoming record before the op runs (the ``MergeFields`` slot semantics — last-write-wins on a @@ -48,9 +48,9 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from sampleflux.core import _apply_op -from sampleflux.items import Record -from sampleflux.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Save, Use, _read_output +from recordstream.core import _apply_op +from recordstream.items import Record +from recordstream.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Save, Use, _read_output logger = get_logger(__name__) @@ -65,7 +65,7 @@ class FlowStep(NamedTuple): name: str op: Optional[Any] # live op callable; None = pure fan-in / identity step - from_: Optional[str] # None = previous step (first step: the source sample) + from_: Optional[str] # None = previous step (first step: the source record) bind: Dict[str, str] # param -> "step" | "step.attr" | "step[key]" merge_from: Tuple[str, ...] = () # typed fan-in: union these steps' FIELDS, in slot order @@ -249,8 +249,8 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T class FlowGraph(torch.utils.data.Dataset[Record]): """Named-step graph engine — executes a ``flow:`` document natively. - The readable twin of :class:`~sampleflux.core.Flux`: steps run in document order over - a per-sample environment of named results, with fan-out isolation (copy-on-read, move + The readable twin of :class:`~recordstream.core.Stream`: steps run in document order over + a per-record environment of named results, with fan-out isolation (copy-on-read, move on last read) and automatic cell lifetimes. Any FlowGraph converts to a flat op list for the serial engine (:func:`to_ops`) and back (:func:`from_ops`) — execution parity between the two is a pinned contract. @@ -259,7 +259,7 @@ class FlowGraph(torch.utils.data.Dataset[Record]): source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. outputs: Name of the step whose result is yielded. Blank (default) = the last step. - chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single samples. + chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single records. """ def __init__( @@ -318,16 +318,16 @@ def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": @classmethod def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": """Lift a flat ``{ops: [...]}`` YAML document into a FlowGraph (via :func:`from_ops`).""" - from sampleflux.core import Flux + from recordstream.core import Stream - flux = Flux.from_ops_yaml(path, source=source) - flow_doc, outputs = from_ops(flux.ops) + stream = Stream.from_ops_yaml(path, source=source) + flow_doc, outputs = from_ops(stream.ops) return cls(source=source, flow=flow_doc, outputs=outputs) # -- execution --------------------------------------------------------- def _run(self, seed: Any) -> Optional[Any]: - """Run one sample through the steps; ``None`` = filtered (an op returned None).""" + """Run one record through the steps; ``None`` = filtered (an op returned None).""" steps, outputs = self._ensure_parsed() readers = _result_readers(steps, outputs) env: Dict[str, Any] = {} @@ -346,22 +346,22 @@ def read_result(name: str, *, copy: bool) -> Any: prev: Optional[str] = None for step in steps: - # 1. the input sample (implicit stream reads move; explicit fan-out reads copy) + # 1. the input record (implicit stream reads move; explicit fan-out reads copy) if step.from_ is not None: - sample = read_result(step.from_, copy=True) + record = read_result(step.from_, copy=True) elif prev is not None: - sample = read_result(prev, copy=False) + record = read_result(prev, copy=False) else: - sample = seed + record = seed # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) if step.merge_from: - if not isinstance(sample, dict): + if not isinstance(record, dict): raise TypeError( f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " - f"{type(sample).__name__} — expected a record dict." + f"{type(record).__name__} — expected a record dict." ) - merged = dict(sample) + merged = dict(record) for ref in step.merge_from: value = read_result(ref, copy=True) if not isinstance(value, dict): @@ -370,16 +370,16 @@ def read_result(name: str, *, copy: bool) -> Any: f"{type(value).__name__}, expected a record" ) merged.update(value) - sample = merged + record = merged - # 3. per-sample parameter binds + # 3. per-record parameter binds if step.op is not None: op = step.op if getattr(op, "EXPANDS", False): raise NotImplementedError( f"flow step {step.name!r}: {type(op).__name__!r} is a 1→N expanding op — " "FlowGraph steps are strictly 1→1 (a named-step env has one result per step). " - "Run expanding pipelines through the Flux engine (iterable-only)." + "Run expanding pipelines through the Stream engine (iterable-only)." ) for param, ref in step.bind.items(): parsed = _split_bind_ref(ref) @@ -397,12 +397,12 @@ def read_result(name: str, *, copy: bool) -> Any: # "step[key]" = the named entry; bare "step" = the whole record. value = value[parsed.key] setattr(op, param, value) - result = _apply_op(sample, op) + result = _apply_op(record, op) if result is None: return None - sample = result + record = result - env[step.name] = sample + env[step.name] = record prev = step.name return cast(Optional[Record], env.get(outputs)) if outputs in env else None @@ -410,11 +410,11 @@ def read_result(name: str, *, copy: bool) -> Any: def __iter__(self) -> Iterator[Any]: if self.source is None: return - it = self._iter_samples() + it = self._iter_records() if self._chunk_size > 0: batch: List[Record] = [] - for sample in it: - batch.append(sample) + for record in it: + batch.append(record) if len(batch) == self._chunk_size: yield batch batch = [] @@ -423,7 +423,7 @@ def __iter__(self) -> Iterator[Any]: else: yield from it - def _iter_samples(self) -> Iterator[Record]: + def _iter_records(self) -> Iterator[Record]: if self._workers > 1: yield from self._iter_parallel() return @@ -435,11 +435,11 @@ def _iter_samples(self) -> Iterator[Record]: def _iter_parallel(self) -> Iterator[Record]: """Multiprocess execution — delegates to the serial engine over the LOWERED op list.""" - from sampleflux.core import Flux + from recordstream.core import Stream assert self.source is not None - flux = Flux(source=self.source, ops=to_ops(self.steps, self.output_step)).parallel(self._workers) - yield from flux + stream = Stream(source=self.source, ops=to_ops(self.steps, self.output_step)).parallel(self._workers) + yield from stream def __len__(self) -> int: from collections.abc import Sized @@ -460,7 +460,7 @@ def __getitem__(self, index: int) -> Any: ) result = self._run(raw) if result is None: - raise IndexError(f"Sample {index} filtered out by the flow") + raise IndexError(f"Record {index} filtered out by the flow") return result def parallel(self, workers: int = 4) -> "FlowGraph": @@ -469,7 +469,7 @@ def parallel(self, workers: int = 4) -> "FlowGraph": return self def batch(self, chunk_size: int) -> "FlowGraph": - """Group yielded samples into lists of ``chunk_size``.""" + """Group yielded records into lists of ``chunk_size``.""" self._chunk_size = chunk_size return self @@ -477,11 +477,11 @@ def collect(self) -> List[Any]: """Materialize the full stream into a list.""" return list(self) - def to_flux(self) -> Any: - """The serial-engine twin: a Flux running the LOWERED flat op list (same results).""" - from sampleflux.core import Flux + def to_stream(self) -> Any: + """The serial-engine twin: a Stream running the LOWERED flat op list (same results).""" + from recordstream.core import Stream - return Flux(source=self.source, ops=to_ops(self.steps, self.output_step)) + return Stream(source=self.source, ops=to_ops(self.steps, self.output_step)) # --------------------------------------------------------------------------- @@ -492,7 +492,7 @@ def to_flux(self) -> Any: def to_ops(steps: Union[Sequence[FlowStep], Dict[str, Any]], outputs: str = "") -> List[Any]: """Lower a flow (parsed steps or a raw flow mapping) into a flat context-ops list. - The result runs on the plain serial :class:`~sampleflux.core.Flux` engine and is the + The result runs on the plain serial :class:`~recordstream.core.Stream` engine and is the serialization form a graph exporter's serial mode emits. Cell names are the step names (deterministic, diffable); liveness is compiled into ``drop`` flags so a well-formed graph leaves the Context empty. A purely linear flow lowers to the bare @@ -753,12 +753,12 @@ def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: return flow_map, out -def flow_yaml_to_flux(path: str, source: Optional[Any] = None) -> Any: - """Convenience: load a ``flow:`` YAML and return the SERIAL engine (lowered Flux).""" - from sampleflux.core import Flux +def flow_yaml_to_stream(path: str, source: Optional[Any] = None) -> Any: + """Convenience: load a ``flow:`` YAML and return the SERIAL engine (lowered Stream).""" + from recordstream.core import Stream doc = _confluid_resolve(path) if not isinstance(doc, dict) or "flow" not in doc: - raise ValueError(f"flow_yaml_to_flux: {path!r} has no 'flow:' mapping") + raise ValueError(f"flow_yaml_to_stream: {path!r} has no 'flow:' mapping") parsed, outputs = parse_flow(doc["flow"], str(doc.get("outputs", "") or "")) - return Flux(source=source, ops=to_ops(parsed, outputs)) + return Stream(source=source, ops=to_ops(parsed, outputs)) diff --git a/sampleflux/io.py b/recordstream/io.py similarity index 94% rename from sampleflux/io.py rename to recordstream/io.py index 8333ae6..b678583 100644 --- a/sampleflux/io.py +++ b/recordstream/io.py @@ -3,7 +3,7 @@ Storage backends never inspect item internals: they call :func:`encode_item` to get a flat :class:`EncodedItem` (registered type name + array payload + scalar attrs) and :func:`decode_item` to rebuild the item. The DEFAULT structural codec covers both item -shapes (an :class:`~sampleflux.items.NDArrayItem` subclass → the array + its declared +shapes (an :class:`~recordstream.items.NDArrayItem` subclass → the array + its declared attrs; a dataclass wrapper with a ``data`` field → the payload + the remaining fields), so an externally-registered item type — a domain package's signal item, a user type — serializes with ZERO storage-code changes. :func:`register_io` overrides the codec for types whose @@ -13,8 +13,8 @@ pseudo type tag ``"plain"`` and decodes back verbatim, so scalar metadata keys ride the same layout as typed values. -The registered TYPE NAME (via :func:`~sampleflux.items.register_item` / -:func:`~sampleflux.items.get_item_type`) is the on-disk type tag — decoding requires the +The registered TYPE NAME (via :func:`~recordstream.items.register_item` / +:func:`~recordstream.items.get_item_type`) is the on-disk type tag — decoding requires the item type to be registered (imported) in the reading process, exactly like the confluid ``!class:`` contract. """ @@ -24,7 +24,7 @@ from dataclasses import is_dataclass from typing import Any, Callable, Dict, Tuple, cast -from sampleflux.items import NDArrayItem, Record, get_item_type, is_item, item_data +from recordstream.items import NDArrayItem, Record, get_item_type, is_item, item_data __all__ = [ "EncodedItem", diff --git a/sampleflux/items.py b/recordstream/items.py similarity index 97% rename from sampleflux/items.py rename to recordstream/items.py index d01ec0a..7fed0e2 100644 --- a/sampleflux/items.py +++ b/recordstream/items.py @@ -1,6 +1,6 @@ """Typed values — the vocabulary a record is made of, each value OWNING its metadata. -A sample is a plain ``dict`` (the :data:`Record` alias) whose values are TYPED: an +A record is a plain ``dict`` (the :data:`Record` alias) whose values are TYPED: an :class:`Image` carries its ``layout``, a :class:`Label` its ``classes``, a :class:`Regions` its ``canvas`` reference frame. Ops dispatch on these types (the torchvision-v2 ``tv_tensors`` idea) — there is no wrapper container and no role tags; @@ -27,7 +27,7 @@ a transform kernel never has to special-case "is this a subclass or a wrapper". Extensibility: any type decorated with :func:`register_item` becomes a first-class item — -the dispatch registry (:mod:`sampleflux.dispatch`) and a visual editor's socket-type map can +the dispatch registry (:mod:`recordstream.dispatch`) and a visual editor's socket-type map can see it. A downstream package (a signal item, a user type) adds one class + one decorator, no core edit. @@ -45,7 +45,7 @@ _ItemT = TypeVar("_ItemT") -#: A sample record — a PLAIN dict of typed values. There is deliberately no container +#: A record record — a PLAIN dict of typed values. There is deliberately no container #: class: ops receive and return ordinary dicts, so library transforms that already #: understand dicts (torchvision v2) or named kwargs (albumentations) run as-is. Record = Dict[str, Any] diff --git a/sampleflux/labels.py b/recordstream/labels.py similarity index 89% rename from sampleflux/labels.py rename to recordstream/labels.py index bcfd4ee..29076c8 100644 --- a/sampleflux/labels.py +++ b/recordstream/labels.py @@ -1,7 +1,7 @@ """``LabelMap`` — a bidirectional class-name ↔ integer-id map. -The *fittable* companion to the config-pinned :class:`~sampleflux.ops.target.EncodeTarget` / -:class:`~sampleflux.ops.target.DecodeTarget`. Those ops carry an explicit ``mapping`` that is +The *fittable* companion to the config-pinned :class:`~recordstream.ops.target.EncodeTarget` / +:class:`~recordstream.ops.target.DecodeTarget`. Those ops carry an explicit ``mapping`` that is **pinned in config, NOT fitted** at run time, so train / eval / predict share one identical label→id ordering. :class:`LabelMap` is the piece that *produces* such a pinned mapping: @@ -9,7 +9,7 @@ (backed by scikit-learn's ``LabelEncoder``) — the one-time fit that happens at **train** time. * :meth:`LabelMap.save` / :meth:`LabelMap.load` persist it (in marainer's ``class_names.json`` format) so **eval / predict** reload the *same* mapping rather than refitting on a subset. -* :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the sampleflux ops that apply it. +* :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the recordstream ops that apply it. So fitting happens once, then the mapping is pinned/persisted — it does NOT contradict the "mapping pinned in config, not fitted" discipline of the ops; it is how the pin gets created. @@ -17,7 +17,7 @@ Zero-arg constructible (``LabelMap()`` succeeds with an empty mapping) and side-effect-free in ``__init__`` per the workspace "Lazy Initialization & Zero-Arg Construction" convention; the non-empty requirement is validated lazily in the properties, not in the constructor. scikit-learn -is imported lazily inside :meth:`fit` so importing sampleflux never pulls it in. +is imported lazily inside :meth:`fit` so importing recordstream never pulls it in. """ import json @@ -26,7 +26,7 @@ from confluid import configurable -from sampleflux.ops.target import DecodeTarget, EncodeTarget +from recordstream.ops.target import DecodeTarget, EncodeTarget @configurable @@ -35,7 +35,7 @@ class LabelMap: Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`label_names`, builds the - :class:`~sampleflux.ops.target.EncodeTarget` / :class:`~sampleflux.ops.target.DecodeTarget` + :class:`~recordstream.ops.target.EncodeTarget` / :class:`~recordstream.ops.target.DecodeTarget` that apply it, and round-trips to disk in marainer's ``class_names.json`` format. Args: @@ -74,11 +74,11 @@ def inverse(self) -> Dict[int, str]: return {v: k for k, v in self._require().items()} def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTarget: - """Return an :class:`~sampleflux.ops.target.EncodeTarget` transform that maps name → id via this map.""" + """Return an :class:`~recordstream.ops.target.EncodeTarget` transform that maps name → id via this map.""" return EncodeTarget(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTarget: - """Return a :class:`~sampleflux.ops.target.DecodeTarget` transform that maps id → name via this map.""" + """Return a :class:`~recordstream.ops.target.DecodeTarget` transform that maps id → name via this map.""" return DecodeTarget(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) @classmethod diff --git a/recordstream/ops/__init__.py b/recordstream/ops/__init__.py new file mode 100644 index 0000000..3ba1d9f --- /dev/null +++ b/recordstream/ops/__init__.py @@ -0,0 +1,66 @@ +""" +RecordStream operations (record-dict ops). + +Submodules: + - recordstream.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / + connected_component_bboxes / resolve_expression helpers) + - recordstream.ops.torch: ToTensor (+ to_tensor helper) + - recordstream.ops.image: ConvertToImage (+ value_to_image / normalize_to_uint8 …) + - recordstream.ops.target: EncodeTarget, DecodeTarget, + CocoToTorchVisionDetection, MasksToDetectionBoxes + - recordstream.ops.structure: RenameField, DropField, CopyField, SelectFields + - recordstream.ops.parallel: Parallel (worker-pool sub-pipeline) + - recordstream.ops.enable: Enable (toggle an op-list via one named CLI flag) + - recordstream.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) + - recordstream.ops.configure: ConfigureOp (per-record parameter injection) + - recordstream.ops.formula: FormulaOp (math formula over one record entry) + - recordstream.ops.sink: RecordSinkOp (adapt a DataSink as a pass-through op) + - recordstream.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-record + Context graph plane — the flat-list building blocks a branchy flow: document lowers to) + - recordstream.ops.debug: PrintRecordOp (per-record summary probe) + +The sequential composer ``Pipeline`` lives in :mod:`recordstream.transform` (package-root +export) — one list mixing native ops with bare albumentations / torchvision-v2 transforms. +""" + +from recordstream.ops.configure import ConfigureOp +from recordstream.ops.context import Apply, Capture, Drop, MergeFields, Save, Use +from recordstream.ops.debug import PrintRecordOp +from recordstream.ops.enable import Enable +from recordstream.ops.formula import FormulaOp +from recordstream.ops.image import ConvertToImage +from recordstream.ops.numpy import ConnectedComponents, Threshold +from recordstream.ops.parallel import Parallel +from recordstream.ops.random_apply import RandomApply +from recordstream.ops.sink import RecordSinkOp +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes +from recordstream.ops.torch import ToTensor + +__all__ = [ + "Apply", + "Capture", + "CocoToTorchVisionDetection", + "ConfigureOp", + "ConnectedComponents", + "ConvertToImage", + "CopyField", + "DecodeTarget", + "Drop", + "DropField", + "Enable", + "EncodeTarget", + "FormulaOp", + "MasksToDetectionBoxes", + "MergeFields", + "Parallel", + "PrintRecordOp", + "RandomApply", + "RenameField", + "Save", + "RecordSinkOp", + "SelectFields", + "Threshold", + "ToTensor", + "Use", +] diff --git a/sampleflux/ops/configure.py b/recordstream/ops/configure.py similarity index 91% rename from sampleflux/ops/configure.py rename to recordstream/ops/configure.py index 1100cbf..2a93ace 100644 --- a/sampleflux/ops/configure.py +++ b/recordstream/ops/configure.py @@ -7,7 +7,7 @@ ``target`` to the ORIGINAL record. Modality-neutral — it threads any record through any ops — so it lives in core -sampleflux (compose group, alongside ``Pipeline`` / ``Enable`` / ``RandomApply``). +recordstream (compose group, alongside ``Pipeline`` / ``Enable`` / ``RandomApply``). """ from typing import Any, List, Optional, cast @@ -15,7 +15,7 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.items import Record, item_data +from recordstream.items import Record, item_data @configurable(category="op", group="compose") @@ -35,11 +35,11 @@ class ConfigureOp: .. code-block:: yaml - - !class:sampleflux.ops.configure.ConfigureOp + - !class:recordstream.ops.configure.ConfigureOp ops: - - !class:sampleflux.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} + - !class:recordstream.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} source: image - target: !class:sampleflux.ops.numpy.Threshold + target: !class:recordstream.ops.numpy.Threshold low_op: ">=" param: low_level @@ -74,7 +74,7 @@ def __call__(self, record: Record) -> Optional[Record]: self.target = flow(self.target) # _apply_op = the engine's op-family dispatch, so bare library transforms # work in the compute chain and as target exactly as in a bare ops list. - from sampleflux.core import _apply_op + from recordstream.core import _apply_op current: Record = record for i, op in enumerate(self.ops): diff --git a/sampleflux/ops/context.py b/recordstream/ops/context.py similarity index 94% rename from sampleflux/ops/context.py rename to recordstream/ops/context.py index d3eebc3..88e8308 100644 --- a/sampleflux/ops/context.py +++ b/recordstream/ops/context.py @@ -1,10 +1,10 @@ -"""Context ops — move data between the per-record :class:`~sampleflux.context.Context` and the stream. +"""Context ops — move data between the per-record :class:`~recordstream.context.Context` and the stream. The six flat-list building blocks of graph-shaped pipelines: ``Save`` (fork snapshot), ``Use`` (branch start), ``Drop`` (cell hygiene), ``Apply`` (per-record parameter from a cell), ``Capture`` (an op's ``@output`` into a cell), and ``MergeFields`` (fan-in). A branchy canvas graph or ``flow:`` document lowers to a plain sequential op list containing -these (``sampleflux.flow.to_ops``), executable by the ordinary ``Flux`` engine — and lifts +these (``recordstream.flow.to_ops``), executable by the ordinary ``Stream`` engine — and lifts back (``from_ops``). Graph wiring lives on the engine-created Context data plane, so the record itself stays @@ -19,8 +19,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.context import require -from sampleflux.items import Record +from recordstream.context import require +from recordstream.items import Record _MISSING = object() @@ -100,7 +100,7 @@ def __init__(self, name: str = "", drop: bool = False) -> None: self.name = str(name) self.drop = bool(drop) - def __call__(self, sample: Any) -> Any: + def __call__(self, record: Any) -> Any: if not self.name: raise ValueError("Use: 'name' (the context cell to read) is required") ctx = require("Use") @@ -172,7 +172,7 @@ def __init__( self.key = str(key) self.drop = bool(drop) - def __call__(self, sample: Any) -> Optional[Any]: + def __call__(self, record: Any) -> Optional[Any]: if self.op is None: raise ValueError("Apply: an 'op' to configure and apply is required") if not self.param: @@ -189,9 +189,9 @@ def __call__(self, sample: Any) -> Optional[Any]: setattr(op, self.param, value) # _apply_op = the engine's op-family dispatch, so a bare library transform wired as # the wrapped op applies exactly as in a bare ops list. - from sampleflux.core import _apply_op + from recordstream.core import _apply_op - return _apply_op(sample, op) + return _apply_op(record, op) def close(self) -> None: """Propagate close() to the wrapped op if it owns resources.""" @@ -207,7 +207,7 @@ class Capture: Records a wrapped op's live ``@output``: the wrapped op runs once (stochastic-correct — the value is read from the actual run, never recomputed) and each requested ``@output`` is stored as a raw cell value for a later ``Apply``/``Mix`` to read. The - returned sample is ``op(sample)`` — transformations are kept. + returned record is ``op(record)`` — transformations are kept. Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call, so a ``Capture()`` built from YAML costs nothing. @@ -249,7 +249,7 @@ def __call__(self, record: Record) -> Optional[Record]: ctx = require("Capture") op = cast(Any, self.op) # _apply_op = the engine's op-family dispatch (bare library transforms capture too). - from sampleflux.core import _apply_op + from recordstream.core import _apply_op result = _apply_op(record, op) if result is None: @@ -275,7 +275,7 @@ class MergeFields: Each source cell (a record saved by an earlier branch) contributes its ENTRIES, united in listed order with last-write-wins on a key collision (the deterministic slot-order rule; avoid a deliberate collision by renaming on the producing branch — - ``sampleflux.ops.structure.RenameField``). ``keys`` selects a subset of a source's + ``recordstream.ops.structure.RenameField``). ``keys`` selects a subset of a source's entries before the union. Args: diff --git a/sampleflux/ops/debug.py b/recordstream/ops/debug.py similarity index 96% rename from sampleflux/ops/debug.py rename to recordstream/ops/debug.py index 723b71d..9aed78c 100644 --- a/sampleflux/ops/debug.py +++ b/recordstream/ops/debug.py @@ -5,11 +5,11 @@ from confluid import configurable from loggair import get_logger -from sampleflux.items import Record, is_item, item_data +from recordstream.items import Record, is_item, item_data logger = get_logger(__name__) -# Per-sample output is DIAGNOSTIC, so the logger level is restricted to trace/debug (the workspace +# Per-record output is DIAGNOSTIC, so the logger level is restricted to trace/debug (the workspace # "Diagnostic Log Levels" mandate — never info/warning for per-iteration events). Console visibility # comes from ``to_console`` (a plain ``print``), independent of the log level. LogLevel = Literal["trace", "debug"] @@ -55,7 +55,7 @@ def _summarize_metadata(metadata: Any) -> str: @configurable(category="op", group="debug") -class PrintSampleOp: +class PrintRecordOp: """Log / print a summary of each record passing through (a pass-through op). A pipeline probe: emits a compact description of the record — each typed value's @@ -78,7 +78,7 @@ class PrintSampleOp: def __init__( self, - label: str = "sample", + label: str = "record", level: LogLevel = "debug", include_data: bool = True, include_metadata: bool = True, diff --git a/sampleflux/ops/enable.py b/recordstream/ops/enable.py similarity index 86% rename from sampleflux/ops/enable.py rename to recordstream/ops/enable.py index f7e96a1..29a6488 100644 --- a/sampleflux/ops/enable.py +++ b/recordstream/ops/enable.py @@ -3,7 +3,7 @@ A compose-group op (alongside ``Pipeline`` / ``Parallel``): wrap an inner op-list so the whole chain can be switched on or off from one boolean attribute whose name becomes the CLI flag. Modality-neutral — it threads any record -through any ops — so it lives in core sampleflux, not a domain package. +through any ops — so it lives in core recordstream, not a domain package. """ from typing import List, Optional, Tuple @@ -11,7 +11,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.items import Record +from recordstream.items import Record logger = get_logger(__name__) @@ -22,7 +22,7 @@ class Enable: ``ops`` is a list; even a single-op guard uses ``ops: [op]``. The wrapper threads each record through every op in sequence — same semantics as - listing them inline in ``Flux.ops`` — so a whole visualization chain + listing them inline in ``Stream.ops`` — so a whole visualization chain shares one toggle instead of needing a wrapper per op. The toggle flag is supplied in YAML as an *extra* kwarg whose name becomes @@ -35,10 +35,10 @@ class Enable: .. code-block:: yaml - - !class:sampleflux.ops.enable.Enable + - !class:recordstream.ops.enable.Enable visualize: false # ← any boolean attribute name works; this name IS the CLI flag ops: - - !class:sampleflux.ops.image.ConvertToImage {} + - !class:recordstream.ops.image.ConvertToImage {} - !class:waivefront.visualizers.SaveImage output_dir: ./segments_png @@ -46,9 +46,9 @@ class Enable: .. code-block:: bash - sampleflux run pipeline.yaml --visualize true - sampleflux run pipeline.yaml --visualize+ # polarity shorthand → True - sampleflux run pipeline.yaml --visualize- # polarity shorthand → False + recordstream run pipeline.yaml --visualize true + recordstream run pipeline.yaml --visualize+ # polarity shorthand → True + recordstream run pipeline.yaml --visualize- # polarity shorthand → False Inner ops stay deferred (not materialized) until the wrapper actually fires for the first time, so guarding expensive-to-construct ops with @@ -65,11 +65,11 @@ class Enable: .. code-block:: yaml - - !class:sampleflux.ops.enable.Enable + - !class:recordstream.ops.enable.Enable name: overlay # dotted-override key enable: false # generic toggle — the name scopes it ops: [render-with-overlays, save-to ./debug_png] - - !class:sampleflux.ops.enable.Enable + - !class:recordstream.ops.enable.Enable name: labelstudio enable: false ops: [render-clean, save-to ./ls_png] @@ -79,18 +79,18 @@ class Enable: .. code-block:: bash # Targeted — only the overlay chain fires. - sampleflux run pipeline.yaml --overlay.enable true - sampleflux run pipeline.yaml --overlay.enable+ # polarity shorthand → True + recordstream run pipeline.yaml --overlay.enable true + recordstream run pipeline.yaml --overlay.enable+ # polarity shorthand → True # Broadcast — every Fluid with an `enable` kwarg flips. - sampleflux run pipeline.yaml --enable true + recordstream run pipeline.yaml --enable true ``name`` is a plain string on the instance; Confluid's post-construction paradigm setattr's it automatically from YAML with no ctor change. Constraints: * ``ops`` is required and must be a non-empty list — validated **lazily** - on first call (zero-arg construction stays valid per the sampleflux + on first call (zero-arg construction stays valid per the recordstream "Lazy Initialization & Zero-Arg Construction" convention). * Exactly one boolean attribute (other than ``ops`` / ``name`` and dunders) may be set on the wrapper — that's the toggle. @@ -143,7 +143,7 @@ def __call__(self, record: Record) -> Optional[Record]: # _apply_op = the engine's op-family dispatch, so bare library transforms # run under the toggle exactly as in a bare ops list. - from sampleflux.core import _apply_op + from recordstream.core import _apply_op current: Optional[Record] = record for i, op in enumerate(self.ops): diff --git a/sampleflux/ops/formula.py b/recordstream/ops/formula.py similarity index 98% rename from sampleflux/ops/formula.py rename to recordstream/ops/formula.py index cb7857a..ab9f31c 100644 --- a/sampleflux/ops/formula.py +++ b/recordstream/ops/formula.py @@ -15,7 +15,7 @@ import numpy as _np from confluid import configurable -from sampleflux.items import Record, item_data, with_data +from recordstream.items import Record, item_data, with_data # Every public ``math`` symbol + the scalar built-in helpers, mirroring the canvas Math # node's namespace. The bound variable shadows same-named constants (e.g. ``e``). diff --git a/sampleflux/ops/image.py b/recordstream/ops/image.py similarity index 97% rename from sampleflux/ops/image.py rename to recordstream/ops/image.py index eb81297..b198200 100644 --- a/sampleflux/ops/image.py +++ b/recordstream/ops/image.py @@ -1,9 +1,9 @@ -"""Generic, modality-agnostic image conversion for SampleFlux pipelines. +"""Generic, modality-agnostic image conversion for RecordStream pipelines. This is the single home for "turn an arbitrary value into an image": the :class:`ConvertToImage` op plus the library functions -(:func:`value_to_image` / :func:`sample_to_image`) that back it and the GUI -sample preview. It lives in sampleflux (not waivefront) because the conversion is +(:func:`value_to_image` / :func:`record_to_image`) that back it and the GUI +record preview. It lives in recordstream (not waivefront) because the conversion is fully generic — a 2-D map, a CHW tensor, a PIL image, a boolean mask all render the same way regardless of domain — so every project (waivefront's spectrogram render, any image dataset preview, GUI viewer nodes) reuses ONE implementation. @@ -26,11 +26,11 @@ from loggair import get_logger from PIL import Image, ImageDraw -from sampleflux.items import Image as ImageItem -from sampleflux.items import NDArrayItem, Record, item_data -from sampleflux.transform import Transform +from recordstream.items import Image as ImageItem +from recordstream.items import NDArrayItem, Record, item_data +from recordstream.transform import Transform -logger = get_logger("sampleflux.ops.image") +logger = get_logger("recordstream.ops.image") def normalize_to_uint8( @@ -57,7 +57,7 @@ def normalize_to_uint8( # Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob -# across the workspace (``value_to_image`` / ``sample_to_image`` / ``ConvertToImage`` and, via +# across the workspace (``value_to_image`` / ``record_to_image`` / ``ConvertToImage`` and, via # re-export, waivefront's renderers) AND for GUI colormap dropdowns (which read ``COLORMAPS``). # A closed ``Literal`` (never a bare ``str``) makes the choice self-documenting and machine- # introspectable: visual-editor palettes, navigaitor's form-spec, and MCP tool schemas enumerate the @@ -170,8 +170,8 @@ def _bound_longest_side(rgb: np.ndarray, max_size: int) -> np.ndarray: def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: """Render an arbitrary value (any record entry) to an ``(H, W, 3)`` uint8 RGB image. - A generic, modality-agnostic preview usable from any SampleFlux pipeline (and - by a GUI sample extractor, which renders the selected field). Handles: + A generic, modality-agnostic preview usable from any RecordStream pipeline (and + by a GUI record extractor, which renders the selected field). Handles: * ``PIL.Image`` — converted to RGB; * ``torch.Tensor`` — detached to numpy (CHW collapsed to HWC below); @@ -193,7 +193,7 @@ def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 5 return _bound_longest_side(_render_rgb(value, colormap), max_size) -def sample_to_image(record: Record, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: +def record_to_image(record: Record, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: """Render a record's first array-bearing value to an ``(H, W, 3)`` uint8 RGB image for display. Thin wrapper over :func:`value_to_image` (which does the modality-agnostic rendering) @@ -209,7 +209,7 @@ def sample_to_image(record: Record, colormap: Colormap = "viridis", max_size: in arr = _coerce_to_ndarray(item_data(value)) if arr is not None and arr.ndim in (2, 3): return value_to_image(item_data(value), colormap=colormap, max_size=max_size) - raise ValueError(f"sample_to_image: no array-bearing value in record (keys: {list(record)})") + raise ValueError(f"record_to_image: no array-bearing value in record (keys: {list(record)})") # --------------------------------------------------------------------------- # @@ -221,7 +221,7 @@ def sample_to_image(record: Record, colormap: Colormap = "viridis", max_size: in # values. Pure functions (NOT @configurable ops): they measure/derive, they don't # transform a record, so they're library helpers like value_to_image — not canvas # nodes. They live here (not in the GUI node) so the computation is reusable -# and unit-tested, per the workspace "rendering/analysis lives in sampleflux" mandate. +# and unit-tested, per the workspace "rendering/analysis lives in recordstream" mandate. # --------------------------------------------------------------------------- # @@ -602,7 +602,7 @@ class ConvertToImage(Transform): """An array-bearing field → an ``Image`` item. Reads an array-bearing field from the record and writes a fresh - :class:`~sampleflux.Image` item (HWC ``uint8`` RGB) under ``output`` (it is the + :class:`~recordstream.Image` item (HWC ``uint8`` RGB) under ``output`` (it is the pipeline's working image). Any other field passes through untouched. Rendering is byte-identical to the legacy op — it reuses the SAME @@ -682,7 +682,7 @@ def __call__(self, record: Record) -> Record: "ConvertToImage", "normalize_to_uint8", "value_to_image", - "sample_to_image", + "record_to_image", "select_channel", "channel_count", "array_histogram", diff --git a/sampleflux/ops/numpy.py b/recordstream/ops/numpy.py similarity index 94% rename from sampleflux/ops/numpy.py rename to recordstream/ops/numpy.py index fc5c51d..d146353 100644 --- a/sampleflux/ops/numpy.py +++ b/recordstream/ops/numpy.py @@ -7,8 +7,8 @@ from confluid import configurable from loggair import get_logger -from sampleflux.items import Mask, NDArrayItem, Record, Regions, item_data -from sampleflux.transform import Transform +from recordstream.items import Mask, NDArrayItem, Record, Regions, item_data +from recordstream.transform import Transform logger = get_logger(__name__) @@ -126,7 +126,7 @@ class Threshold(Transform): Reads the array at ``field`` (blank = the first array-bearing item in the record) and thresholds it into a boolean mask with the bound / comparison / expression math (:func:`threshold_array`), - writing a :class:`~sampleflux.Mask` item under ``output`` (a threshold mask is an + writing a :class:`~recordstream.Mask` item under ``output`` (a threshold mask is an intermediate that a later op — e.g. :class:`ConnectedComponents` — consumes). Any other key passes through untouched. @@ -194,8 +194,8 @@ def connected_component_bboxes( Components smaller than ``min_area_bins`` are dropped. ``connectivity`` is ``4`` (orthogonal neighbors) or ``8`` (orthogonal + diagonal). Shared by :class:`ConnectedComponents` - AND :func:`sampleflux.ops.target.masks_to_detection` (its ``connected=True`` mode). Requires - ``scipy`` (``pip install sampleflux[vision]``). + AND :func:`recordstream.ops.target.masks_to_detection` (its ``connected=True`` mode). Requires + ``scipy`` (``pip install recordstream[vision]``). """ if min_area_bins < 1: raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") @@ -206,7 +206,7 @@ def connected_component_bboxes( except ImportError as exc: raise ImportError( "connected-components labeling requires scipy. " - "Install with `pip install sampleflux[vision]` or add scipy to your environment." + "Install with `pip install recordstream[vision]` or add scipy to your environment." ) from exc structure = generate_binary_structure(2, 1 if connectivity == 4 else 2) @@ -235,14 +235,14 @@ def connected_component_bboxes( class ConnectedComponents(Transform): """A boolean ``Mask`` → a ``Regions`` item. - Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the + Reads the :class:`~recordstream.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the first array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into ``(row_min, row_max, col_min, col_max)`` inclusive bin-box tuples via - :func:`connected_component_bboxes`, writing them as a :class:`~sampleflux.Regions` item under + :func:`connected_component_bboxes`, writing them as a :class:`~recordstream.Regions` item under ``output`` (RAW detections, not model predictions). Any other key passes through. Components smaller than ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or - 8-neighborhood. Requires ``scipy`` (``pip install sampleflux[vision]``). + 8-neighborhood. Requires ``scipy`` (``pip install recordstream[vision]``). Args: min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). diff --git a/sampleflux/ops/parallel.py b/recordstream/ops/parallel.py similarity index 84% rename from sampleflux/ops/parallel.py rename to recordstream/ops/parallel.py index 9ca67fb..bdc7092 100644 --- a/sampleflux/ops/parallel.py +++ b/recordstream/ops/parallel.py @@ -1,12 +1,12 @@ """``Parallel`` — explicit parallel sub-pipeline op. -Place inside a :class:`~sampleflux.core.Flux`'s ops list to dispatch each -upstream sample through an inner sub-pipeline (``self.ops``) in a +Place inside a :class:`~recordstream.core.Stream`'s ops list to dispatch each +upstream record through an inner sub-pipeline (``self.ops``) in a spawn-context worker pool. Bounded prefetch caps outstanding work so the executor queue can't grow unboundedly with source length. Falls back to inline sequential application when invoked as a regular -per-sample op (e.g. via :meth:`Flux.__getitem__`) so random access remains +per-record op (e.g. via :meth:`Stream.__getitem__`) so random access remains correct. Note: @@ -25,8 +25,8 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from sampleflux.core import _worker_task -from sampleflux.items import Record +from recordstream.core import _worker_task +from recordstream.items import Record @configurable(category="op", group="compose") @@ -51,10 +51,10 @@ def _materialize_ops(self) -> None: self.ops[i] = flow(op) def __call__(self, record: Record) -> Optional[Record]: - # Inline fallback for non-streaming callers (e.g. Flux.__getitem__). Routed through + # Inline fallback for non-streaming callers (e.g. Stream.__getitem__). Routed through # _apply_op — the same op-family dispatch the streamed route's _worker_task uses — # so bare library transforms behave identically. - from sampleflux.core import _apply_op + from recordstream.core import _apply_op self._materialize_ops() current: Optional[Record] = record @@ -64,7 +64,7 @@ def __call__(self, record: Record) -> Optional[Record]: current = _apply_op(current, op) return current - def stream(self, samples: Iterable[Optional[Record]]) -> Iterator[Optional[Record]]: + def stream(self, records: Iterable[Optional[Record]]) -> Iterator[Optional[Record]]: """Stream-level dispatch with bounded prefetch (in-order yield).""" if self.workers < 1: raise ValueError(f"Parallel(workers={self.workers!r}): must be >= 1") @@ -72,12 +72,12 @@ def stream(self, samples: Iterable[Optional[Record]]) -> Iterator[Optional[Recor ctx = multiprocessing.get_context("spawn") limit = max(2 * self.workers, self.workers + 1) - from sampleflux.core import _extra_op_families + from recordstream.core import _extra_op_families with concurrent.futures.ProcessPoolExecutor(max_workers=self.workers, mp_context=ctx) as executor: pending: "deque[concurrent.futures.Future[Optional[Record]]]" = deque() extra_families = _extra_op_families() # ship third-party op families to the workers - for s in samples: + for s in records: if s is None: continue pending.append(executor.submit(_worker_task, s, self.ops, extra_families)) diff --git a/sampleflux/ops/random_apply.py b/recordstream/ops/random_apply.py similarity index 93% rename from sampleflux/ops/random_apply.py rename to recordstream/ops/random_apply.py index 325f5fc..822bb0c 100644 --- a/sampleflux/ops/random_apply.py +++ b/recordstream/ops/random_apply.py @@ -5,7 +5,7 @@ fraction of the time. Records that are skipped pass through unchanged. Modality-neutral — it threads any record through any op — so it lives -in core sampleflux, not a domain package. +in core recordstream, not a domain package. """ import random @@ -14,7 +14,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.items import Record +from recordstream.items import Record logger = get_logger(__name__) @@ -34,7 +34,7 @@ class RandomApply: .. code-block:: yaml - - !class:sampleflux.ops.random_apply.RandomApply + - !class:recordstream.ops.random_apply.RandomApply probability: 0.5 op: !class:albumentations.HorizontalFlip {p: 1.0} @@ -70,7 +70,7 @@ def __call__(self, record: Record) -> Optional[Record]: # _apply_op is the engine's op-family dispatch — routing through it (instead of # op(record)) lets a bare albumentations / torchvision-v2 transform nest inside # the gate exactly as it would sit in a bare ops list. - from sampleflux.core import _apply_op + from recordstream.core import _apply_op op = flow(self.op) if isinstance(self.op, Fluid) else self.op self.op = op # cache the flowed op so we only flow once diff --git a/sampleflux/ops/sink.py b/recordstream/ops/sink.py similarity index 79% rename from sampleflux/ops/sink.py rename to recordstream/ops/sink.py index d0cc454..20790fb 100644 --- a/sampleflux/ops/sink.py +++ b/recordstream/ops/sink.py @@ -1,10 +1,10 @@ -"""``RecordSinkOp`` — adapt a :class:`~sampleflux.storage.base.DataSink` as a pass-through op. +"""``RecordSinkOp`` — adapt a :class:`~recordstream.storage.base.DataSink` as a pass-through op. Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, a domain package's JSON sinks …) slot into a record-based op chain: on first call it opens the sink, every call writes the record and returns it unchanged, and ``close()`` flushes + closes. Modality-neutral (duck-typed ``open``/``write``/``close``), -so it lives in core sampleflux. +so it lives in core recordstream. """ from typing import Any @@ -12,18 +12,18 @@ from confluid import configurable from loggair import get_logger -from sampleflux.items import Record +from recordstream.items import Record logger = get_logger(__name__) @configurable(category="op", group="sink") class RecordSinkOp: - """Adapter: wrap a :class:`sampleflux.storage.base.DataSink` as a pass-through op. + """Adapter: wrap a :class:`recordstream.storage.base.DataSink` as a pass-through op. Sinks implement the ``open()`` / ``write(record)`` / ``close()`` protocol and - are normally attached to a :class:`sampleflux.processing.DatasetProcessor` as - the flux's terminal sink. This adapter lets the same sinks slot into any + are normally attached to a :class:`recordstream.processing.DatasetProcessor` as + the stream's terminal sink. This adapter lets the same sinks slot into any record-based op chain (e.g. persisting a prediction pipeline's outputs mid-chain). @@ -34,8 +34,8 @@ class RecordSinkOp: YAML:: - - !class:sampleflux.ops.sink.RecordSinkOp - sink: !class:sampleflux.storage.hdf5.HDF5Sink + - !class:recordstream.ops.sink.RecordSinkOp + sink: !class:recordstream.storage.hdf5.HDF5Sink path: ./records.h5 Args: diff --git a/sampleflux/ops/structure.py b/recordstream/ops/structure.py similarity index 99% rename from sampleflux/ops/structure.py rename to recordstream/ops/structure.py index 2cc8f48..2ee8217 100644 --- a/sampleflux/ops/structure.py +++ b/recordstream/ops/structure.py @@ -11,7 +11,7 @@ from confluid import configurable -from sampleflux.items import Record +from recordstream.items import Record __all__ = ["RenameField", "DropField", "CopyField", "SelectFields"] diff --git a/sampleflux/ops/target.py b/recordstream/ops/target.py similarity index 94% rename from sampleflux/ops/target.py rename to recordstream/ops/target.py index 5496780..3363ffd 100644 --- a/sampleflux/ops/target.py +++ b/recordstream/ops/target.py @@ -6,7 +6,7 @@ train / eval / predict share one identical ordering. * :class:`CocoToTorchVisionDetection` turns a HuggingFace / COCO ``objects`` annotation (``{bbox, category}``) into a torchvision detection target rendered as a - :class:`~sampleflux.Regions` item. + :class:`~recordstream.Regions` item. * :class:`MasksToDetectionBoxes` derives detection boxes from a segmentation ``Mask``. The detection conversions are the modality-neutral, image-detection counterparts of @@ -19,8 +19,8 @@ import numpy as np from confluid import configurable -from sampleflux.items import Label, Mask, Record, Regions, item_data -from sampleflux.transform import Transform +from recordstream.items import Label, Mask, Record, Regions, item_data +from recordstream.transform import Transform #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo #: fails at the call site and UIs / form-specs enumerate the choices. @@ -36,10 +36,10 @@ def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: return mapping[value] if ignore_unknown: return default - sample_keys = list(mapping)[:8] + record_keys = list(mapping)[:8] suffix = "..." if len(mapping) > 8 else "" raise KeyError( - f"{op_name}: value {value!r} not in mapping (keys: {sample_keys}{suffix}). " + f"{op_name}: value {value!r} not in mapping (keys: {record_keys}{suffix}). " "Pass ignore_unknown=True to substitute `default` instead." ) @@ -112,7 +112,7 @@ def masks_to_detection( boxes: list = [] if connected: - from sampleflux.ops.numpy import connected_component_bboxes + from recordstream.ops.numpy import connected_component_bboxes for r0, r1, c0, c1 in connected_component_bboxes(mask != 0, min_area, connectivity): boxes.append((float(c0), float(r0), float(c1 + 1), float(r1 + 1))) @@ -138,10 +138,10 @@ def masks_to_detection( class EncodeTarget(Transform): """A class-NAME ``Label`` → a class-ID ``Label``. - Reads a :class:`~sampleflux.Label` field (``field``; blank picks the first ``Label``) + Reads a :class:`~recordstream.Label` field (``field``; blank picks the first ``Label``) whose ``.value`` is a raw class name and maps it to its class id through the config-pinned ``mapping`` — the declarative ``LabelEncoder`` analogue. The result is a new - :class:`~sampleflux.Label` (carrying the source label's ``classes`` vocabulary) written + :class:`~recordstream.Label` (carrying the source label's ``classes`` vocabulary) written under ``output`` — blank (default) replaces the source field in place. Args: @@ -200,10 +200,10 @@ def __call__(self, record: Record) -> Record: class DecodeTarget(Transform): """A class-ID ``Label`` → a class-NAME ``Label`` (inverse of :class:`EncodeTarget`). - Reads a :class:`~sampleflux.Label` field (``field``; blank picks the first ``Label``) + Reads a :class:`~recordstream.Label` field (``field``; blank picks the first ``Label``) whose ``.value`` is an encoded class id and maps it back to its label name through ``mapping`` — the readback half used in prediction / reporting. The result is a new - :class:`~sampleflux.Label` written under ``output`` (blank replaces in place). + :class:`~recordstream.Label` written under ``output`` (blank replaces in place). Args: mapping: Lookup from class id → label name, e.g. ``{2: "DJI AVATA2", ...}``. Must be non-empty. @@ -261,9 +261,9 @@ def __call__(self, record: Record) -> Record: class CocoToTorchVisionDetection(Transform): """A COCO / HF ``objects`` annotation → a target ``Regions``. - Reads a source field (``field``; blank picks the first :class:`~sampleflux.Label`, else the + Reads a source field (``field``; blank picks the first :class:`~recordstream.Label`, else the first field) carrying a HuggingFace / COCO ``objects`` mapping and rewrites it to the - torchvision detection target, riding as a :class:`~sampleflux.Regions` item under + torchvision detection target, riding as a :class:`~recordstream.Regions` item under ``output`` (``boxes`` = the ``[N, 4]`` float32 xyxy tensor, ``labels`` = the ``[N]`` int64 class-id tensor). An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract). @@ -324,9 +324,9 @@ def __call__(self, record: Record) -> Record: class MasksToDetectionBoxes(Transform): """A segmentation ``Mask`` → a target ``Regions``. - Reads the :class:`~sampleflux.Mask` at ``field`` (blank = the first ``Mask`` in the record, + Reads the :class:`~recordstream.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the first array-bearing item) as a 2-D integer mask and derives one tight - ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~sampleflux.Regions` item + ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~recordstream.Regions` item under ``output``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. Args: diff --git a/sampleflux/ops/torch.py b/recordstream/ops/torch.py similarity index 93% rename from sampleflux/ops/torch.py rename to recordstream/ops/torch.py index 472efb2..9dc3c78 100644 --- a/sampleflux/ops/torch.py +++ b/recordstream/ops/torch.py @@ -4,8 +4,8 @@ import torch from confluid import configurable -from sampleflux.items import NDArrayItem, Record, item_data -from sampleflux.transform import Transform +from recordstream.items import NDArrayItem, Record, item_data +from recordstream.transform import Transform def to_tensor(img: Any, normalize: bool = True, mode: Optional[str] = None) -> torch.Tensor: @@ -41,7 +41,7 @@ class ToTensor(Transform): """An array-bearing field → a LIVE CHW-float ``torch.Tensor`` record value. Reads the payload of an array-bearing field (blank ``field`` picks the first array/PIL-bearing - item — typically the :class:`~sampleflux.Image` a :class:`~sampleflux.ops.image.ConvertToImage` + item — typically the :class:`~recordstream.Image` a :class:`~recordstream.ops.image.ConvertToImage` produced), runs the HWC→CHW transpose + ``normalize`` conversion (:func:`to_tensor`), and writes the resulting ``torch.Tensor`` back AS-IS. By default it REPLACES the resolved field in place (``output`` blank); set ``output`` to write a NEW key instead. Any other key passes @@ -50,7 +50,7 @@ class ToTensor(Transform): The output is a PLAIN record value (a record holds arbitrary values — the ``"plain"`` codec tag covers storage): ``collate_records`` stacks torch tensors natively (``torch.stack``), a torchvision ``transforms.v2`` op downstream transforms it as-is, and array sinks convert via - ``to_numpy`` on write. It is deliberately NOT wrapped in an :class:`~sampleflux.Image` — an + ``to_numpy`` on write. It is deliberately NOT wrapped in an :class:`~recordstream.Image` — an ``NDArrayItem`` coerces through ``np.asarray`` and cannot hold a live tensor. Args: diff --git a/sampleflux/processing.py b/recordstream/processing.py similarity index 63% rename from sampleflux/processing.py rename to recordstream/processing.py index 68cb2da..90bdecf 100644 --- a/sampleflux/processing.py +++ b/recordstream/processing.py @@ -1,13 +1,13 @@ """Generic source→sink pipeline runner. -:class:`DatasetProcessor` orchestrates a :class:`~sampleflux.core.Flux` from source +:class:`DatasetProcessor` orchestrates a :class:`~recordstream.core.Stream` from source to sink — a runnable that drives whole-dataset processing (windowing, format conversion, data acquisition) with an optional console progress bar. It is the -generic, modality-neutral data-pipeline runner: it iterates the flux and writes +generic, modality-neutral data-pipeline runner: it iterates the stream and writes each item to the sink, carrier-agnostic (it never inspects item internals), so it -works for any ``Flux`` regardless of what flows through it. +works for any ``Stream`` regardless of what flows through it. -Wired as the ``runnable:`` object of a config and run via ``sampleflux run``, or +Wired as the ``runnable:`` object of a config and run via ``recordstream run``, or docked into a visual-editor canvas as a runnable node. """ @@ -18,32 +18,32 @@ from loggair import get_logger from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeRemainingColumn -from sampleflux.core import Flux -from sampleflux.runnable import ProgressReporting -from sampleflux.storage.base import Storage +from recordstream.core import Stream +from recordstream.runnable import ProgressReporting +from recordstream.storage.base import Storage logger = get_logger(__name__) -def _flux_total(flux: Flux) -> Optional[int]: - """``len(flux.source)`` when the source is sized, else ``None`` (a glob-based / streaming source).""" +def _stream_total(stream: Stream) -> Optional[int]: + """``len(stream.source)`` when the source is sized, else ``None`` (a glob-based / streaming source).""" try: - return len(flux.source) # type: ignore[arg-type] + return len(stream.source) # type: ignore[arg-type] except (TypeError, AttributeError): return None @configurable class DatasetProcessor(ProgressReporting): - """Orchestrate a SampleFlux pipeline from source to sink. + """Orchestrate a RecordStream pipeline from source to sink. Args: - flux: The :class:`~sampleflux.core.Flux` to execute. Required to run; + stream: The :class:`~recordstream.core.Stream` to execute. Required to run; defaulted to ``None`` for zero-arg construction (validated in :meth:`run`, the workspace lazy-construction rule). - sink: Optional sink; when absent, samples are materialized to a list. + sink: Optional sink; when absent, records are materialized to a list. show_progress: If ``True``, wrap iteration with a ``rich.progress`` bar. - The total is derived from ``len(flux.source)`` when available; an + The total is derived from ``len(stream.source)`` when available; an unsized source falls back to a count-only bar. Default ``False``. progress_desc: Optional label for the progress bar (defaults to ``"DatasetProcessor"``). Ignored when ``show_progress`` is ``False``. @@ -51,74 +51,74 @@ class DatasetProcessor(ProgressReporting): def __init__( self, - flux: Optional[Flux] = None, + stream: Optional[Stream] = None, sink: Optional[Any] = None, show_progress: bool = False, progress_desc: Optional[str] = None, ) -> None: - self.flux = flux + self.stream = stream self.sink = sink self.show_progress = show_progress self.progress_desc = progress_desc def run(self) -> None: logger.info("Starting DatasetProcessor...") - if self.flux is None: - raise ValueError("DatasetProcessor.run() requires a 'flux' — none was configured.") + if self.stream is None: + raise ValueError("DatasetProcessor.run() requires a 'stream' — none was configured.") # Confluid keeps Class kwargs deferred (post-construction paradigm) so - # when the processor was loaded from YAML, ``self.flux``, its source, + # when the processor was loaded from YAML, ``self.stream``, its source, # its ops, and ``self.sink`` may all be Fluid stubs. Materialize them # here so callers don't need to know. from confluid import flow from confluid.fluid import Fluid - flux = flow(self.flux) if isinstance(self.flux, Fluid) else self.flux - if isinstance(flux.source, Fluid): - flux.source = flow(flux.source) - flux.ops = [flow(op) if isinstance(op, Fluid) else op for op in flux.ops] - self.flux = flux + stream = flow(self.stream) if isinstance(self.stream, Fluid) else self.stream + if isinstance(stream.source, Fluid): + stream.source = flow(stream.source) + stream.ops = [flow(op) if isinstance(op, Fluid) else op for op in stream.ops] + self.stream = stream sink = flow(self.sink) if isinstance(self.sink, Fluid) else self.sink - iterator = self._wrap_progress(flux) + iterator = self._wrap_progress(stream) # Drive an executor's progress bar (a GUI canvas) per item — independent of the console # ``show_progress`` rich bar; a no-op when no progress callback was injected. - total = _flux_total(flux) + total = _stream_total(stream) desc = self.progress_desc or "DatasetProcessor" if sink: logger.info(f"Streaming data to sink: {sink.__class__.__name__}") - # Replicates sampleflux.core.Flux.to_sink so we can iterate through + # Replicates recordstream.core.Stream.to_sink so we can iterate through # our progress wrapper while preserving the Storage context + flush. sink_ctx: Any = sink if isinstance(sink, Storage) else nullcontext() count = 0 with sink_ctx: - for sample in iterator: - sink.write(sample) + for record in iterator: + sink.write(record) count += 1 self._report_progress(count, total, desc) sink.flush() - logger.info(f"Streamed {count} sample(s) to sink.") + logger.info(f"Streamed {count} record(s) to sink.") else: logger.info("No sink provided. Materializing data in-memory.") results = [] - for count, sample in enumerate(iterator, start=1): - results.append(sample) + for count, record in enumerate(iterator, start=1): + results.append(record) self._report_progress(count, total, desc) - logger.info(f"Processed {len(results)} samples.") + logger.info(f"Processed {len(results)} records.") logger.info("Processing complete.") - def _wrap_progress(self, flux: Flux) -> Iterable[Any]: - """Wrap the flux iterator in rich.progress when ``show_progress`` is enabled. + def _wrap_progress(self, stream: Stream) -> Iterable[Any]: + """Wrap the stream iterator in rich.progress when ``show_progress`` is enabled. Source length is probed defensively — not every DataSource implements ``__len__`` (e.g. glob-based streaming sources). Missing lengths degrade to a count-only bar instead of breaking the run. """ if not self.show_progress: - return flux + return stream desc = self.progress_desc or "DatasetProcessor" - return _ProgressIter(flux, total=_flux_total(flux), desc=desc) + return _ProgressIter(stream, total=_stream_total(stream), desc=desc) class _ProgressIter: @@ -126,12 +126,12 @@ class _ProgressIter: Kept as a class (not a generator) so ``DatasetProcessor._wrap_progress`` can return a value that is truthy to ``bool()`` even when empty, matching - the contract of raw ``Flux`` which behaves like a ``Sized`` (``Flux`` + the contract of raw ``Stream`` which behaves like a ``Sized`` (``Stream`` subclasses ``torch.utils.data.Dataset``). """ - def __init__(self, flux: Flux, total: Optional[int], desc: str) -> None: - self._flux = flux + def __init__(self, stream: Stream, total: Optional[int], desc: str) -> None: + self._stream = stream self._total = total self._desc = desc @@ -145,8 +145,8 @@ def __iter__(self) -> Iterator[Any]: transient=True, ) as progress: task = progress.add_task(self._desc, total=self._total) - for sample in self._flux: - yield sample + for record in self._stream: + yield record progress.update(task, advance=1) diff --git a/sampleflux/projection.py b/recordstream/projection.py similarity index 91% rename from sampleflux/projection.py rename to recordstream/projection.py index b9d54b0..c3bd7f5 100644 --- a/sampleflux/projection.py +++ b/recordstream/projection.py @@ -1,4 +1,4 @@ -"""Key projection for SampleFlux sources — read only the record keys you need. +"""Key projection for RecordStream sources — read only the record keys you need. Walking a source for a single key (the canonical case: counting classes from the label key) should not pay for constructing the values you don't need — e.g. decoding image @@ -16,14 +16,14 @@ * Every public function is a lazy generator (**Lazy Evaluation** mandate) — nothing materializes the whole source. * :func:`num_classes` (integer class-id semantics) is a free function, *not* a - method on the generic :class:`~sampleflux.core.Flux` engine — counting classes is + method on the generic :class:`~recordstream.core.Stream` engine — counting classes is a classification concern, and bolting it onto the task-agnostic engine would - make every ``Flux`` look classification-capable to duck-typed consumers. + make every ``Stream`` look classification-capable to duck-typed consumers. """ from typing import Any, Collection, Iterator, Protocol, runtime_checkable -from sampleflux.items import Label, Record, is_item, item_data +from recordstream.items import Label, Record, is_item, item_data @runtime_checkable @@ -56,8 +56,8 @@ def project(source: Any, keys: Collection[str]) -> Iterator[Record]: def iter_key(source: Any, key: str) -> Iterator[Any]: """Lazily yield each record's ``key`` VALUE (skipping other-key construction when supported). - A :class:`~sampleflux.items.Label` unwraps to its ``.value`` (the class id / name); any - other registered item unwraps to its payload via :func:`~sampleflux.items.item_data`; a + A :class:`~recordstream.items.Label` unwraps to its ``.value`` (the class id / name); any + other registered item unwraps to its payload via :func:`~recordstream.items.item_data`; a plain value passes through verbatim. A record without ``key`` yields ``None``. """ for record in project(source, (key,)): diff --git a/sampleflux/py.typed b/recordstream/py.typed similarity index 100% rename from sampleflux/py.typed rename to recordstream/py.typed diff --git a/sampleflux/runnable.py b/recordstream/runnable.py similarity index 99% rename from sampleflux/runnable.py rename to recordstream/runnable.py index 40de75a..3aeee16 100644 --- a/sampleflux/runnable.py +++ b/recordstream/runnable.py @@ -42,7 +42,7 @@ _ENTRYPOINT_ATTR = "__runnable_entrypoint__" #: A progress sink: ``(value, total, description) -> None``. ``value`` / ``total`` are -#: floats in the same unit (optimizer steps, samples); ``description`` is a short stage label. +#: floats in the same unit (optimizer steps, records); ``description`` is a short stage label. ProgressCallback = Callable[[float, float, str], None] diff --git a/sampleflux/sources.py b/recordstream/sources.py similarity index 94% rename from sampleflux/sources.py rename to recordstream/sources.py index 4b411f1..30e41e2 100644 --- a/sampleflux/sources.py +++ b/recordstream/sources.py @@ -5,7 +5,7 @@ from confluid import configurable from loggair import get_logger -from sampleflux.items import Image, Label, Record +from recordstream.items import Image, Label, Record logger = get_logger(__name__) @@ -13,7 +13,7 @@ def _pass_through(item: Any) -> Any: """Pass a wrapped source's item through verbatim. - Every carrier is a plain record dict; the view sources + Every carrier is a plain dict; the view sources (:class:`DatasetSplit` / :class:`RangeSource` / :class:`ConcatSource`) only slice/index, they never inspect payloads, so a source's records flow through them unchanged. """ @@ -63,15 +63,15 @@ def _resolve_metadata_features( @configurable(category="source") class HuggingFaceSource: """ - SampleFlux Source for Hugging Face Datasets, yielding plain record dicts. + RecordStream Source for Hugging Face Datasets, yielding plain record dicts. Key mapping (the record layout): - * the ``input_feature`` value (image / array) -> an :class:`~sampleflux.Image` under the + * the ``input_feature`` value (image / array) -> an :class:`~recordstream.Image` under the record key ``"image"``; - * the ``target_feature`` value (label) -> a :class:`~sampleflux.Label` under the record key + * the ``target_feature`` value (label) -> a :class:`~recordstream.Label` under the record key ``"class"``; - * each ``metadata_features`` column -> its own :class:`~sampleflux.Label` keyed by the column + * each ``metadata_features`` column -> its own :class:`~recordstream.Label` keyed by the column name, plus the source-provenance ``hf_path`` / ``hf_split`` Labels. Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md @@ -86,7 +86,7 @@ class HuggingFaceSource: input_feature: Dataset feature column mapped onto the ``"image"`` record key (an ``Image`` item). target_feature: Dataset feature column mapped onto the ``"class"`` record key (a ``Label`` item). metadata_features: Columns -> per-column ``Label`` entries; ``None``=none, ``"*"``=all-but-i/o, else a list. - count: Optional cap on the number of samples yielded (useful for fast smoke runs). + count: Optional cap on the number of records yielded (useful for fast smoke runs). name: Optional HF subset/config name (e.g. for multi-config datasets). """ @@ -194,7 +194,7 @@ def __getitem__(self, index: int) -> Record: def project(self, keys: Collection[str]) -> Iterator[Record]: """Yield key-restricted records — the ``SupportsProjection`` efficient path. - Only the requested keys are built, so a label-only walk (e.g. :func:`~sampleflux.num_classes`) + Only the requested keys are built, so a label-only walk (e.g. :func:`~recordstream.num_classes`) skips decoding the image entirely: ``"image"`` -> the input feature, ``"class"`` -> the target Label, plus any requested metadata-column / provenance keys. """ @@ -211,11 +211,11 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: yield self._to_record(item, metadata_features, keys=want) def __len__(self) -> int: - # A ``count`` of 0 (or None) means "all samples", matching __iter__'s + # A ``count`` of 0 (or None) means "all records", matching __iter__'s # ``limit = self.count or len(...)``. Returning a bare ``self.count`` here # would report 0 for the common "0 == unlimited" case, making the source # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) - # even though iteration yields every sample. + # even though iteration yields every record. return self.count or len(self.dataset) @@ -244,15 +244,15 @@ class DatasetSplit: ``.test`` — and because two ``!ref:`` to the same key flow the *same* instance, the partition and the source load are shared across all three references:: - my_split: !class:sampleflux.sources.DatasetSplit() + my_split: !class:recordstream.sources.DatasetSplit() source: !ref:hf_train val_fraction: 0.1 test_fraction: 0.1 seed: 42 - train_set: !class:sampleflux.core.Flux() + train_set: !class:recordstream.core.Stream() source: !ref:my_split.train - val_set: !class:sampleflux.core.Flux() + val_set: !class:recordstream.core.Stream() source: !ref:my_split.val **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one @@ -262,13 +262,13 @@ class DatasetSplit: degenerate split where ``train`` is the whole source and ``val`` / ``test`` are empty. The wrapped source must implement ``__len__`` and ``__getitem__``. Lazy: only index - arithmetic happens up front; samples are produced on demand. + arithmetic happens up front; records are produced on demand. Args: source: The underlying indexable source (defaults to ``None``; validated lazily on first use). split: View this iterates as a source — ``train`` / ``val`` / ``test`` (``None`` ⇒ ``train``). - val_fraction: Fraction of samples assigned to the ``val`` view. Must be in ``(0, 1)``. - test_fraction: Fraction of samples assigned to the ``test`` view. Must be in ``(0, 1)``. + val_fraction: Fraction of records assigned to the ``val`` view. Must be in ``(0, 1)``. + test_fraction: Fraction of records assigned to the ``test`` view. Must be in ``(0, 1)``. seed: Seed for the deterministic shuffle. Required when any fraction is set. """ @@ -319,7 +319,7 @@ def _partition(self) -> Dict[str, List[int]]: One shuffle seeded by ``seed`` (skipped when no fraction is set, so the degenerate "all train" case keeps source order); layout is ``[train | val | test]``. ``max(1, …)`` - guarantees a held-out split gets at least one sample on tiny sources. + guarantees a held-out split gets at least one record on tiny sources. """ n = len(self.source) val_fraction = self.val_fraction or 0.0 @@ -398,7 +398,7 @@ class RangeSource: The plain-slice counterpart to :class:`DatasetSplit` (which shuffles + partitions) — extracted from DatasetSplit's old "range mode". Negative ``start`` / ``stop`` count from the end; both are clamped to ``[0, len(source)]``. Lazy: only index arithmetic happens - up front; samples are produced on demand. + up front; records are produced on demand. The wrapped source must implement ``__len__`` and ``__getitem__``. @@ -453,11 +453,11 @@ def __len__(self) -> int: class ConcatSource: """Concatenates multiple indexable sources into one longer indexable source. - The indexable counterpart to :class:`sampleflux.core.JointFlux` (which is iteration-only): + The indexable counterpart to :class:`recordstream.core.JointStream` (which is iteration-only): ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning sub-source, so a ``ConcatSource`` can itself be wrapped by :class:`DatasetSplit` / :class:`RangeSource`. (Distinct from :class:`waivefront.paired.AnnotationJoinSource`, which - *column-joins* annotations onto samples — this one *concatenates* sequences end to end.) + *column-joins* annotations onto records — this one *concatenates* sequences end to end.) Each sub-source must implement ``__len__`` and ``__getitem__``. diff --git a/sampleflux/storage/base.py b/recordstream/storage/base.py similarity index 93% rename from sampleflux/storage/base.py rename to recordstream/storage/base.py index 91e2d21..2d6d017 100644 --- a/sampleflux/storage/base.py +++ b/recordstream/storage/base.py @@ -4,7 +4,7 @@ import numpy as np import torch -from sampleflux.items import Record +from recordstream.items import Record #: Root-attribute format tag stamped on stores written in the record key-group layout. TYPED_FORMAT = "typedrecord-v1" @@ -21,12 +21,12 @@ def require_record_format(found: Any, where: str) -> None: """Raise unless ``found`` is the record-layout tag ``"typedrecord-v1"``. There is deliberately NO backward compatibility with pre-record layouts: a store whose - ``sampleflux_format`` tag is missing or different was written before the plain-dict + ``recordstream_format`` tag is missing or different was written before the plain-dict record model and must be re-generated with a current sink. """ if found == TYPED_FORMAT: return - detail = "no sampleflux_format tag" if found is None else f"sampleflux_format={found!r}" + detail = "no recordstream_format tag" if found is None else f"recordstream_format={found!r}" raise ValueError( f"{where}: {detail} — expected {TYPED_FORMAT!r}. Pre-record-model datasets are not " "readable/appendable; re-generate them with a current sink." @@ -46,7 +46,7 @@ def to_numpy(data: Any) -> Any: @runtime_checkable class DataSource(Protocol): - """Minimum contract for a SampleFlux data source.""" + """Minimum contract for a RecordStream data source.""" def __iter__(self) -> Iterator[Record]: """Iterate over records in the source.""" @@ -59,7 +59,7 @@ def __len__(self) -> int: @runtime_checkable class DataSink(Protocol): - """Minimum contract for a SampleFlux data sink.""" + """Minimum contract for a RecordStream data sink.""" def write(self, record: Record) -> None: """Write a single record to the sink.""" diff --git a/sampleflux/storage/cache.py b/recordstream/storage/cache.py similarity index 100% rename from sampleflux/storage/cache.py rename to recordstream/storage/cache.py diff --git a/sampleflux/storage/directory.py b/recordstream/storage/directory.py similarity index 84% rename from sampleflux/storage/directory.py rename to recordstream/storage/directory.py index c74d674..797d556 100644 --- a/sampleflux/storage/directory.py +++ b/recordstream/storage/directory.py @@ -5,9 +5,9 @@ import confluid import numpy as np -from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item -from sampleflux.items import Record -from sampleflux.storage.base import ( +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record +from recordstream.storage.base import ( PLAIN_VALUE, TYPED_FORMAT, DataSink, @@ -18,7 +18,7 @@ to_numpy, ) -#: Record-layout filenames inside each per-sample directory. +#: Record-layout filenames inside each per-record directory. _FIELDS_JSON = "fields.json" _FIELDS_NPZ = "fields.npz" @@ -59,13 +59,13 @@ def _write_record(self, record: Record) -> None: value's non-array payload rides its ``attrs`` under ``"value"``, JSON-marked when structured); ``fields.npz`` carries the array halves — payloads keyed by record key, array-valued attrs keyed ``.``. Every value serializes through the - :mod:`sampleflux.io` codec, so externally-registered item types round-trip with no + :mod:`recordstream.io` codec, so externally-registered item types round-trip with no storage edits. """ - sample_dir = self.path / f"{self._counter:06d}" - sample_dir.mkdir(parents=True, exist_ok=True) + record_dir = self.path / f"{self._counter:06d}" + record_dir.mkdir(parents=True, exist_ok=True) - spec: Dict[str, Any] = {"sampleflux_format": TYPED_FORMAT, "fields": []} + spec: Dict[str, Any] = {"recordstream_format": TYPED_FORMAT, "fields": []} payloads: Dict[str, Any] = {} for key, value in record.items(): encoded = encode_item(value) @@ -99,9 +99,9 @@ def _write_record(self, record: Record) -> None: for name, attr_value in arrays.items(): payloads[f"{key}.{name}"] = np.asarray(attr_value) - (sample_dir / _FIELDS_JSON).write_text(json.dumps(spec, indent=2)) + (record_dir / _FIELDS_JSON).write_text(json.dumps(spec, indent=2)) if payloads: - np.savez(sample_dir / _FIELDS_NPZ, **payloads) + np.savez(record_dir / _FIELDS_NPZ, **payloads) self._counter += 1 def flush(self) -> None: @@ -123,23 +123,23 @@ def __init__(self, path: Union[str, Path] = "") -> None: # Lazy / zero-arg: store config only; the directory is scanned lazily on iteration. self.path = Path(path) - def _sample_dirs(self) -> list: + def _record_dirs(self) -> list: if not self.path.exists(): raise FileNotFoundError(f"DirectorySource: {self.path} does not exist") return sorted(p for p in self.path.iterdir() if p.is_dir() and (p / _FIELDS_JSON).exists()) def __iter__(self) -> Iterator[Record]: - for sample_dir in self._sample_dirs(): - yield self._read(sample_dir) + for record_dir in self._record_dirs(): + yield self._read(record_dir) def __len__(self) -> int: - return len(self._sample_dirs()) + return len(self._record_dirs()) @staticmethod - def _read(sample_dir: Path) -> Record: - spec = json.loads((sample_dir / _FIELDS_JSON).read_text()) - require_record_format(spec.get("sampleflux_format"), "DirectorySource") - npz_path = sample_dir / _FIELDS_NPZ + def _read(record_dir: Path) -> Record: + spec = json.loads((record_dir / _FIELDS_JSON).read_text()) + require_record_format(spec.get("recordstream_format"), "DirectorySource") + npz_path = record_dir / _FIELDS_NPZ payloads = dict(np.load(npz_path, allow_pickle=False)) if npz_path.exists() else {} record: Record = {} payload: Any diff --git a/sampleflux/storage/hdf5.py b/recordstream/storage/hdf5.py similarity index 90% rename from sampleflux/storage/hdf5.py rename to recordstream/storage/hdf5.py index 61a3704..f2230d3 100644 --- a/sampleflux/storage/hdf5.py +++ b/recordstream/storage/hdf5.py @@ -7,9 +7,9 @@ from confluid import configurable from loggair import get_logger -from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item -from sampleflux.items import Record -from sampleflux.storage.base import ( +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record +from recordstream.storage.base import ( PLAIN_VALUE, TYPED_FORMAT, DataSink, @@ -21,7 +21,7 @@ to_numpy, ) -logger = get_logger("sampleflux.storage.hdf5") +logger = get_logger("recordstream.storage.hdf5") #: Reserved key-group attr names in the record layout (never item attrs). _TYPE_ATTR = "__item_type__" @@ -29,7 +29,7 @@ def _read_record(group: h5py.Group) -> Record: - """Decode one ``sNNNNNN`` sample group of the record key-group layout.""" + """Decode one ``sNNNNNN`` record group of the record key-group layout.""" order = json.loads(group.attrs[_ORDER_ATTR]) record: Record = {} payload: Any @@ -74,7 +74,7 @@ def __init__(self, path: Union[str, Path] = "") -> None: def open(self) -> "HDF5Source": if self._file is None: handle = h5py.File(self.path, "r") - found = handle.attrs.get("sampleflux_format") + found = handle.attrs.get("recordstream_format") if found != TYPED_FORMAT: handle.close() require_record_format(found, "HDF5Source") @@ -103,9 +103,9 @@ def iter_metadata(self) -> "Iterator[tuple[str, dict]]": """(prefix, metadata) per record WITHOUT loading data arrays (SupportsMetadataScan). Array-valued metadata appears as shape/dtype stub strings — see - :func:`sampleflux.storage.query.scan_hdf5_metadata`. + :func:`recordstream.storage.query.scan_hdf5_metadata`. """ - from sampleflux.storage.query import scan_hdf5_metadata + from recordstream.storage.query import scan_hdf5_metadata yield from scan_hdf5_metadata(self.path) @@ -152,19 +152,19 @@ def write(self, record: Any) -> None: def _write_record(self, record: Record) -> None: """One record in the key-group layout. - Layout: root attr ``sampleflux_format = "typedrecord-v1"``; per record a group + Layout: root attr ``recordstream_format = "typedrecord-v1"``; per record a group ``sNNNNNN`` (attr ``__field_order__`` preserves insertion order) holding one subgroup per KEY with the ``__item_type__`` attr + the item's plain attrs, the payload as ``data``, and array-valued attrs as datasets under ``attrs/``. A ``"plain"`` value stores an array payload as ``data`` and any other payload as the ``value`` attr (JSON-marked when structured). Every value serializes through the - :mod:`sampleflux.io` codec, so externally-registered item types round-trip with no + :mod:`recordstream.io` codec, so externally-registered item types round-trip with no storage edits. """ assert self._file is not None - existing = self._file.attrs.get("sampleflux_format") + existing = self._file.attrs.get("recordstream_format") if existing is None and len(self._file) == 0: - self._file.attrs["sampleflux_format"] = TYPED_FORMAT + self._file.attrs["recordstream_format"] = TYPED_FORMAT elif existing != TYPED_FORMAT: require_record_format(existing, "HDF5Sink") diff --git a/sampleflux/storage/query.py b/recordstream/storage/query.py similarity index 95% rename from sampleflux/storage/query.py rename to recordstream/storage/query.py index bb24c26..ebe5ceb 100644 --- a/sampleflux/storage/query.py +++ b/recordstream/storage/query.py @@ -1,6 +1,6 @@ """Queryable metadata — filter stored records by metadata predicates WITHOUT loading arrays. -Two pieces (mirroring the ``sampleflux.projection`` protocol-plus-fallback design): +Two pieces (mirroring the ``recordstream.projection`` protocol-plus-fallback design): - :class:`SupportsMetadataScan` — a source opts in by implementing ``iter_metadata() -> Iterator[(key, metadata_dict)]`` that reads ONLY the metadata @@ -28,12 +28,12 @@ from confluid import configurable from loggair import get_logger -from sampleflux.io import PLAIN_TYPE, encode_item -from sampleflux.items import Record -from sampleflux.ops.formula import _FORMULA_NAMESPACE -from sampleflux.storage.base import PLAIN_VALUE, require_record_format, restore_attrs +from recordstream.io import PLAIN_TYPE, encode_item +from recordstream.items import Record +from recordstream.ops.formula import _FORMULA_NAMESPACE +from recordstream.storage.base import PLAIN_VALUE, require_record_format, restore_attrs -logger = get_logger("sampleflux.storage.query") +logger = get_logger("recordstream.storage.query") __all__ = [ "MetadataFilterSource", @@ -97,7 +97,7 @@ def scan_hdf5_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: (e.g. ``"signal.samplerate > 1e6"``). """ with h5py.File(str(path), "r") as handle: - require_record_format(handle.attrs.get("sampleflux_format"), "scan_hdf5_metadata") + require_record_format(handle.attrs.get("recordstream_format"), "scan_hdf5_metadata") for name in sorted(k for k in handle.keys() if k.startswith("s")): group = handle[name] nested: Dict[str, Any] = {} @@ -122,7 +122,7 @@ def scan_zarr_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: import zarr root = zarr.open_group(str(path), mode="r") - require_record_format(root.attrs.get("sampleflux_format"), "scan_zarr_metadata") + require_record_format(root.attrs.get("recordstream_format"), "scan_zarr_metadata") for name in sorted(root.group_keys()): group = cast(Any, root[name]) nested: Dict[str, Any] = {} diff --git a/sampleflux/storage/zarr.py b/recordstream/storage/zarr.py similarity index 91% rename from sampleflux/storage/zarr.py rename to recordstream/storage/zarr.py index 10cdbb5..7604d50 100644 --- a/sampleflux/storage/zarr.py +++ b/recordstream/storage/zarr.py @@ -6,9 +6,9 @@ import numpy as np import zarr -from sampleflux.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item -from sampleflux.items import Record -from sampleflux.storage.base import ( +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record +from recordstream.storage.base import ( PLAIN_VALUE, TYPED_FORMAT, DataSink, @@ -26,7 +26,7 @@ def _read_record(grp: "zarr.Group") -> Record: - """Decode one ``sample_NNNNNN`` group of the record key-group layout.""" + """Decode one ``record_NNNNNN`` group of the record key-group layout.""" order = json.loads(str(grp.attrs[_ORDER_ATTR])) record: Record = {} payload: Any @@ -89,13 +89,13 @@ def write(self, record: Any) -> None: def _write_record(self, record: Record) -> None: """One record in the key-group layout (the Zarr twin of HDF5Sink._write_record).""" assert self._root is not None - existing = self._root.attrs.get("sampleflux_format") + existing = self._root.attrs.get("recordstream_format") if existing is None and not any(True for _ in self._root.group_keys()): - self._root.attrs["sampleflux_format"] = TYPED_FORMAT + self._root.attrs["recordstream_format"] = TYPED_FORMAT elif existing != TYPED_FORMAT: require_record_format(existing, "ZarrGroupSink") - grp = self._root.require_group(f"sample_{self._counter:06d}") + grp = self._root.require_group(f"record_{self._counter:06d}") grp.attrs[_ORDER_ATTR] = json.dumps(list(record.keys())) for key, value in record.items(): encoded = encode_item(value) @@ -139,7 +139,7 @@ def __init__(self, path: Union[str, Path] = "") -> None: def open(self) -> "ZarrGroupSource": if self._root is None: root = zarr.open_group(self.path, mode="r") - require_record_format(root.attrs.get("sampleflux_format"), "ZarrGroupSource") + require_record_format(root.attrs.get("recordstream_format"), "ZarrGroupSource") self._root = root return self @@ -161,7 +161,7 @@ def __len__(self) -> int: def iter_metadata(self) -> "Iterator[tuple[str, dict]]": """(group name, ``.zattrs`` metadata) per record WITHOUT loading arrays (SupportsMetadataScan).""" - from sampleflux.storage.query import scan_zarr_metadata + from recordstream.storage.query import scan_zarr_metadata yield from scan_zarr_metadata(self.path) @@ -219,7 +219,7 @@ def write(self, record: Any) -> None: # variation does not fit a single stacked array; use ZarrGroupSink for that. key, value = next(iter(record.items())) encoded = encode_item(value) - existing = self._data_arr.attrs.get("sampleflux_format") + existing = self._data_arr.attrs.get("recordstream_format") if existing is None: if self._data_arr.shape[0] > 0: require_record_format(None, "ZarrBatchSink") @@ -230,7 +230,7 @@ def write(self, record: Any) -> None: "layout — use ZarrGroupSink." ) self._data_arr.attrs.update( - {"sampleflux_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} + {"recordstream_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} ) elif existing != TYPED_FORMAT: require_record_format(existing, "ZarrBatchSink") @@ -261,7 +261,7 @@ def __init__(self, path: Union[str, Path] = "") -> None: def open(self) -> "ZarrBatchSource": if self._data_arr is None: arr = zarr.open_array(store=f"{self.path}/data", mode="r") - require_record_format(arr.attrs.get("sampleflux_format"), "ZarrBatchSource") + require_record_format(arr.attrs.get("recordstream_format"), "ZarrBatchSource") self._data_arr = arr return self @@ -278,7 +278,7 @@ def __iter__(self) -> Iterator[Record]: field = str(attrs["__field__"]) type_name = str(attrs[_TYPE_ATTR]) item_attrs = restore_attrs( - {k: v for k, v in attrs.items() if k not in ("sampleflux_format", _TYPE_ATTR, "__field__")}, {} + {k: v for k, v in attrs.items() if k not in ("recordstream_format", _TYPE_ATTR, "__field__")}, {} ) for i in range(self._data_arr.shape[0]): payload = np.asarray(self._data_arr[i]) diff --git a/sampleflux/transform.py b/recordstream/transform.py similarity index 92% rename from sampleflux/transform.py rename to recordstream/transform.py index b1c15d8..f1517bb 100644 --- a/sampleflux/transform.py +++ b/recordstream/transform.py @@ -1,8 +1,8 @@ """``Transform`` — type-dispatched record ops with once-per-record parameters, plus ``Pipeline``. -A transform samples its random / configured parameters ONCE per record (:meth:`Transform.get_params`), +A transform records its random / configured parameters ONCE per record (:meth:`Transform.get_params`), then walks the dict and, for each value whose type it handles, applies the registered -kernel (:mod:`sampleflux.dispatch`). Values it does not handle pass through untouched. +kernel (:mod:`recordstream.dispatch`). Values it does not handle pass through untouched. Two properties fall out of this shape for free: @@ -15,9 +15,9 @@ Targeting is by TYPE; the ``field`` parameter pins an op to one named key when a record holds several values of a handled type. -sampleflux ships NO native augmentation ops — geometric/photometric augmentation comes from +recordstream ships NO native augmentation ops — geometric/photometric augmentation comes from the libraries (torchvision ``transforms.v2`` / albumentations) dropped into an ops list -AS-IS; the engine invokes each op family natively (see ``sampleflux.core._apply_op``). +AS-IS; the engine invokes each op family natively (see ``recordstream.core._apply_op``). There are no wrapper/adapter classes. """ @@ -25,8 +25,8 @@ from confluid import configurable -from sampleflux.dispatch import Kernel, dispatch, register_kernel -from sampleflux.items import Record, item_data, with_data +from recordstream.dispatch import Kernel, dispatch, register_kernel +from recordstream.items import Record, item_data, with_data __all__ = [ "Transform", @@ -40,7 +40,7 @@ class Transform: """Base class for type-dispatched record ops (see the module docstring). Subclasses declare ``handles`` (the value types they process) and register a kernel per - type via ``@MyOp.kernel(ItemType)``. Override :meth:`get_params` to sample shared + type via ``@MyOp.kernel(ItemType)``. Override :meth:`get_params` to record shared parameters once per record. **The type-interface attributes** (``handles`` / ``consumes`` / ``optional`` / @@ -81,7 +81,7 @@ def kernel(cls, item_type: type) -> Callable[[Kernel], Kernel]: return register_kernel(cls, item_type) def get_params(self, record: Record) -> Dict[str, Any]: - """Sample the shared parameters for one call. Default: no params.""" + """Record the shared parameters for one call. Default: no params.""" return {} def __call__(self, record: Record) -> Optional[Record]: @@ -121,7 +121,7 @@ def __call__(self, record: Record) -> Optional[Record]: # _apply_op = the engine's op-family dispatch, so a bare albumentations / # torchvision-v2 transform nests here exactly as in a bare ops list. - from sampleflux.core import _apply_op + from recordstream.core import _apply_op current: Optional[Record] = record for i, op in enumerate(self.transforms): diff --git a/sampleflux/workflow.py b/recordstream/workflow.py similarity index 94% rename from sampleflux/workflow.py rename to recordstream/workflow.py index 5e9fd7f..3ca42b3 100644 --- a/sampleflux/workflow.py +++ b/recordstream/workflow.py @@ -1,7 +1,7 @@ """Higher-order runnables — compose other runnables into a workflow. A *runnable* is any object exposing a no-arg ``run(self)`` (a trainer, an -evaluator, a :class:`~sampleflux.processing.DatasetProcessor`). This module adds +evaluator, a :class:`~recordstream.processing.DatasetProcessor`). This module adds Confluid-``@configurable`` *combinators* that HOLD other runnables and orchestrate them — the runnable-level analogue of the composing ops (``Pipeline`` / ``Parallel`` / ``Enable``): @@ -14,16 +14,16 @@ ``__call__(self) -> bool`` (:class:`PathExists` / :class:`Not` / :class:`AllOf` / :class:`AnyOf`) — so a whole workflow (steps, branches, AND the conditions that pick them) serialises to ONE Confluid YAML document, runs via the generic -``sampleflux run workflow.yaml`` runner, and — being plain ``@configurable`` +``recordstream run workflow.yaml`` runner, and — being plain ``@configurable`` classes — is surfaced by discovery / a visual editor with no bespoke glue. Branches are held as INSTANCES and only ``run()`` when selected. Example:: - !class:sampleflux.workflow.Sequence + !class:recordstream.workflow.Sequence steps: - !lazy:DownloadData - - !class:sampleflux.workflow.Conditional - condition: !class:sampleflux.workflow.PathExists { path: $MODEL_ROOT/model.ckpt } + - !class:recordstream.workflow.Conditional + condition: !class:recordstream.workflow.PathExists { path: $MODEL_ROOT/model.ckpt } if_false: !lazy:TrainModel # cache miss -> train if_true: null # cache hit -> skip, fall through - !lazy:Evaluate @@ -41,8 +41,8 @@ conditions are flowed lazily inside ``run()`` (a Confluid ``!class:`` / ``!lazy:`` member arrives as a deferred ``Fluid`` stub and is materialised on demand). -The combinators inherit :class:`~sampleflux.runnable.TorchRunner` and -:class:`~sampleflux.runnable.ProgressReporting` so a workflow runs correctly on a +The combinators inherit :class:`~recordstream.runnable.TorchRunner` and +:class:`~recordstream.runnable.ProgressReporting` so a workflow runs correctly on a GUI canvas: a combinator may wrap a *trainer*, so it declares ``__torch_runner__`` (the executor re-enables autograd for the whole run — otherwise an inner ``loss.backward()`` dies under the executor's inference mode; restoring autograd is @@ -57,7 +57,7 @@ from confluid.fluid import Fluid from loggair import get_logger -from sampleflux.runnable import ProgressReporting, TorchRunner +from recordstream.runnable import ProgressReporting, TorchRunner logger = get_logger(__name__) diff --git a/sampleflux/ops/__init__.py b/sampleflux/ops/__init__.py deleted file mode 100644 index d2d7aaa..0000000 --- a/sampleflux/ops/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -SampleFlux operations (record-dict ops). - -Submodules: - - sampleflux.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / - connected_component_bboxes / resolve_expression helpers) - - sampleflux.ops.torch: ToTensor (+ to_tensor helper) - - sampleflux.ops.image: ConvertToImage (+ value_to_image / normalize_to_uint8 …) - - sampleflux.ops.target: EncodeTarget, DecodeTarget, - CocoToTorchVisionDetection, MasksToDetectionBoxes - - sampleflux.ops.structure: RenameField, DropField, CopyField, SelectFields - - sampleflux.ops.parallel: Parallel (worker-pool sub-pipeline) - - sampleflux.ops.enable: Enable (toggle an op-list via one named CLI flag) - - sampleflux.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) - - sampleflux.ops.configure: ConfigureOp (per-record parameter injection) - - sampleflux.ops.formula: FormulaOp (math formula over one record entry) - - sampleflux.ops.sink: RecordSinkOp (adapt a DataSink as a pass-through op) - - sampleflux.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-record - Context graph plane — the flat-list building blocks a branchy flow: document lowers to) - - sampleflux.ops.debug: PrintSampleOp (per-record summary probe) - -The sequential composer ``Pipeline`` lives in :mod:`sampleflux.transform` (package-root -export) — one list mixing native ops with bare albumentations / torchvision-v2 transforms. -""" - -from sampleflux.ops.configure import ConfigureOp -from sampleflux.ops.context import Apply, Capture, Drop, MergeFields, Save, Use -from sampleflux.ops.debug import PrintSampleOp -from sampleflux.ops.enable import Enable -from sampleflux.ops.formula import FormulaOp -from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.numpy import ConnectedComponents, Threshold -from sampleflux.ops.parallel import Parallel -from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.sink import RecordSinkOp -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields -from sampleflux.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes -from sampleflux.ops.torch import ToTensor - -__all__ = [ - "Apply", - "Capture", - "CocoToTorchVisionDetection", - "ConfigureOp", - "ConnectedComponents", - "ConvertToImage", - "CopyField", - "DecodeTarget", - "Drop", - "DropField", - "Enable", - "EncodeTarget", - "FormulaOp", - "MasksToDetectionBoxes", - "MergeFields", - "Parallel", - "PrintSampleOp", - "RandomApply", - "RenameField", - "Save", - "RecordSinkOp", - "SelectFields", - "Threshold", - "ToTensor", - "Use", -] diff --git a/tests/_fixtures.py b/tests/_fixtures.py index e5986e8..c13e642 100644 --- a/tests/_fixtures.py +++ b/tests/_fixtures.py @@ -1,6 +1,6 @@ """Test-local record-model fixtures. -``FixtureFlip`` is the former native ``HorizontalFlip`` kept ONLY as a test fixture: sampleflux +``FixtureFlip`` is the former native ``HorizontalFlip`` kept ONLY as a test fixture: recordstream ships no native augmentation transforms (geometric/photometric augmentation comes from torchvision v2 / albumentations invoked natively by the engine's op-family dispatch), but the kernel-dispatch machinery (once-per-record params, per-type kernels, MRO resolution, the @@ -11,7 +11,7 @@ import numpy as np -from sampleflux import Image, Mask, Record, Regions, Transform, item_data, with_data +from recordstream import Image, Mask, Record, Regions, Transform, item_data, with_data class FixtureFlip(Transform): diff --git a/tests/test_cache.py b/tests/test_cache.py index 4deff70..f817241 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -8,7 +8,7 @@ import pytest -from sampleflux.storage.cache import CacheBudgetExceeded, DiskCache +from recordstream.storage.cache import CacheBudgetExceeded, DiskCache def _write(payload: bytes) -> Callable[[Path], None]: diff --git a/tests/test_categories.py b/tests/test_categories.py index 2c1e95b..678e9bc 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -1,5 +1,5 @@ # mypy: disable-error-code="attr-defined,union-attr" -"""Discovery-category coverage for sampleflux ``@configurable`` classes. +"""Discovery-category coverage for recordstream ``@configurable`` classes. These ``category=`` tags drive navigaitor's ``list_configurable_classes(category=...)`` MCP tool and, downstream, the visual-editor form-spec picker (``get_node_form_spec``). @@ -9,30 +9,30 @@ from confluid.registry import get_registry -from sampleflux import Pipeline -from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.ops.configure import ConfigureOp -from sampleflux.ops.context import Apply, Capture, Drop, MergeFields, Save, Use -from sampleflux.ops.debug import PrintSampleOp -from sampleflux.ops.enable import Enable -from sampleflux.ops.formula import FormulaOp -from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.numpy import ConnectedComponents, Threshold -from sampleflux.ops.parallel import Parallel -from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.sink import RecordSinkOp -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields -from sampleflux.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes -from sampleflux.ops.torch import ToTensor -from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource -from sampleflux.storage.directory import DirectorySink -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.zarr import ZarrBatchSink, ZarrGroupSink +from recordstream import Pipeline +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp +from recordstream.ops.configure import ConfigureOp +from recordstream.ops.context import Apply, Capture, Drop, MergeFields, Save, Use +from recordstream.ops.debug import PrintRecordOp +from recordstream.ops.enable import Enable +from recordstream.ops.formula import FormulaOp +from recordstream.ops.image import ConvertToImage +from recordstream.ops.numpy import ConnectedComponents, Threshold +from recordstream.ops.parallel import Parallel +from recordstream.ops.random_apply import RandomApply +from recordstream.ops.sink import RecordSinkOp +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes +from recordstream.ops.torch import ToTensor +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource +from recordstream.storage.directory import DirectorySink +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source +from recordstream.storage.zarr import ZarrBatchSink, ZarrGroupSink def test_engine_classes_tagged() -> None: - assert Flux.__confluid_category__ == "engine" - assert JointFlux.__confluid_category__ == "engine" + assert Stream.__confluid_category__ == "engine" + assert JointStream.__confluid_category__ == "engine" def test_raw_callable_wrappers_uncategorised() -> None: @@ -76,7 +76,7 @@ def test_op_classes_tagged() -> None: Apply, Capture, MergeFields, - PrintSampleOp, + PrintRecordOp, ): assert cls.__confluid_category__ == "op", cls.__name__ @@ -99,7 +99,7 @@ def test_op_group_tags() -> None: assert ToTensor.__confluid_group__ == "torch" assert ConvertToImage.__confluid_group__ == "image" assert SelectFields.__confluid_group__ == "structure" - assert PrintSampleOp.__confluid_group__ == "debug" + assert PrintRecordOp.__confluid_group__ == "debug" assert EncodeTarget.__confluid_group__ == "structure" assert DecodeTarget.__confluid_group__ == "structure" assert CocoToTorchVisionDetection.__confluid_group__ == "structure" @@ -117,7 +117,7 @@ def test_op_group_tags() -> None: def test_categories_enumerable_via_registry() -> None: registry = get_registry() - assert {"Flux", "JointFlux"} <= registry.list_classes(category="engine") + assert {"Stream", "JointStream"} <= registry.list_classes(category="engine") assert "DatasetSplit" not in registry.list_classes(category="engine") assert not ({"FilterOp", "WrappedOp"} & registry.list_classes(category="engine")) assert {"HuggingFaceSource", "DatasetSplit", "RangeSource", "ConcatSource"} <= registry.list_classes( diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index 9ca2e33..f03f06a 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -1,10 +1,10 @@ -"""Tests for the `sampleflux run` CLI dispatch (sampleflux.cli.run).""" +"""Tests for the `recordstream run` CLI dispatch (recordstream.cli.run).""" from typing import List from confluid import Class -from sampleflux.cli import run +from recordstream.cli import run def test_run_calls_runnable_run() -> None: diff --git a/tests/test_discovery.py b/tests/test_discovery.py index f0d271b..7d5797c 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -4,11 +4,11 @@ import pytest -from sampleflux.discovery import get_callable_path, introspect_callable, resolve_callable, scan_module +from recordstream.discovery import get_callable_path, introspect_callable, resolve_callable, scan_module def sample_func(a: int, b: str = "default") -> str: - """Sample docstring.""" + """Record docstring.""" return f"{a}-{b}" @@ -45,7 +45,7 @@ def test_resolve_callable_errors() -> None: # Test AttributeError with pytest.raises(AttributeError): - resolve_callable("sampleflux.discovery:nonexistent_func") + resolve_callable("recordstream.discovery:nonexistent_func") def test_introspect_errors() -> None: @@ -111,7 +111,7 @@ def test_resolve_callable_direct() -> None: def test_introspect_callable() -> None: schema = introspect_callable(sample_func) assert schema["name"] == "sample_func" - assert schema["doc"] == "Sample docstring." + assert schema["doc"] == "Record docstring." assert len(schema["parameters"]) == 2 p0 = schema["parameters"][0] @@ -160,7 +160,7 @@ class ClassInScript: assert "ClassInScript" in names # Standard module scan - schemas_self = scan_module("sampleflux.discovery") + schemas_self = scan_module("recordstream.discovery") names_self = [s["name"] for s in schemas_self] assert "scan_module" in names_self assert "get_callable_path" in names_self diff --git a/tests/test_dispatch.py b/tests/test_dispatch.py index 2eea385..070d9c0 100644 --- a/tests/test_dispatch.py +++ b/tests/test_dispatch.py @@ -2,8 +2,8 @@ from typing import Any, Dict -from sampleflux import Image, Label, Mask, Regions, Transform -from sampleflux.dispatch import dispatch, get_kernel, register_kernel, registered_kernels +from recordstream import Image, Label, Mask, Regions, Transform +from recordstream.dispatch import dispatch, get_kernel, register_kernel, registered_kernels from tests._fixtures import FixtureFlip diff --git a/tests/test_enable.py b/tests/test_enable.py index b97b6fe..e525991 100644 --- a/tests/test_enable.py +++ b/tests/test_enable.py @@ -9,7 +9,7 @@ import pytest -from sampleflux.ops.enable import Enable +from recordstream.ops.enable import Enable def _tag(record): diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py index 1a8da78..ae89664 100644 --- a/tests/test_entrypoint.py +++ b/tests/test_entrypoint.py @@ -1,6 +1,6 @@ """Tests for the runnable entry-point marker (entrypoint / runnable_entrypoints).""" -from sampleflux.runnable import entrypoint, entrypoint_tasks, runnable_entrypoints +from recordstream.runnable import entrypoint, entrypoint_tasks, runnable_entrypoints class _Runnable: diff --git a/tests/test_io.py b/tests/test_io.py index f1a0c52..31b8fe6 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,4 +1,4 @@ -"""The item codec registry (``sampleflux.io``) — default structural codec, overrides, records, +"""The item codec registry (``recordstream.io``) — default structural codec, overrides, records, the ``"plain"`` codec path.""" from dataclasses import dataclass @@ -6,7 +6,7 @@ import numpy as np import pytest -from sampleflux import ( +from recordstream import ( EncodedItem, Image, Label, @@ -18,7 +18,7 @@ register_io, register_item, ) -from sampleflux.io import PLAIN_TYPE +from recordstream.io import PLAIN_TYPE @register_item diff --git a/tests/test_items.py b/tests/test_items.py index e18f282..59406a0 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -1,6 +1,6 @@ """Typed items — array-subclass attribute preservation, wrappers, payload accessors, registry. -Only the MODALITY-NEUTRAL core items live in sampleflux (Image / Mask / Regions / Label). The +Only the MODALITY-NEUTRAL core items live in recordstream (Image / Mask / Regions / Label). The data-bearing-wrapper and multi-attribute-array paths (which the signal-domain items in a domain package exercise for real) are covered here with small test-local item types, so the core stays tested without importing a domain package. @@ -11,7 +11,7 @@ import numpy as np import pytest -from sampleflux.items import ( +from recordstream.items import ( Image, Label, Mask, diff --git a/tests/test_labels.py b/tests/test_labels.py index cc08cdf..20c10e8 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -1,12 +1,12 @@ -"""Tests for :class:`sampleflux.labels.LabelMap` — the fittable name↔id label map.""" +"""Tests for :class:`recordstream.labels.LabelMap` — the fittable name↔id label map.""" import json import pytest -from sampleflux import Label -from sampleflux.labels import LabelMap -from sampleflux.ops.target import DecodeTarget, EncodeTarget +from recordstream import Label +from recordstream.labels import LabelMap +from recordstream.ops.target import DecodeTarget, EncodeTarget # --------------------------------------------------------------------------- # Construction & lazy validation @@ -78,7 +78,7 @@ def test_from_label_names_empty_raises() -> None: # --------------------------------------------------------------------------- -# encode_op / decode_op produce working sampleflux ops +# encode_op / decode_op produce working recordstream ops # --------------------------------------------------------------------------- diff --git a/tests/test_lazy_construction.py b/tests/test_lazy_construction.py index 4f65b5c..a6c3925 100644 --- a/tests/test_lazy_construction.py +++ b/tests/test_lazy_construction.py @@ -1,11 +1,11 @@ -"""Pins the "Lazy Initialization & Zero-Arg Construction" convention for ALL sampleflux configurables. +"""Pins the "Lazy Initialization & Zero-Arg Construction" convention for ALL recordstream configurables. -Every ``@configurable`` class in sampleflux MUST be constructible with no arguments and do no +Every ``@configurable`` class in recordstream MUST be constructible with no arguments and do no functional work in ``__init__`` (no I/O, no network, no eager materialization). This walks the whole package, discovers every ``@configurable`` class, and asserts ``Cls()`` succeeds — so a newly-added class that violates the convention (a required ctor arg, or a constructor that opens a file / loads a dataset) fails here. See confluid ``AGENTS.md`` → "Lazy Initialization & Zero-Arg -Construction" and sampleflux ``AGENTS.md`` → "Lazy Evaluation". +Construction" and recordstream ``AGENTS.md`` → "Lazy Evaluation". """ import importlib @@ -14,13 +14,13 @@ import pytest -import sampleflux +import recordstream -def _all_sampleflux_configurables() -> List[type]: - """Import every sampleflux submodule and collect the ``@configurable`` classes defined in sampleflux.""" +def _all_recordstream_configurables() -> List[type]: + """Import every recordstream submodule and collect the ``@configurable`` classes defined in recordstream.""" seen: dict = {} - for modinfo in pkgutil.walk_packages(sampleflux.__path__, prefix="sampleflux."): + for modinfo in pkgutil.walk_packages(recordstream.__path__, prefix="recordstream."): try: module = importlib.import_module(modinfo.name) except Exception: # pragma: no cover - optional/heavy deps absent in some envs @@ -29,18 +29,18 @@ def _all_sampleflux_configurables() -> List[type]: if ( isinstance(obj, type) and getattr(obj, "__confluid_configurable__", False) - and getattr(obj, "__module__", "").startswith("sampleflux") + and getattr(obj, "__module__", "").startswith("recordstream") ): seen[f"{obj.__module__}.{obj.__qualname__}"] = obj return list(seen.values()) -_CONFIGURABLES = _all_sampleflux_configurables() +_CONFIGURABLES = _all_recordstream_configurables() def test_discovery_found_the_configurables() -> None: # Guard against the walker silently finding nothing (which would make the parametrized - # test below vacuously pass). sampleflux has well over a dozen @configurable classes. + # test below vacuously pass). recordstream has well over a dozen @configurable classes. assert len(_CONFIGURABLES) >= 20 @@ -55,7 +55,7 @@ def test_zero_arg_construction(cls: type) -> None: def test_sources_do_not_materialize_on_construction() -> None: # The lazy caches stay empty until first use — no dataset load / partition / offset compute # happens in __init__. - from sampleflux.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource + from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource assert HuggingFaceSource()._dataset is None assert DatasetSplit()._views == {} diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index 7415d88..bf7f381 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -1,4 +1,4 @@ -"""Guard: every node-facing sampleflux Source/Op documents all its constructor params. +"""Guard: every node-facing recordstream Source/Op documents all its constructor params. These classes surface in visual editors (as widget tooltips) and MCP form-specs (as pydantic ``Field(description=...)``) purely from their docstring ``Args:`` block — see @@ -12,31 +12,26 @@ import pytest from confluid import parse_param_docs # type: ignore[import-not-found] -from sampleflux import Pipeline, Transform -from sampleflux.core import FilterOp, Flux, JointFlux, WrappedOp -from sampleflux.ops.configure import ConfigureOp -from sampleflux.ops.context import Apply, Capture, Drop, MergeFields, Save, Use -from sampleflux.ops.debug import PrintSampleOp -from sampleflux.ops.enable import Enable -from sampleflux.ops.formula import FormulaOp -from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.numpy import ConnectedComponents, Threshold -from sampleflux.ops.parallel import Parallel -from sampleflux.ops.random_apply import RandomApply -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields -from sampleflux.ops.target import ( - CocoToTorchVisionDetection, - DecodeTarget, - EncodeTarget, - MasksToDetectionBoxes, -) -from sampleflux.ops.torch import ToTensor -from sampleflux.sources import HuggingFaceSource +from recordstream import Pipeline, Transform +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp +from recordstream.ops.configure import ConfigureOp +from recordstream.ops.context import Apply, Capture, Drop, MergeFields, Save, Use +from recordstream.ops.debug import PrintRecordOp +from recordstream.ops.enable import Enable +from recordstream.ops.formula import FormulaOp +from recordstream.ops.image import ConvertToImage +from recordstream.ops.numpy import ConnectedComponents, Threshold +from recordstream.ops.parallel import Parallel +from recordstream.ops.random_apply import RandomApply +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes +from recordstream.ops.torch import ToTensor +from recordstream.sources import HuggingFaceSource _NODE_CLASSES = [ HuggingFaceSource, - Flux, - JointFlux, + Stream, + JointStream, FilterOp, WrappedOp, Transform, @@ -64,7 +59,7 @@ Enable, Parallel, RandomApply, - PrintSampleOp, + PrintRecordOp, ] diff --git a/tests/test_op_families.py b/tests/test_op_families.py index 9dd64a6..4d5520a 100644 --- a/tests/test_op_families.py +++ b/tests/test_op_families.py @@ -2,7 +2,7 @@ Pins the record-model headline: native type-dispatched ops, BARE albumentations transforms (kwarg-vocabulary call, one joint draw, item re-wrap), and BARE torchvision ``transforms.v2`` -transforms (dict call) all sit in ONE ``Flux.ops`` list with no wrapper/adapter classes — +transforms (dict call) all sit in ONE ``Stream.ops`` list with no wrapper/adapter classes — plus the family classifiers, YAML mapping-form ops docs, spawn-parallel with a bare library op, ``field=`` targeting, and the ``WrappedOp``/``FilterOp`` raw-callable routes. """ @@ -17,8 +17,8 @@ from confluid import configurable from torchvision.transforms import v2 -from sampleflux import FilterOp, Image, Label, Mask, Pipeline, Record, Transform, WrappedOp -from sampleflux.core import Flux, _apply_op, _is_albumentations, _is_torchvision_v2 +from recordstream import FilterOp, Image, Label, Mask, Pipeline, Record, Transform, WrappedOp +from recordstream.core import Stream, _apply_op, _is_albumentations, _is_torchvision_v2 # --------------------------------------------------------------------------- # @@ -128,7 +128,7 @@ def test_extra_record_entries_never_reach_the_library(self) -> None: # --------------------------------------------------------------------------- # -# Mixed ops list end-to-end through Flux. +# Mixed ops list end-to-end through Stream. # --------------------------------------------------------------------------- # class TestMixedOpsList: _OPS = [ @@ -139,15 +139,15 @@ class TestMixedOpsList: ] def test_iteration(self) -> None: - flux = Flux(source=[_base_record()], ops=list(self._OPS)) - (out,) = list(flux) + stream = Stream(source=[_base_record()], ops=list(self._OPS)) + (out,) = list(stream) assert isinstance(out["image"], torch.Tensor) assert tuple(out["image"].shape) == (3, 4, 4) assert out["class"].value == "drone_x" # rode through every family untouched def test_random_access(self) -> None: - flux = Flux(source=[_base_record(0), _base_record(1)], ops=list(self._OPS)) - out = flux[1] + stream = Stream(source=[_base_record(0), _base_record(1)], ops=list(self._OPS)) + out = stream[1] assert isinstance(out["image"], torch.Tensor) and tuple(out["image"].shape) == (3, 4, 4) def test_pipeline_nests_the_same_families(self) -> None: @@ -164,8 +164,8 @@ def test_mapping_form_bare_albumentations_entry(self, tmp_path: Path) -> None: path = tmp_path / "ops.yaml" path.write_text("ops:\n - !class:albumentations.HorizontalFlip {p: 1.0}\n") record = _base_record() - flux = Flux.from_ops_yaml(str(path), source=[record]) - (out,) = list(flux) + stream = Stream.from_ops_yaml(str(path), source=[record]) + (out,) = list(stream) assert np.array_equal(np.asarray(out["image"]), np.asarray(record["image"])[:, ::-1]) assert isinstance(out["image"], Image) @@ -175,10 +175,10 @@ def test_configurable_ctor_param_bound_from_yaml(self, tmp_path: Path) -> None: path = tmp_path / "ops.yaml" path.write_text("ops:\n - !class:tests.test_op_families.AddOffset {offset: 3.0}\n") record = {"image": Image(np.zeros((2, 3, 3), dtype=np.float32))} - flux = Flux.from_ops_yaml(str(path), source=[record]) - (out,) = list(flux) # a @configurable entry stays a deferred marker until route entry + stream = Stream.from_ops_yaml(str(path), source=[record]) + (out,) = list(stream) # a @configurable entry stays a deferred marker until route entry assert np.allclose(np.asarray(out["image"]), 3.0) - (op,) = flux.ops # _check_ops_materialized flowed + cached the live op in place + (op,) = stream.ops # _check_ops_materialized flowed + cached the live op in place assert isinstance(op, AddOffset) and op.offset == 3.0 @@ -187,8 +187,8 @@ def test_configurable_ctor_param_bound_from_yaml(self, tmp_path: Path) -> None: # --------------------------------------------------------------------------- # def test_spawn_parallel_with_bare_albumentations_op() -> None: records = [{**r, "idx": i} for i, r in enumerate(spawn_records())] - flux = Flux(source=records, ops=[A.HorizontalFlip(p=1.0), FilterOp(keep_even_gain)]).parallel(2) - results = flux.collect() + stream = Stream(source=records, ops=[A.HorizontalFlip(p=1.0), FilterOp(keep_even_gain)]).parallel(2) + results = stream.collect() assert [int(r["idx"]) for r in results] == [0, 2] # FilterOp dropped odd records in workers for out, want in zip(results, [records[0], records[2]]): assert isinstance(out["image"], Image) # item type survived pickle + re-wrap @@ -210,7 +210,7 @@ def test_field_targets_one_of_two_image_keys() -> None: # --------------------------------------------------------------------------- # -# Raw-callable routes: WrappedOp / Flux.map / FilterOp drops everywhere. +# Raw-callable routes: WrappedOp / Stream.map / FilterOp drops everywhere. # --------------------------------------------------------------------------- # def double(x: np.ndarray) -> np.ndarray: """Module-level payload function for WrappedOp (stored as an importable path).""" @@ -242,29 +242,29 @@ def test_missing_key_raises(self) -> None: with pytest.raises(KeyError, match="nope"): WrappedOp(double, key="nope")({"m": Mask(np.ones(2))}) - def test_flux_map_key(self) -> None: - flux = Flux(source=[{"m": Mask(np.ones((2, 2)))}]).map(double, key="m") - (out,) = list(flux) + def test_stream_map_key(self) -> None: + stream = Stream(source=[{"m": Mask(np.ones((2, 2)))}]).map(double, key="m") + (out,) = list(stream) assert isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 2.0) class TestFilterDropRoutes: def test_sequential_iteration_drops(self) -> None: records = [{"i": 0}, {"i": 1}, {"i": 2}] - flux = Flux(source=records, ops=[FilterOp(lambda r: r["i"] != 1)]) - assert [r["i"] for r in flux] == [0, 2] + stream = Stream(source=records, ops=[FilterOp(lambda r: r["i"] != 1)]) + assert [r["i"] for r in stream] == [0, 2] def test_getitem_on_filtered_record_raises_index_error(self) -> None: - flux = Flux(source=[{"i": 0}], ops=[FilterOp(lambda r: False)]) + stream = Stream(source=[{"i": 0}], ops=[FilterOp(lambda r: False)]) with pytest.raises(IndexError, match="filtered out"): - flux[0] + stream[0] def test_pipeline_propagates_drop(self) -> None: assert Pipeline([FilterOp(lambda r: False)])({"i": 0}) is None - def test_flux_filter_helper(self) -> None: - flux = Flux(source=[{"i": 0}, {"i": 1}]).filter(lambda r: r["i"] > 0) - assert [r["i"] for r in flux] == [1] + def test_stream_filter_helper(self) -> None: + stream = Stream(source=[{"i": 0}, {"i": 1}]).filter(lambda r: r["i"] > 0) + assert [r["i"] for r in stream] == [1] def test_unset_predicate_raises_lazily(self) -> None: with pytest.raises(ValueError, match="predicate"): @@ -300,7 +300,7 @@ def invoke_fakelib_override(record: Record, op: FakeLibScale) -> Record: @pytest.fixture() def family_registry(): """Snapshot/restore the global registry so registrations never leak between tests.""" - from sampleflux import core + from recordstream import core snapshot = list(core._OP_FAMILIES) yield @@ -309,27 +309,27 @@ def family_registry(): class TestOpFamilyRegistry: def test_builtins_are_registered_through_the_same_registry(self) -> None: - from sampleflux import registered_op_families + from recordstream import registered_op_families assert registered_op_families()[:2] == ("albumentations", "torchvision_v2") def test_registered_family_dispatches_via_invoker(self, family_registry) -> None: - from sampleflux import register_op_family + from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) out = _apply_op(_base_record(), FakeLibScale(factor=3.0)) assert out is not None and out["gain_db"] == -9.0 # -3.0 * 3 — via the invoker, op never called assert isinstance(out["image"], Image) # rest of the record untouched - def test_registered_family_runs_in_flux_ops_list(self, family_registry) -> None: - from sampleflux import register_op_family + def test_registered_family_runs_in_stream_ops_list(self, family_registry) -> None: + from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) - out = list(Flux(source=[_base_record()], ops=[FakeLibScale(factor=2.0), lambda r: {**r, "tag": 1}])) + out = list(Stream(source=[_base_record()], ops=[FakeLibScale(factor=2.0), lambda r: {**r, "tag": 1}])) assert out[0]["gain_db"] == -6.0 and out[0]["tag"] == 1 # mixes with native ops in ONE list def test_last_registered_family_wins_overlap(self, family_registry) -> None: - from sampleflux import register_op_family + from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) register_op_family("fakelib_specific", is_fakelib, invoke_fakelib_override) # same matcher, later @@ -337,7 +337,7 @@ def test_last_registered_family_wins_overlap(self, family_registry) -> None: assert out is not None and out["gain_db"] == -999.0 def test_reregistering_name_replaces_in_place(self, family_registry) -> None: - from sampleflux import register_op_family, registered_op_families + from recordstream import register_op_family, registered_op_families register_op_family("fakelib", is_fakelib, invoke_fakelib) n = len(registered_op_families()) @@ -351,11 +351,11 @@ def test_unmatched_op_falls_back_to_native_call(self, family_registry) -> None: assert out is not None and out["native"] is True def test_spawn_parallel_ships_family_to_workers(self, family_registry) -> None: - from sampleflux import register_op_family + from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) - flux = Flux(source=spawn_records(), ops=[FakeLibScale(factor=2.0)]).parallel(2) - results = list(flux) + stream = Stream(source=spawn_records(), ops=[FakeLibScale(factor=2.0)]).parallel(2) + results = list(stream) assert len(results) == 4 assert all(r["gain_db"] == -6.0 for r in results) # invoker ran INSIDE the workers @@ -363,7 +363,7 @@ def test_spawn_parallel_ships_family_to_workers(self, family_registry) -> None: class TestFormulaReducers: def test_array_reducers_are_function_style(self) -> None: # amax/amin/mean/std/median are pre-bound numpy callables in the sandbox namespace. - from sampleflux.ops.formula import FormulaOp + from recordstream.ops.formula import FormulaOp rec = {"image": Image(np.arange(16, dtype=np.float32).reshape(4, 4) / 15.0)} out = FormulaOp(formula="amax(a) * 0.5", field="image")(rec) @@ -373,6 +373,6 @@ def test_attribute_reduction_is_not_part_of_the_contract(self) -> None: # a.max() depends on numpy's lazy-import cache (KeyError '__import__' in a cold # process): the FUNCTION form is the sanctioned spelling. We only pin that the # function form never regresses; the attribute form is deliberately unpinned. - from sampleflux.ops.formula import _FORMULA_NAMESPACE + from recordstream.ops.formula import _FORMULA_NAMESPACE assert {"amax", "amin", "mean", "std", "median"} <= set(_FORMULA_NAMESPACE) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 8713322..1e84b25 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -2,8 +2,8 @@ import numpy as np -from sampleflux import Image, item_data -from sampleflux.core import Flux +from recordstream import Image, item_data +from recordstream.core import Stream def heavy_op(x: np.ndarray) -> np.ndarray: @@ -16,7 +16,7 @@ def test_parallel_execution() -> None: start = time.time() # Use a real top-level function for pickling; key= targets the record entry's payload. - pipeline = Flux(source).map(heavy_op, key="x").parallel(workers=4) + pipeline = Stream(source).map(heavy_op, key="x").parallel(workers=4) results = pipeline.collect() duration = time.time() - start diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 73ed047..1565677 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -9,8 +9,8 @@ import numpy as np import pytest -from sampleflux import FilterOp, Image, Label, Mask, Pipeline, Record -from sampleflux.ops.structure import RenameField +from recordstream import FilterOp, Image, Label, Mask, Pipeline, Record +from recordstream.ops.structure import RenameField from tests._fixtures import FixtureFlip @@ -24,7 +24,7 @@ def test_package_imports_without_torchvision(self) -> None: # op-family dispatch detects them by MRO module NAME — no import), so discovery stays # safe on hosts missing the libraries. code = ( - "import sys; import sampleflux; " + "import sys; import recordstream; " "assert 'torchvision' not in sys.modules; " "assert 'albumentations' not in sys.modules" ) diff --git a/tests/test_runnable.py b/tests/test_runnable.py index 42b96ea..48d380e 100644 --- a/tests/test_runnable.py +++ b/tests/test_runnable.py @@ -2,7 +2,7 @@ from typing import List -from sampleflux.runnable import ProgressReporting, TorchRunner +from recordstream.runnable import ProgressReporting, TorchRunner def test_torch_runner_flag() -> None: diff --git a/tests/test_structure_ops.py b/tests/test_structure_ops.py index e261894..a97fd21 100644 --- a/tests/test_structure_ops.py +++ b/tests/test_structure_ops.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from sampleflux import Image, Label, Record, Regions -from sampleflux.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream import Image, Label, Record, Regions +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields def _record() -> Record: diff --git a/tests/test_transform.py b/tests/test_transform.py index 41b1da1..d9a712d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,6 +1,6 @@ """Transforms — type dispatch, once-per-record params, cross-key consistency, ``field=`` pin. -Native-kernel machinery is pinned via the test fixture ``FixtureFlip`` (sampleflux ships no +Native-kernel machinery is pinned via the test fixture ``FixtureFlip`` (recordstream ships no native augmentation transforms — libraries drop into ops lists bare, invoked natively by the engine's op-family dispatch). """ @@ -8,7 +8,7 @@ import numpy as np import pytest -from sampleflux import Image, Label, Mask, Pipeline, Record, Regions, Transform, as_transform +from recordstream import Image, Label, Mask, Pipeline, Record, Regions, Transform, as_transform from tests._fixtures import FixtureFlip diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py index 153401e..068ae9c 100644 --- a/tests/test_typed_collate.py +++ b/tests/test_typed_collate.py @@ -6,7 +6,7 @@ import pytest import torch -from sampleflux import Image, Label, Mask, Record, Regions, collate, collate_records, get_collate, register_item +from recordstream import Image, Label, Mask, Record, Regions, collate, collate_records, get_collate, register_item @register_item @@ -111,7 +111,7 @@ def test_generic_collate_leaves_ragged_regions_as_lists() -> None: def test_registered_task_collate_produces_its_own_batch_contract() -> None: import torch - from sampleflux import collate, get_collate, register_collate + from recordstream import collate, get_collate, register_collate @register_collate("_test_detection") def detection_collate(items): diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py index 9ce8f69..753a431 100644 --- a/tests/test_typed_detection_target_ops.py +++ b/tests/test_typed_detection_target_ops.py @@ -1,14 +1,14 @@ """The two detection target-shaping ops over dict records. Pins the native transforms that build a detection pipeline's torchvision-style -``{boxes, labels}`` target as a :class:`~sampleflux.Regions` item: +``{boxes, labels}`` target as a :class:`~recordstream.Regions` item: -* :class:`sampleflux.ops.target.CocoToTorchVisionDetection` — a HuggingFace / COCO ``objects`` +* :class:`recordstream.ops.target.CocoToTorchVisionDetection` — a HuggingFace / COCO ``objects`` annotation → a target ``Regions``; -* :class:`sampleflux.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Regions``. +* :class:`recordstream.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Regions``. Each op REUSES its conversion helper, so the op's ``boxes`` / ``labels`` tensors are pinned -byte-identical to the helper (parity). sampleflux-only — no domain-package import. +byte-identical to the helper (parity). recordstream-only — no domain-package import. """ import numpy as np @@ -16,8 +16,8 @@ import torch from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Label, Mask, Regions, collate_records -from sampleflux.ops.target import ( +from recordstream import Image, Label, Mask, Regions, collate_records +from recordstream.ops.target import ( CocoToTorchVisionDetection, MasksToDetectionBoxes, coco_to_detection, diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index 1a813d4..f198c86 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -5,10 +5,10 @@ import numpy as np import pytest -from sampleflux import FlowGraph, Flux, Image, Label, Mask, Pipeline, Record, Transform, to_ops -from sampleflux.flow import from_ops, parse_flow -from sampleflux.ops.context import MergeFields -from sampleflux.ops.structure import RenameField, SelectFields +from recordstream import FlowGraph, Image, Label, Mask, Pipeline, Record, Stream, Transform, to_ops +from recordstream.flow import from_ops, parse_flow +from recordstream.ops.context import MergeFields +from recordstream.ops.structure import RenameField, SelectFields class _AddOffset(Transform): @@ -159,11 +159,11 @@ def _flow(self) -> Dict[str, Any]: "out": {"from": "boosted", "merge_from": ["mask_only"]}, } - def test_to_ops_runs_on_flux(self) -> None: + def test_to_ops_runs_on_stream(self) -> None: # The lowered flat op list (MergeFields wiring) matches the native FlowGraph result. steps, outputs = parse_flow(self._flow()) native = list(FlowGraph(source=[_seed(0.25)], flow=self._flow(), outputs="out")) - lowered = list(Flux(source=[_seed(0.25)], ops=to_ops(steps, outputs))) + lowered = list(Stream(source=[_seed(0.25)], ops=to_ops(steps, outputs))) assert len(native) == len(lowered) == 1 assert list(native[0].keys()) == list(lowered[0].keys()) assert np.array_equal(np.asarray(native[0]["image"]), np.asarray(lowered[0]["image"])) @@ -175,7 +175,7 @@ def test_round_trip_from_ops(self) -> None: assert any(isinstance(op, MergeFields) for op in ops) lifted, lifted_out = from_ops(ops) relowered = to_ops(*parse_flow(lifted, lifted_out)) - native = list(Flux(source=[_seed(0.5)], ops=relowered)) + native = list(Stream(source=[_seed(0.5)], ops=relowered)) assert len(native) == 1 and "mask" in native[0] def test_key_bind_round_trip(self) -> None: @@ -198,25 +198,27 @@ def __call__(self, record: Record) -> Record: # the key-bind grammar survives the round trip final_step = lifted["final"] if "final" in lifted else list(lifted.values())[-1] assert isinstance(final_step, dict) and final_step["bind"]["item"].endswith("[image]") - (out,) = list(Flux(source=[_seed(0.0)], ops=ops)) + (out,) = list(Stream(source=[_seed(0.0)], ops=ops)) assert np.allclose(np.asarray(out["echo"]), 3.0) -class TestRecordsThroughFlux: - def test_flux_carries_record_dicts_verbatim(self) -> None: - flux = Flux(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) - (out,) = list(flux) +class TestRecordsThroughStream: + def test_stream_carries_record_dicts_verbatim(self) -> None: + stream = Stream(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) + (out,) = list(stream) assert isinstance(out, dict) and np.allclose(np.asarray(out["image"]), 2.0) def test_getitem(self) -> None: - flux = Flux(source=[_seed(1.0), _seed(2.0)], ops=[RenameField(src="label", dst="klass")]) - out = flux[1] + stream = Stream(source=[_seed(1.0), _seed(2.0)], ops=[RenameField(src="label", dst="klass")]) + out = stream[1] assert "klass" in out and np.allclose(np.asarray(out["image"]), 2.0) def test_compose_ops_route_records(self) -> None: # Pipeline (the compose-group grouping op — TransformChain's replacement). - flux = Flux(source=[_seed(1.0)], ops=[Pipeline(transforms=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])]) - (out,) = list(flux) + stream = Stream( + source=[_seed(1.0)], ops=[Pipeline(transforms=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])] + ) + (out,) = list(stream) assert np.allclose(np.asarray(out["image"]), 4.0) @@ -229,10 +231,10 @@ def _doc(self) -> str: return """ flow: spec: {} - masked: !class:sampleflux.ops.numpy.Threshold {low_level: 0.5, from: spec} - thresh: !class:sampleflux.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} + masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} + thresh: !class:recordstream.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} gated: - op: !class:sampleflux.ops.numpy.Threshold {output: gated_mask} + op: !class:recordstream.ops.numpy.Threshold {output: gated_mask} from: spec bind: low_level: thresh[image] @@ -254,11 +256,11 @@ def test_yaml_bind_via_plain_mapping_step(self, tmp_path) -> None: assert int(np.asarray(out["mask"]).sum()) == 8 # fixed 0.5 threshold assert int(np.asarray(out["gated_mask"]).sum()) == 6 # per-record amax(a)*0.6 bind - def test_yaml_bind_parity_with_lowered_flux(self, tmp_path) -> None: + def test_yaml_bind_parity_with_lowered_stream(self, tmp_path) -> None: path = tmp_path / "graph.yaml" path.write_text(self._doc()) a = list(FlowGraph.from_yaml(str(path), source=[self._record()]))[0] - b = list(Flux.from_flow_yaml(str(path), source=[self._record()]))[0] + b = list(Stream.from_flow_yaml(str(path), source=[self._record()]))[0] assert np.array_equal(np.asarray(a["gated_mask"]), np.asarray(b["gated_mask"])) assert np.array_equal(np.asarray(a["mask"]), np.asarray(b["mask"])) @@ -270,7 +272,7 @@ def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path) -> None """ flow: spec: {} - gated: !class:sampleflux.ops.numpy.Threshold + gated: !class:recordstream.ops.numpy.Threshold from: spec bind: low_level: spec[image] diff --git a/tests/test_typed_generic_ops.py b/tests/test_typed_generic_ops.py index 0aeccd4..e2bb95a 100644 --- a/tests/test_typed_generic_ops.py +++ b/tests/test_typed_generic_ops.py @@ -3,21 +3,21 @@ Pins the three native type-changing transforms that run the detection/segmentation front-end on plain record dicts: -* :class:`sampleflux.ops.image.ConvertToImage` — array-bearing key → ``Image`` item; -* :class:`sampleflux.ops.numpy.Threshold` — array key → boolean ``Mask`` item; -* :class:`sampleflux.ops.numpy.ConnectedComponents` — ``Mask`` → ``Regions`` item. +* :class:`recordstream.ops.image.ConvertToImage` — array-bearing key → ``Image`` item; +* :class:`recordstream.ops.numpy.Threshold` — array key → boolean ``Mask`` item; +* :class:`recordstream.ops.numpy.ConnectedComponents` — ``Mask`` → ``Regions`` item. Each op REUSES its shared math helper, so the op output is pinned identical to the helper -(parity). sampleflux-only — no domain-package import. +(parity). recordstream-only — no domain-package import. """ import numpy as np import pytest from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Mask, Regions -from sampleflux.ops.image import ConvertToImage, _bound_longest_side, _render_rgb -from sampleflux.ops.numpy import ConnectedComponents, Threshold, connected_component_bboxes, threshold_array +from recordstream import Image, Mask, Regions +from recordstream.ops.image import ConvertToImage, _bound_longest_side, _render_rgb +from recordstream.ops.numpy import ConnectedComponents, Threshold, connected_component_bboxes, threshold_array def _ramp_2d() -> np.ndarray: diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py index cd5eaa2..e59c0d9 100644 --- a/tests/test_typed_storage.py +++ b/tests/test_typed_storage.py @@ -7,12 +7,12 @@ import numpy as np import pytest -from sampleflux import Image, Label, Regions, register_item -from sampleflux.storage.base import TYPED_FORMAT, require_record_format, restore_attrs, split_attrs -from sampleflux.storage.directory import DirectorySink, DirectorySource -from sampleflux.storage.hdf5 import HDF5Sink, HDF5Source -from sampleflux.storage.query import MetadataFilterSource, record_metadata, scan_hdf5_metadata, scan_zarr_metadata -from sampleflux.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource +from recordstream import Image, Label, Regions, register_item +from recordstream.storage.base import TYPED_FORMAT, require_record_format, restore_attrs, split_attrs +from recordstream.storage.directory import DirectorySink, DirectorySource +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source +from recordstream.storage.query import MetadataFilterSource, record_metadata, scan_hdf5_metadata, scan_zarr_metadata +from recordstream.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource @register_item @@ -106,14 +106,14 @@ def test_hdf5_source_rejects_old_typedsample_tag(self, tmp_path: Path) -> None: # NO backward compat: a pre-record-model store must fail loudly with the clear error. path = tmp_path / "old.h5" with h5py.File(path, "w") as handle: - handle.attrs["sampleflux_format"] = "typedsample-v1" + handle.attrs["recordstream_format"] = "typedsample-v1" with pytest.raises(ValueError, match="typedrecord-v1"): HDF5Source(path=path).open() def test_hdf5_sink_rejects_appending_to_old_tag(self, tmp_path: Path) -> None: path = tmp_path / "old.h5" with h5py.File(path, "w") as handle: - handle.attrs["sampleflux_format"] = "typedsample-v1" + handle.attrs["recordstream_format"] = "typedsample-v1" sink = HDF5Sink(path=path) with sink: with pytest.raises(ValueError, match="typedrecord-v1"): @@ -124,7 +124,7 @@ def test_zarr_group_source_rejects_old_tag(self, tmp_path: Path) -> None: path = str(tmp_path / "old.zarr") root = zarr.open_group(path, mode="a") - root.attrs["sampleflux_format"] = "typedsample-v1" + root.attrs["recordstream_format"] = "typedsample-v1" with pytest.raises(ValueError, match="typedrecord-v1"): ZarrGroupSource(path=path).open() @@ -147,7 +147,7 @@ def test_format_tag_stamped(self, tmp_path: Path) -> None: with HDF5Sink(path=path, overwrite=True) as sink: sink.write(_records()[0]) with h5py.File(path, "r") as handle: - assert handle.attrs["sampleflux_format"] == TYPED_FORMAT + assert handle.attrs["recordstream_format"] == TYPED_FORMAT def test_non_dict_write_raises(self, tmp_path: Path) -> None: # The sink only accepts a record dict; a bare array is rejected loudly. diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py index d90e41c..683d391 100644 --- a/tests/test_typed_target_ops.py +++ b/tests/test_typed_target_ops.py @@ -3,11 +3,12 @@ Pins the native transforms that build a classification pipeline's model INPUT array and its encoded TARGET ``Label`` on plain record dicts: -* :class:`sampleflux.ops.torch.ToTensor` — array-bearing key → a LIVE CHW-float ``torch.Tensor`` (a plain record value); -* :class:`sampleflux.ops.target.EncodeTarget` / ``DecodeTarget`` — class-name ↔ class-id ``Label``. +* :class:`recordstream.ops.torch.ToTensor` — array-bearing key → a LIVE CHW-float ``torch.Tensor`` + (a plain record value); +* :class:`recordstream.ops.target.EncodeTarget` / ``DecodeTarget`` — class-name ↔ class-id ``Label``. Each op REUSES its shared conversion helper, so the op output is pinned identical to the -helper (parity). sampleflux-only — no domain-package import. +helper (parity). recordstream-only — no domain-package import. """ import numpy as np @@ -15,10 +16,10 @@ import torch from confluid.registry import get_registry, resolve_class -from sampleflux import Image, Label, Mask, collate_records, item_data -from sampleflux.ops.image import ConvertToImage -from sampleflux.ops.target import DecodeTarget, EncodeTarget -from sampleflux.ops.torch import ToTensor, to_tensor +from recordstream import Image, Label, Mask, collate_records, item_data +from recordstream.ops.image import ConvertToImage +from recordstream.ops.target import DecodeTarget, EncodeTarget +from recordstream.ops.torch import ToTensor, to_tensor _MAP = {"cat": 0, "dog": 1, "fox": 2} _INV = {0: "cat", 1: "dog", 2: "fox"} @@ -157,7 +158,7 @@ def test_encode_no_label_field_raises(self) -> None: # --------------------------------------------------------------------------- # -# End-to-end classification input/target path (sampleflux-only). +# End-to-end classification input/target path (recordstream-only). # --------------------------------------------------------------------------- # def test_classification_input_and_target_chain() -> None: # Source-shaped record: an HWC image + a class-NAME label — key names carry meaning. diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 97d8e4e..d5a7b84 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -1,4 +1,4 @@ -"""Tests for sampleflux.workflow — higher-order runnable combinators. +"""Tests for recordstream.workflow — higher-order runnable combinators. Covers Sequence / Conditional / Switch orchestration, the PathExists / Not / AllOf / AnyOf predicates, zero-arg construction (the lazy-construction mandate), @@ -13,7 +13,7 @@ import pytest from confluid import configurable -from sampleflux.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch +from recordstream.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch # A module-global run log so the @configurable runnables below survive a Confluid # round-trip: their only config is a tag, and run() appends it here. @@ -190,18 +190,18 @@ def test_predicate_zero_arg_construct_and_call(cls: Any) -> None: # --------------------------------------------------------------------------- # -# FluxStudio canvas integration — TorchRunner + ProgressReporting forwarding +# StreamStudio canvas integration — TorchRunner + ProgressReporting forwarding # --------------------------------------------------------------------------- # @pytest.mark.parametrize("cls", [Sequence, Conditional, Switch]) def test_combinators_declare_torch_runner(cls: Any) -> None: - # A combinator may wrap a trainer, so it declares __torch_runner__ — FluxStudio's executor + # A combinator may wrap a trainer, so it declares __torch_runner__ — StreamStudio's executor # re-enables autograd for the whole run (otherwise the inner loss.backward() dies under # ComfyUI's inference_mode). assert cls().__torch_runner__ is True def test_progress_callback_forwarded_to_running_branch() -> None: - from sampleflux.runnable import ProgressReporting + from recordstream.runnable import ProgressReporting received: List[str] = [] From 7ad371b6ee43de1068f345264e9f000b0d11edb5 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 27 Jul 2026 17:05:20 +0200 Subject: [PATCH 044/102] refactor(enable)!: make the toggle a declared `enabled` parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: `Enable`'s dynamic toggle naming is retired. A wrapper is now spelled `name: visualize` + `enabled: false` and toggled with `--visualize.enabled true`; the old bare `visualize: false` kwarg raises a ValueError on first record with the replacement spelling in the message. The old design made ANY boolean instance attribute the toggle, which only YAML could reach — the loader is the sole channel for undeclared keys. Every other front-end read the signature and came up empty: `to_pydantic(Enable)` exposed only `ops` (schema/form/canvas built a node with no widget), `Enable(ops=[...], visualize=True)` raised "Extra inputs are not permitted", and `accepts_key(Enable, "visualize")` was False so the advertised bare broadcast was silently dropped. `enabled: bool = True` is now a declared, defaulted parameter exposed as a settable property (non-bool raises TypeError, so a quoted YAML `"true"` fails at its file:line instead of being silently truthy), and `name` scopes the CLI flag. `flag_name` is gone. Docs: README gets a usage section, architecture.md a decision record (§6), AGENTS.md the generalised mandate — if a front-end must set it, declare it. Also: `Storage.open`/`__enter__` return `Self` so `with HDF5Sink(...) as s` keeps the concrete backend type, and `DirectorySource` declares the `DataSource` protocol it already implemented. --- AGENTS.md | 1 + README.md | 28 ++++++ docs/architecture.md | 85 ++++++++++++++++ examples/storage_roundtrip.py | 8 +- recordstream/ops/enable.py | 156 ++++++++++++++---------------- recordstream/storage/base.py | 8 +- recordstream/storage/directory.py | 3 +- tests/test_enable.py | 141 +++++++++++++++++++-------- tests/test_op_families.py | 16 +-- tests/test_typed_collate.py | 5 +- tests/test_typed_flow.py | 9 +- 11 files changed, 320 insertions(+), 140 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 31fa280..217888c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). - **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. diff --git a/README.md b/README.md index 6b9eae9..88ce038 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,34 @@ ops: low_level: 0.5 ``` +### Toggling a branch from the CLI (`Enable`) + +Wrap any stretch of an ops list in `Enable` to switch the whole chain on or off from one flag. +The toggle is the declared `enabled` parameter; `name` identifies the wrapper so several of them +toggle independently: + +```yaml +ops: + - !class:recordstream.ops.numpy.Threshold {low_level: 0.5} + - !class:recordstream.ops.enable.Enable + name: visualize # ← names THIS wrapper; scopes its CLI flag + enabled: false # ← off by default; the chain below is skipped + ops: + - !class:recordstream.ops.image.ConvertToImage {} + - !class:recordstream.ops.debug.PrintRecordOp {} +``` + +```bash +recordstream run pipeline.yaml --visualize.enabled true # this wrapper only +recordstream run pipeline.yaml --visualize.enabled+ # polarity shorthand → True +recordstream run pipeline.yaml --enabled false # broadcast: every Enable off +``` + +Inner ops are not materialized until the wrapper first fires, so gating an expensive chain with +`enabled: false` costs nothing at startup. In Python the same wrapper is one call — +`Enable(ops=[...], name="visualize", enabled=False)` — which is what lets a visual editor or a +generated tool schema set the toggle too (see [docs/architecture.md](docs/architecture.md#6-every-knob-is-a-declared-parameter--the-enable-toggle-2026-07-27)). + ## 📚 Documentation | Page | Covers | diff --git a/docs/architecture.md b/docs/architecture.md index 5c1f635..acad247 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -448,3 +448,88 @@ both = Stream.joint([stream_a, stream_b]) # Stream(source=JointS a category and group. - **Do not add a discovery category to `FilterOp`/`WrappedOp`** — surfacing a raw-callable parameter on a canvas is a dead widget; the taxonomy is pinned in `tests/test_categories.py`. + +## 6. Every knob is a DECLARED parameter — the `Enable` toggle (2026-07-27) + +### Context + +`Enable` gates an inner ops list behind one boolean. Its original design leaned on Confluid's +post-construction paradigm: **any** boolean attribute set on the instance was the toggle, and that +attribute's NAME became the CLI flag — `visualize: false` in YAML produced `--visualize`, and a +`name:` was only needed to disambiguate two wrappers. Nothing about the toggle was declared; it +existed purely as a runtime attribute Confluid setattr'd from an unrecognised YAML key. + +That works for exactly one front-end — hand-written YAML — because only the YAML loader has a +channel for undeclared keys. Every other caller reads the *signature*: + +- `to_pydantic(Enable).model_fields` returned `['ops']`, so a schema/form/canvas generator built a + node with no toggle at all — the wrapper rendered as a pass-through and then raised at runtime. +- `Enable(ops=[...], visualize=True)` raised `ValidationError: Extra inputs are not permitted` + (the generated config model forbids extras), so neither Python nor a generated tool call could + construct a toggled wrapper — the only spelling was construct-then-setattr. +- `confluid.accepts_key(Enable, "visualize")` was `False`, so liquifai *silently dropped* the bare + broadcast the docstring advertised (`--visualize true`). Only `--.` landed, and + only via the addressed branch's "the key is already in the YAML kwargs" escape hatch. + +A knob that only YAML can reach is a knob three of the four front-ends cannot offer. + +### Decision + +The toggle is a **declared, defaulted constructor parameter** — `enabled: bool = True` — exposed as +a **settable property**, and instance identity is the **declared `name`**, which scopes the flag to +`--.enabled`. Dynamic toggle naming is retired. + +A property rather than a plain attribute for two reasons: `confluid.accepts_key` admits "public +settable class attributes", so the property keeps `enabled` overridable independently of the +signature; and it gives ONE funnel to reject a non-bool, so a quoted YAML `enabled: "true"` fails +at its `file:line` instead of being silently truthy. + +The retired form is not silently ignored: a stray public boolean attribute (what `visualize: false` +now lands as) raises on first record with the replacement spelling in the message. + +### Consequences + +- One declaration serves every front-end: YAML key, `--enabled` / `--.enabled` override, + Python kwarg, generated tool/form schema, canvas widget. No front-end-specific glue. +- The generalisable rule: **if a front-end must set it, declare it.** A value that only ever + arrives via post-construction setattr is reachable from YAML alone. +- Breaking change: `visualize: false` (and any other dynamic toggle name) must become + `name: visualize` + `enabled: false`; the CLI flag becomes `--visualize.enabled`. +- `flag_name` is gone — with a fixed toggle name there is nothing to introspect. +- Strictness is deliberate: `enabled` accepts only `bool`. Every CLI form already delivers a real + bool (`--enabled true`, `--enabled=false`, `--enabled+`, `enabled=true`), so the rejection only + catches genuinely ambiguous config. + +### Example + +```yaml +- !class:recordstream.ops.enable.Enable + name: visualize + enabled: false + ops: [ !class:recordstream.ops.image.ConvertToImage {} ] +``` + +```bash +recordstream run pipeline.yaml --visualize.enabled true # addressed: this wrapper +recordstream run pipeline.yaml --enabled false # broadcast: every wrapper +``` + +```python +op = Enable(ops=[convert], name="visualize", enabled=False) # one call — no setattr step +op.enabled = True # property setter; non-bool raises + +to_pydantic(Enable).model_fields # {'ops', 'enabled', 'name'} — the schema surface +accepts_broadcast(Enable, "enabled") # True — the bare --enabled form now lands +``` + +### What you may change (and where it's documented) + +- **Adding a knob to any op**: declare it in `__init__` with a default and an `Args:` line. Reach + for post-construction setattr only for values a *config layer* injects, never for a user-facing + switch. Usage lives in the project README (`Toggling a branch from the CLI`). +- **Distinguishing instances**: use `name:` — Confluid reads it for hierarchy labelling and + liquifai for `--.` addressing. Do not invent a per-class flag vocabulary; that was + the retired design. +- **More than one switch in a chain**: use several `Enable` wrappers with distinct names rather + than teaching one wrapper several toggles — each name is independently addressable, and the + broadcast form still flips them all. diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py index 3e6e326..2823412 100644 --- a/examples/storage_roundtrip.py +++ b/examples/storage_roundtrip.py @@ -18,6 +18,7 @@ import tempfile from pathlib import Path +from typing import List, Tuple, Union import numpy as np @@ -27,6 +28,11 @@ from recordstream.storage.query import MetadataFilterSource from recordstream.storage.zarr import ZarrGroupSink, ZarrGroupSource +#: The three backends are independent classes (no shared sink/source base beyond ``Storage``), +#: so the pair list is spelled as a union to keep ``write``/``flush``/iteration typed. +AnySink = Union[HDF5Sink, ZarrGroupSink, DirectorySink] +AnySource = Union[HDF5Source, ZarrGroupSource, DirectorySource] + def make_records(n: int = 4) -> list: rng = np.random.default_rng(0) @@ -55,7 +61,7 @@ def main() -> None: with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) - pairs = [ + pairs: List[Tuple[AnySink, AnySource]] = [ (HDF5Sink(path=work / "store.h5"), HDF5Source(path=work / "store.h5")), (ZarrGroupSink(path=work / "store.zarr"), ZarrGroupSource(path=work / "store.zarr")), (DirectorySink(path=work / "store_dir"), DirectorySource(path=work / "store_dir")), diff --git a/recordstream/ops/enable.py b/recordstream/ops/enable.py index 29a6488..45dfe12 100644 --- a/recordstream/ops/enable.py +++ b/recordstream/ops/enable.py @@ -1,12 +1,12 @@ -"""``Enable`` — toggle one or more ops on/off via a single named CLI flag. +"""``Enable`` — toggle one or more ops on/off from a single declared flag. A compose-group op (alongside ``Pipeline`` / ``Parallel``): wrap an inner op-list -so the whole chain can be switched on or off from one boolean attribute whose -name becomes the CLI flag. Modality-neutral — it threads any record -through any ops — so it lives in core recordstream, not a domain package. +so the whole chain can be switched on or off from ONE boolean, ``enabled``. +Modality-neutral — it threads any record through any ops — so it lives in core +recordstream, not a domain package. """ -from typing import List, Optional, Tuple +from typing import Any, List, Optional from confluid import configurable from loggair import get_logger @@ -18,124 +18,114 @@ @configurable(category="op", group="compose") class Enable: - """Wrap one or more ops so they can be toggled on/off via a single named CLI flag. + """Wrap one or more ops so the whole chain can be switched on or off. ``ops`` is a list; even a single-op guard uses ``ops: [op]``. The wrapper threads each record through every op in sequence — same semantics as listing them inline in ``Stream.ops`` — so a whole visualization chain shares one toggle instead of needing a wrapper per op. - The toggle flag is supplied in YAML as an *extra* kwarg whose name becomes - the CLI hook — Confluid's post-construction setattr promotes it to an - instance attribute, and Liquify's ``-- `` overrides match - Fluid kwargs by name (see - :func:`liquifai.core._merge_overrides_into_fluids`). + The toggle is ``enabled``: a DECLARED constructor parameter exposed as a + settable property. Being declared is what makes it reachable from every + front-end — a YAML key, a CLI override, a Python kwarg, a generated + tool/form schema, a canvas widget — through the same introspection every + other ``@configurable`` parameter uses. There is no dynamic toggle-attribute + naming: an unrecognised boolean key on this class is an error, not a flag. + + Several wrappers in one pipeline are told apart by ``name``, which scopes + the CLI flag to that instance (``--.enabled``); a bare ``--enabled`` + still broadcasts to every wrapper at once. YAML: .. code-block:: yaml - !class:recordstream.ops.enable.Enable - visualize: false # ← any boolean attribute name works; this name IS the CLI flag + name: visualize # ← names THIS instance; scopes its CLI flag + enabled: false ops: - !class:recordstream.ops.image.ConvertToImage {} - - !class:waivefront.visualizers.SaveImage - output_dir: ./segments_png + - !class:recordstream.ops.debug.PrintRecordOp {} CLI: .. code-block:: bash - recordstream run pipeline.yaml --visualize true - recordstream run pipeline.yaml --visualize+ # polarity shorthand → True - recordstream run pipeline.yaml --visualize- # polarity shorthand → False - - Inner ops stay deferred (not materialized) until the wrapper actually - fires for the first time, so guarding expensive-to-construct ops with - ``Enable(..., visualize=False)`` costs nothing at startup. - - Disambiguating multiple wrappers - -------------------------------- - When two or more ``Enable`` instances live in the same pipeline, give - each a ``name:`` in YAML and use the GENERIC toggle name ``enable`` — - the name scopes the flag, so a semantic attribute name per wrapper is - unnecessary. ``name`` becomes the preferred identifier in Confluid's - hierarchy (``--help``) and Liquify's override matcher, so you can - toggle them independently: - - .. code-block:: yaml - - - !class:recordstream.ops.enable.Enable - name: overlay # dotted-override key - enable: false # generic toggle — the name scopes it - ops: [render-with-overlays, save-to ./debug_png] - - !class:recordstream.ops.enable.Enable - name: labelstudio - enable: false - ops: [render-clean, save-to ./ls_png] + # Targeted — only the wrapper named `visualize` flips. + recordstream run pipeline.yaml --visualize.enabled true + recordstream run pipeline.yaml --visualize.enabled+ # polarity shorthand → True + recordstream run pipeline.yaml --visualize.enabled- # polarity shorthand → False - CLI: + # Broadcast — every Enable in the config flips. + recordstream run pipeline.yaml --enabled false - .. code-block:: bash + Python: - # Targeted — only the overlay chain fires. - recordstream run pipeline.yaml --overlay.enable true - recordstream run pipeline.yaml --overlay.enable+ # polarity shorthand → True + .. code-block:: python - # Broadcast — every Fluid with an `enable` kwarg flips. - recordstream run pipeline.yaml --enable true + op = Enable(ops=[convert, save], name="visualize", enabled=False) + op.enabled = True # plain attribute write (validated: must be a bool) - ``name`` is a plain string on the instance; Confluid's post-construction - paradigm setattr's it automatically from YAML with no ctor change. + Inner ops stay deferred (not materialized) until the wrapper actually + fires for the first time, so guarding expensive-to-construct ops with + ``enabled: false`` costs nothing at startup. Constraints: * ``ops`` is required and must be a non-empty list — validated **lazily** on first call (zero-arg construction stays valid per the recordstream "Lazy Initialization & Zero-Arg Construction" convention). - * Exactly one boolean attribute (other than ``ops`` / ``name`` and - dunders) may be set on the wrapper — that's the toggle. - ``RuntimeError`` is raised on first call if zero or multiple are - present. - * RESERVED names: the toggle may be ANY boolean attribute name EXCEPT the - class's own members — ``ops``, ``enabled``, ``flag_name`` (read-only - introspection properties; a YAML kwarg with one of those names raises - ``AttributeError`` at configure time). Use ``enable`` as the generic - toggle name; ``enabled`` (the property) then READS whatever toggle is set. + * ``enabled`` must be a ``bool``; a non-bool raises ``TypeError`` at set time. + * Any OTHER boolean attribute set on the wrapper raises ``ValueError`` on + first call. That is the migration guard for the retired dynamic-toggle + form (``visualize: false`` as a bare kwarg), which would otherwise be + accepted silently by the post-construction paradigm and never read. Args: ops: Non-empty list of ops (native or bare library transforms) gated by the toggle. + enabled: Whether the wrapped ops fire. Settable post-construction, from YAML, + and from the CLI (``--enabled`` / ``--.enabled``). + name: Identifier for THIS instance — scopes its CLI flag to ``--.enabled`` + and labels it in ``--help``. Empty (the default) leaves it unnamed, reachable + only by the broadcast form. """ - def __init__(self, ops: Optional[List] = None) -> None: - # Lazy / zero-arg: store config only; the non-empty requirement is enforced lazily in __call__. + def __init__(self, ops: Optional[List] = None, enabled: bool = True, name: str = "") -> None: + # Lazy / zero-arg: store config only; `ops` non-emptiness and stray-toggle + # rejection are enforced lazily on first call. self.ops: List = list(ops) if ops else [] - - def _toggle(self) -> Tuple[str, bool]: - candidates = [ - (k, v) - for k, v in vars(self).items() - if k != "ops" and not k.startswith("_") and not k.startswith("__confluid_") and isinstance(v, bool) - ] - if len(candidates) != 1: - raise RuntimeError( - "Enable requires exactly one boolean toggle attribute (the CLI flag name); " - f"found {len(candidates)}: {[k for k, _ in candidates]}" - ) - return candidates[0] + self.name = name + self.enabled = enabled + self._checked = False @property def enabled(self) -> bool: - _, value = self._toggle() - return value - - @property - def flag_name(self) -> str: - name, _ = self._toggle() - return name - - def __call__(self, record: Record) -> Optional[Record]: + """Whether the wrapped ops fire for each record (the one toggle).""" + return self._enabled + + @enabled.setter + def enabled(self, value: Any) -> None: + # A settable property — not a plain attribute — so `confluid.accepts_key` + # reports it settable and the CLI/YAML override paths admit `enabled`. + if not isinstance(value, bool): + raise TypeError(f"Enable.enabled must be a bool; got {type(value).__name__} ({value!r}).") + self._enabled = value + + def _check(self) -> None: + """Lazy one-time validation, run on the first record.""" if not self.ops: raise ValueError("Enable requires a non-empty 'ops' list.") + stray = [key for key, value in vars(self).items() if isinstance(value, bool) and not key.startswith("_")] + if stray: + raise ValueError( + f"Enable: unexpected boolean attribute(s) {stray} — the toggle is 'enabled'. " + f"Dynamic toggle names are retired: write `name: {stray[0]}` + `enabled: ` " + f"and toggle it with `--{stray[0]}.enabled true`." + ) + + def __call__(self, record: Record) -> Optional[Record]: + if not self._checked: + self._check() + self._checked = True if not self.enabled: return record from confluid import flow diff --git a/recordstream/storage/base.py b/recordstream/storage/base.py index 2d6d017..8caf38c 100644 --- a/recordstream/storage/base.py +++ b/recordstream/storage/base.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict, Iterator, Protocol, Tuple, runtime_checkable +from typing import Any, Dict, Iterator, Protocol, Self, Tuple, runtime_checkable import numpy as np import torch @@ -139,13 +139,15 @@ def _untag_json(value: Any) -> Any: class Storage: """Base class for storage backends providing context manager support.""" - def open(self) -> "Storage": + def open(self) -> Self: return self def close(self) -> None: pass # pragma: no cover - def __enter__(self) -> "Storage": + def __enter__(self) -> Self: + # `Self`, not `Storage`: `with HDF5Sink(...) as sink` must keep the concrete + # backend type so `sink.write(...)` type-checks at the call site. return self.open() def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: diff --git a/recordstream/storage/directory.py b/recordstream/storage/directory.py index 797d556..6cfdd43 100644 --- a/recordstream/storage/directory.py +++ b/recordstream/storage/directory.py @@ -11,6 +11,7 @@ PLAIN_VALUE, TYPED_FORMAT, DataSink, + DataSource, Storage, require_record_format, restore_attrs, @@ -109,7 +110,7 @@ def flush(self) -> None: @confluid.configurable -class DirectorySource(Storage): +class DirectorySource(Storage, DataSource): """Read records written by :class:`DirectorySink` (one ``fields.json`` + ``fields.npz`` per record). The matching source of the sink's record layout (one directory per record, sorted by the diff --git a/tests/test_enable.py b/tests/test_enable.py index e525991..0191309 100644 --- a/tests/test_enable.py +++ b/tests/test_enable.py @@ -1,69 +1,134 @@ """``Enable`` — the one-flag op-list toggle. -Pins the toggle-attribute contract: ANY boolean attribute set post-construction is the -toggle and its NAME is the CLI flag; ``enable`` is the documented generic name for the -named-wrapper pattern (``--overlay.enable``); the class's own members (``ops`` / -``enabled`` / ``flag_name``) are RESERVED — read-only properties reject a same-named -YAML kwarg loudly at configure time. +Pins the toggle contract: the flag is the DECLARED ``enabled`` parameter (a settable +property), several wrappers are told apart by ``name`` (which scopes the CLI flag to +``--.enabled``), and the retired dynamic-toggle form (any boolean attribute name +becoming the flag) is rejected loudly instead of being silently ignored. """ +from typing import List + import pytest +from confluid import accepts_broadcast, accepts_key, to_pydantic +from recordstream import Record from recordstream.ops.enable import Enable -def _tag(record): +def _tag(record: Record) -> Record: return {**record, "seen": True} class TestEnableToggle: - def test_enable_named_toggle_off_passes_through(self) -> None: - op = Enable(ops=[_tag]) - op.enable = False # what `enable: false` in YAML does (post-construction setattr) + def test_default_is_enabled(self) -> None: + # Zero-config wrapper runs its ops — the toggle only ever has to be written to turn it OFF. + assert Enable(ops=[_tag])({"x": 1}) == {"x": 1, "seen": True} + + def test_disabled_passes_through(self) -> None: + op = Enable(ops=[_tag], enabled=False) assert op({"x": 1}) == {"x": 1} assert op.enabled is False - assert op.flag_name == "enable" - def test_enable_named_toggle_on_fires_ops(self) -> None: + def test_constructible_from_python_in_one_call(self) -> None: + # The whole wrapper — ops, toggle, name — comes from the constructor, so a + # generated tool/form/canvas call can build it without post-construction setattr. + op = Enable(ops=[_tag], enabled=False, name="visualize") + assert (op.name, op.enabled, op.ops) == ("visualize", False, [_tag]) + + def test_post_construction_setattr_is_the_yaml_path(self) -> None: + # What `enabled: false` in YAML does when confluid setattr's it after __init__. op = Enable(ops=[_tag]) - op.enable = True + op.enabled = False + assert op({"x": 1}) == {"x": 1} + op.enabled = True assert op({"x": 1}) == {"x": 1, "seen": True} - assert op.enabled is True def test_named_wrappers_toggle_independently(self) -> None: - # Two wrappers, same generic `enable` attr — the instance `name` scopes the CLI flag - # (--overlay.enable vs --labelstudio.enable); here we simulate the post-config state. - overlay, labelstudio = Enable(ops=[_tag]), Enable(ops=[_tag]) - overlay.name, labelstudio.name = "overlay", "labelstudio" - overlay.enable, labelstudio.enable = True, False + # `name` scopes the CLI flag (--overlay.enabled vs --labelstudio.enabled); here we + # simulate the post-override state the two addressed writes produce. + overlay = Enable(ops=[_tag], name="overlay", enabled=True) + labelstudio = Enable(ops=[_tag], name="labelstudio", enabled=False) assert overlay({"x": 1}) == {"x": 1, "seen": True} assert labelstudio({"x": 1}) == {"x": 1} - def test_semantic_toggle_name_still_works(self) -> None: + def test_non_bool_toggle_raises(self) -> None: + # No silent truthiness: a quoted YAML bool / typo'd value fails at set time. + with pytest.raises(TypeError, match="must be a bool"): + Enable(ops=[_tag], enabled="true") # type: ignore[arg-type] op = Enable(ops=[_tag]) - op.visualize = True - assert op.flag_name == "visualize" - assert op({"x": 1}) == {"x": 1, "seen": True} + with pytest.raises(TypeError, match="must be a bool"): + op.enabled = 1 # type: ignore[assignment] + - def test_reserved_names_raise_on_set(self) -> None: - # `enabled` / `flag_name` are read-only introspection properties — a YAML kwarg - # with one of those names fails loudly instead of silently shadowing the API. - for reserved in ("enabled", "flag_name"): - with pytest.raises(AttributeError): - setattr(Enable(ops=[_tag]), reserved, False) +class TestIntrospectionContract: + """The toggle must be reachable from every front-end, not just YAML.""" - def test_zero_toggles_raises(self) -> None: - with pytest.raises(RuntimeError, match="exactly one boolean toggle"): - Enable(ops=[_tag])({"x": 1}) + def test_declared_parameters_are_the_schema(self) -> None: + # to_pydantic drives navigaitor's MCP/form schemas and StreamStudio's widgets — + # a toggle absent here is a toggle no GUI or tool call can set. + assert set(to_pydantic(Enable).model_fields) == {"ops", "enabled", "name"} + + @pytest.mark.parametrize("key", ["ops", "enabled", "name"]) + def test_keys_are_cli_settable(self, key: str) -> None: + # accepts_key gates liquifai's addressed `--.` writes; + # accepts_broadcast gates the bare `--` form. + assert accepts_key(Enable, key) is True + assert accepts_broadcast(Enable, key) is True + + def test_retired_dynamic_toggle_name_is_not_settable(self) -> None: + assert accepts_key(Enable, "visualize") is False + assert accepts_broadcast(Enable, "visualize") is False + + +class TestLazyValidation: + def test_empty_ops_raises_on_first_call(self) -> None: + op = Enable() # zero-arg construction stays valid + with pytest.raises(ValueError, match="non-empty 'ops'"): + op({"x": 1}) - def test_multiple_toggles_raises(self) -> None: + def test_stray_boolean_attribute_raises_with_migration_hint(self) -> None: + # The retired form (`visualize: false` as a bare YAML kwarg) lands as a + # post-construction attribute nothing reads — reject it instead of silently + # running the ops the user meant to gate. op = Enable(ops=[_tag]) - op.enable, op.visualize = True, False - with pytest.raises(RuntimeError, match="exactly one boolean toggle"): + op.visualize = False # type: ignore[attr-defined] # what the old YAML form produced + with pytest.raises(ValueError, match=r"unexpected boolean attribute\(s\) \['visualize'\]"): op({"x": 1}) - def test_empty_ops_raises(self) -> None: - op = Enable() - op.enable = True - with pytest.raises(ValueError, match="non-empty 'ops'"): + def test_migration_hint_names_the_replacement_spelling(self) -> None: + op = Enable(ops=[_tag]) + op.visualize = False # type: ignore[attr-defined] + with pytest.raises(ValueError) as excinfo: op({"x": 1}) + message = str(excinfo.value) + assert "name: visualize" in message and "enabled:" in message + assert "--visualize.enabled" in message + + def test_validation_runs_once_then_stays_out_of_the_hot_path(self) -> None: + op = Enable(ops=[_tag]) + assert op({"x": 1}) == {"x": 1, "seen": True} + # A stray attribute set AFTER the first record is not re-scanned per record. + op.visualize = False # type: ignore[attr-defined] + assert op({"x": 2}) == {"x": 2, "seen": True} + + +class TestOpsChain: + def test_ops_run_in_sequence(self) -> None: + def _bump(record: Record) -> Record: + return {**record, "x": record["x"] + 1} + + op = Enable(ops=[_bump, _bump, _tag]) + assert op({"x": 1}) == {"x": 3, "seen": True} + + def test_close_propagates_to_inner_ops(self) -> None: + closed: List[str] = [] + + class _Sink: + def __call__(self, record: Record) -> Record: + return record + + def close(self) -> None: + closed.append("sink") + + Enable(ops=[_Sink()]).close() + assert closed == ["sink"] diff --git a/tests/test_op_families.py b/tests/test_op_families.py index 4d5520a..b677ed5 100644 --- a/tests/test_op_families.py +++ b/tests/test_op_families.py @@ -8,7 +8,7 @@ """ from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, Iterator, List, Optional import albumentations as A import numpy as np @@ -298,7 +298,7 @@ def invoke_fakelib_override(record: Record, op: FakeLibScale) -> Record: @pytest.fixture() -def family_registry(): +def family_registry() -> Iterator[None]: """Snapshot/restore the global registry so registrations never leak between tests.""" from recordstream import core @@ -313,7 +313,7 @@ def test_builtins_are_registered_through_the_same_registry(self) -> None: assert registered_op_families()[:2] == ("albumentations", "torchvision_v2") - def test_registered_family_dispatches_via_invoker(self, family_registry) -> None: + def test_registered_family_dispatches_via_invoker(self, family_registry: None) -> None: from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) @@ -321,14 +321,14 @@ def test_registered_family_dispatches_via_invoker(self, family_registry) -> None assert out is not None and out["gain_db"] == -9.0 # -3.0 * 3 — via the invoker, op never called assert isinstance(out["image"], Image) # rest of the record untouched - def test_registered_family_runs_in_stream_ops_list(self, family_registry) -> None: + def test_registered_family_runs_in_stream_ops_list(self, family_registry: None) -> None: from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) out = list(Stream(source=[_base_record()], ops=[FakeLibScale(factor=2.0), lambda r: {**r, "tag": 1}])) assert out[0]["gain_db"] == -6.0 and out[0]["tag"] == 1 # mixes with native ops in ONE list - def test_last_registered_family_wins_overlap(self, family_registry) -> None: + def test_last_registered_family_wins_overlap(self, family_registry: None) -> None: from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) @@ -336,7 +336,7 @@ def test_last_registered_family_wins_overlap(self, family_registry) -> None: out = _apply_op(_base_record(), FakeLibScale()) assert out is not None and out["gain_db"] == -999.0 - def test_reregistering_name_replaces_in_place(self, family_registry) -> None: + def test_reregistering_name_replaces_in_place(self, family_registry: None) -> None: from recordstream import register_op_family, registered_op_families register_op_family("fakelib", is_fakelib, invoke_fakelib) @@ -346,11 +346,11 @@ def test_reregistering_name_replaces_in_place(self, family_registry) -> None: out = _apply_op(_base_record(), FakeLibScale()) assert out is not None and out["gain_db"] == -999.0 - def test_unmatched_op_falls_back_to_native_call(self, family_registry) -> None: + def test_unmatched_op_falls_back_to_native_call(self, family_registry: None) -> None: out = _apply_op(_base_record(), lambda r: {**r, "native": True}) assert out is not None and out["native"] is True - def test_spawn_parallel_ships_family_to_workers(self, family_registry) -> None: + def test_spawn_parallel_ships_family_to_workers(self, family_registry: None) -> None: from recordstream import register_op_family register_op_family("fakelib", is_fakelib, invoke_fakelib) diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py index 068ae9c..45bdc39 100644 --- a/tests/test_typed_collate.py +++ b/tests/test_typed_collate.py @@ -1,6 +1,7 @@ """The record collate — batched record convention (golden shapes consumers rely on).""" from dataclasses import dataclass +from typing import Any, Dict, List, Sequence import numpy as np import pytest @@ -85,7 +86,7 @@ def test_unknown_key_raises_with_known_keys(self) -> None: # The collate REGISTRY: a task collate opts out of the generic folding rules # (the docs/record-model.md detection example — variable-N boxes cannot stack). # --------------------------------------------------------------------------- # -def _ragged_detection_records(): +def _ragged_detection_records() -> List[Record]: return [ { "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), @@ -114,7 +115,7 @@ def test_registered_task_collate_produces_its_own_batch_contract() -> None: from recordstream import collate, get_collate, register_collate @register_collate("_test_detection") - def detection_collate(items): + def detection_collate(items: Sequence[Record]) -> Dict[str, Any]: images = torch.stack([torch.as_tensor(np.asarray(r["image"])).permute(2, 0, 1) for r in items]) targets = [ { diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index f198c86..d0d7c8e 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -1,5 +1,6 @@ """FlowGraph over dict records — merge_from fan-in, step[key]/bare-step bind, lowering parity.""" +from pathlib import Path from typing import Any, Dict, Optional import numpy as np @@ -242,10 +243,10 @@ def _doc(self) -> str: outputs: out """ - def _record(self): + def _record(self) -> Record: return {"image": Image(np.arange(16, dtype=np.float32).reshape(4, 4) / 15.0)} - def test_yaml_bind_via_plain_mapping_step(self, tmp_path) -> None: + def test_yaml_bind_via_plain_mapping_step(self, tmp_path: Path) -> None: # Scalar reserved keys ride in the marker mapping; the nested bind: mapping MUST use # the plain-mapping (op:) step form — a nested mapping under a !class: marker is # consumed by confluid as addressed configuration and never reaches parse_flow. @@ -256,7 +257,7 @@ def test_yaml_bind_via_plain_mapping_step(self, tmp_path) -> None: assert int(np.asarray(out["mask"]).sum()) == 8 # fixed 0.5 threshold assert int(np.asarray(out["gated_mask"]).sum()) == 6 # per-record amax(a)*0.6 bind - def test_yaml_bind_parity_with_lowered_stream(self, tmp_path) -> None: + def test_yaml_bind_parity_with_lowered_stream(self, tmp_path: Path) -> None: path = tmp_path / "graph.yaml" path.write_text(self._doc()) a = list(FlowGraph.from_yaml(str(path), source=[self._record()]))[0] @@ -264,7 +265,7 @@ def test_yaml_bind_parity_with_lowered_stream(self, tmp_path) -> None: assert np.array_equal(np.asarray(a["gated_mask"]), np.asarray(b["gated_mask"])) assert np.array_equal(np.asarray(a["mask"]), np.asarray(b["mask"])) - def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path) -> None: + def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path: Path) -> None: # Pin the confluid behavior that makes the op:-form MANDATORY for bind — if this # ever starts surviving in marker kwargs, the doc rule can be relaxed. path = tmp_path / "graph.yaml" From 1f5ed2f07785b7ae55a4a50220b26734de970101 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 28 Jul 2026 11:52:08 +0200 Subject: [PATCH 045/102] chore(ci): regenerate CI scaffolds for the PyPI-sourced loggair + confluid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `aisland source set pypi` records the switch in the committed project-sources.toml, and the scaffolder regenerates every dependent project's Jenkinsfile / Jenkinsfile.local / .github/workflows/ci.yml from it. The `--no-deps git+…` pre-install line for each is dropped — `.[dev]` resolves them from PyPI now — and the remaining unpublished-Gearlux dependency list is recomputed from the project's own metadata in the same pass, so some projects gain a dep line and others lose one. Generated files: re-run `aisland jenkins scaffold` / `aisland source apply` rather than hand-editing them. --- .github/workflows/ci.yml | 12 ++++-------- Jenkinsfile | 3 +-- Jenkinsfile.local | 3 +-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b97774e..6ae99aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Isort run: isort --check-only . @@ -58,8 +57,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Tests run: | @@ -92,8 +90,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Examples run: | @@ -130,8 +127,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. diff --git a/Jenkinsfile b/Jenkinsfile index e38150d..1fc6fce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,8 +33,7 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main" - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" + sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 8d4af5c..10551cb 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,8 +42,7 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/loggair" - sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/confluid" + sh "if [ -d '${env.WORKSPACE_ROOT}/liquifai' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships From 565a7fe3f6974a40bee59db0f0d3c6a0066d63e0 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 28 Jul 2026 22:40:05 +0200 Subject: [PATCH 046/102] =?UTF-8?q?chore(ci):=20regenerate=20=E2=80=94=20J?= =?UTF-8?q?enkinsfile.local=20no=20longer=20reads=20project=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `aisland jenkins scaffold` now emits the same `[ -d ]`-guarded install line for every internal dependency, whatever its configured source: editable when the directory exists, `git@main` when it does not. That makes the file a function of the dependency list alone, so no `aisland source` change can rewrite it — the guard asks at build time the same question the configuration answers. The runner-side Jenkinsfile / ci.yml drop the per-source ref and always clone `main`, reading only whether a dependency is consumed from PyPI. --- .github/workflows/ci.yml | 12 ------------ Jenkinsfile | 4 ---- Jenkinsfile.local | 2 ++ 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ae99aa..1e4ed00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Isort run: isort --check-only . @@ -55,9 +52,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Tests run: | @@ -88,9 +82,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Examples run: | @@ -125,9 +116,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. diff --git a/Jenkinsfile b/Jenkinsfile index 1fc6fce..b31f608 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,10 +30,6 @@ pipeline { sh "${VENV_BIN}/pip install --upgrade pip uv" echo 'Installing Dependencies...' - // Internal Gearlux dependencies — installed FIRST with --no-deps - // so .[dev] below finds them pre-satisfied instead of hitting PyPI - // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 10551cb..bf67e29 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,7 +42,9 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). + sh "if [ -d '${env.WORKSPACE_ROOT}/confluid' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" sh "if [ -d '${env.WORKSPACE_ROOT}/liquifai' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" + sh "if [ -d '${env.WORKSPACE_ROOT}/loggair' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships From 55e1ead0cceadc2c54bb236aaee34ecf71d9bbbc Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 11:51:09 +0200 Subject: [PATCH 047/102] chore(ci): regenerate scaffolds for the local-sourced confluid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `aisland source set confluid local` moved confluid off PyPI, so the generated CI installs it from GitHub `@main` with `--no-deps` again instead of resolving it from the index. Regenerated artifact — edit the template (aisland), never these files. --- .github/workflows/ci.yml | 12 ++++++++++++ Jenkinsfile | 4 ++++ Jenkinsfile.local | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e4ed00..b6f5989 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,9 @@ jobs: - name: Install dependencies run: | + # Internal Gearlux dependencies — installed FIRST with --no-deps so + # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" - name: Run Isort run: isort --check-only . @@ -52,6 +55,9 @@ jobs: - name: Install dependencies run: | + # Internal Gearlux dependencies — installed FIRST with --no-deps so + # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" - name: Run Tests run: | @@ -82,6 +88,9 @@ jobs: - name: Install dependencies run: | + # Internal Gearlux dependencies — installed FIRST with --no-deps so + # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" - name: Run Examples run: | @@ -116,6 +125,9 @@ jobs: - name: Install dependencies run: | + # Internal Gearlux dependencies — installed FIRST with --no-deps so + # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system -e ".[dev]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. diff --git a/Jenkinsfile b/Jenkinsfile index b31f608..332563a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,6 +30,10 @@ pipeline { sh "${VENV_BIN}/pip install --upgrade pip uv" echo 'Installing Dependencies...' + // Internal Gearlux dependencies — installed FIRST with --no-deps + // so .[dev] below finds them pre-satisfied instead of hitting PyPI + // (Gearlux distribution names are intentionally unpublished on PyPI). + sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships diff --git a/Jenkinsfile.local b/Jenkinsfile.local index bf67e29..6d09076 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,9 +42,9 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "if [ -d '${env.WORKSPACE_ROOT}/confluid' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" sh "if [ -d '${env.WORKSPACE_ROOT}/liquifai' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" sh "if [ -d '${env.WORKSPACE_ROOT}/loggair' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" + sh "if [ -d '${env.WORKSPACE_ROOT}/confluid' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships From 8410df511360c036e0b913f4d29ac821c86ab413 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 15:32:59 +0200 Subject: [PATCH 048/102] feat: MultiLabel item, the collate read-back, and broadcast-correct `recordstream run` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **MultiLabel + label mapping.** New `MultiLabel` item — several classes for one record — as its own type rather than a `Label` holding a list, because only the type distinguishes a genuine multi-label target from an ordinary sequence value that happens to sit under the target key. `is_class_id` is the single rule for "is this already encoded?" (recognising a Python int, a numpy integer and a 0-d array/tensor, excluding bool); `LabelMap.to_ids` always returns a list of int ids, passing encoded values through, so consumers need no name-vs-id branch. `EncodeTarget`/`DecodeTarget` handle both label types. **scikit-learn dropped.** `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line. Ordering is unchanged. **`recordstream.batch` — the collate read-back.** `batch_values` / `batch_tensor` / `batch_metadata` are the inverse of `collate_records` and live beside it: a consumer re-deriving them is re-deriving the collate, and two consumer packages had independently written near-identical private copies with duplicate tests. They carry no dtype or shape opinion — task shaping stays at each model boundary. **`materialize_runnable`.** `recordstream run` bare-flowed the bound `runnable:` node, so a flat config's top-level keys were dropped silently: `train_set` became None (surfacing much later as an empty dataset) and `max_epochs: 3` reverted to the constructor default without surfacing at all. The node is now built against the loaded document. liquifai 0.1.1 fixes this at its own layer; the helper stays until that release is on PyPI, since generated CI clones each local-sourced dependency's main. Docs: architecture records for the read-back and the runnable contract; README, kinds, projection, record-model and runnable pages updated. CI scaffolds regenerated for the liquifai source switch. --- .github/workflows/ci.yml | 4 + AGENTS.md | 7 +- CLAUDE.md | 20 ++++- GEMINI.md | 20 ++++- Jenkinsfile | 1 + Jenkinsfile.local | 2 +- README.md | 4 +- docs/architecture.md | 36 +++++++++ docs/kinds.md | 15 ++++ docs/projection.md | 37 ++++++++- docs/record-model.md | 40 +++++++++- docs/runnable.md | 28 +++++++ pyproject.toml | 1 - recordstream/__init__.py | 8 ++ recordstream/batch.py | 125 +++++++++++++++++++++++++++++++ recordstream/cli.py | 83 ++++++++++++++++---- recordstream/items.py | 73 +++++++++++++++++- recordstream/labels.py | 91 ++++++++++++++++++---- recordstream/ops/target.py | 56 +++++++++----- recordstream/projection.py | 9 ++- tests/test_batch.py | 133 +++++++++++++++++++++++++++++++++ tests/test_cli_materialize.py | 128 +++++++++++++++++++++++++++++++ tests/test_labels.py | 108 +++++++++++++++++++++++++- tests/test_typed_target_ops.py | 2 +- 24 files changed, 958 insertions(+), 73 deletions(-) create mode 100644 recordstream/batch.py create mode 100644 tests/test_batch.py create mode 100644 tests/test_cli_materialize.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f5989..c079cdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Isort run: isort --check-only . @@ -58,6 +59,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Tests run: | @@ -91,6 +93,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" - name: Run Examples run: | @@ -128,6 +131,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main + uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. diff --git a/AGENTS.md b/AGENTS.md index 217888c..6da2e6d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. @@ -30,7 +30,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_tensor` (+ stack / `as_tensor` / device move) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the three collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. They carry NO dtype or shape opinion: an `[N]` int64 class-id promotion, an `[N, C]` float multi-hot, an `[N, H, W]` mask promotion are TASK shaping and stay at the consumer's model boundary (one shared function would just be a task switch). Package-root exports; pins: `tests/test_batch.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. @@ -38,7 +38,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing recordstream never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a recordstream dependency for this. +- **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/CLAUDE.md b/CLAUDE.md index 49da027..6da2e6d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,9 +2,21 @@ ## Current state +> **Renamed 2026-07-26 — `sampleflux` → `recordstream`, `Flux` → `Stream`.** The package was named +> for a data model it no longer has: the 2026-07-25 migration made the carrier a **record**, so the +> vocabulary is now one word per concept — a **`Stream`** of **`Record`**s. Import name, distribution +> name, GitHub repo, console script (`recordstream run`), every `recordstream-*` entry point, the +> `RECORDSTREAM_*` StreamStudio socket types, and the on-disk root attr (`recordstream_format`, +> value still `typedrecord-v1`) all moved together; `JointFlux.fluxes` is `JointStream.streams`. +> **No back-compat aliases** — a pre-rename config, saved canvas, or store must be re-pointed +> (a store missing `recordstream_format` raises the usual re-generate error). The word *sample* is +> now reserved for its OTHER meanings and was deliberately NOT renamed: a discrete-time signal +> sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still +> *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). + Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. @@ -17,7 +29,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_tensor` (+ stack / `as_tensor` / device move) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the three collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. They carry NO dtype or shape opinion: an `[N]` int64 class-id promotion, an `[N, C]` float multi-hot, an `[N, H, W]` mask promotion are TASK shaping and stay at the consumer's model boundary (one shared function would just be a task switch). Package-root exports; pins: `tests/test_batch.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. @@ -25,7 +38,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing recordstream never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a recordstream dependency for this. +- **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/GEMINI.md b/GEMINI.md index 49da027..6da2e6d 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -2,9 +2,21 @@ ## Current state +> **Renamed 2026-07-26 — `sampleflux` → `recordstream`, `Flux` → `Stream`.** The package was named +> for a data model it no longer has: the 2026-07-25 migration made the carrier a **record**, so the +> vocabulary is now one word per concept — a **`Stream`** of **`Record`**s. Import name, distribution +> name, GitHub repo, console script (`recordstream run`), every `recordstream-*` entry point, the +> `RECORDSTREAM_*` StreamStudio socket types, and the on-disk root attr (`recordstream_format`, +> value still `typedrecord-v1`) all moved together; `JointFlux.fluxes` is `JointStream.streams`. +> **No back-compat aliases** — a pre-rename config, saved canvas, or store must be re-pointed +> (a store missing `recordstream_format` raises the usual re-generate error). The word *sample* is +> now reserved for its OTHER meanings and was deliberately NOT renamed: a discrete-time signal +> sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still +> *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). + Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. @@ -17,7 +29,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. +- **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_tensor` (+ stack / `as_tensor` / device move) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the three collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. They carry NO dtype or shape opinion: an `[N]` int64 class-id promotion, an `[N, C]` float multi-hot, an `[N, H, W]` mask promotion are TASK shaping and stay at the consumer's model boundary (one shared function would just be a task switch). Package-root exports; pins: `tests/test_batch.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. @@ -25,7 +38,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sklearn `LabelEncoder`, sorted-unique ordering; sklearn is lazy-imported in `fit` so importing recordstream never pulls it in), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). `scikit-learn` is a recordstream dependency for this. +- **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/Jenkinsfile b/Jenkinsfile index 332563a..f39d72b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,6 +34,7 @@ pipeline { // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" + sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 6d09076..1515ae4 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,9 +42,9 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "if [ -d '${env.WORKSPACE_ROOT}/liquifai' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" sh "if [ -d '${env.WORKSPACE_ROOT}/loggair' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "if [ -d '${env.WORKSPACE_ROOT}/confluid' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" + sh "if [ -d '${env.WORKSPACE_ROOT}/liquifai' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships diff --git a/README.md b/README.md index 88ce038..f69326c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `RecordS ## 🚀 Key Features -- **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. +- **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. - **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. - **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. - **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-record `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Stream` engine. @@ -97,7 +97,7 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit | Page | Covers | |---|---| | [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | -| [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`), 1→N expanding ops | +| [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), 1→N expanding ops | | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Stream.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | diff --git a/docs/architecture.md b/docs/architecture.md index acad247..562ef36 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -224,6 +224,39 @@ def yolo_collate(items): loader = DataLoader(stream, batch_size=8, collate_fn=get_collate("yolo")) ``` +### Addendum: the READ-BACK lives here too (`recordstream.batch`, 2026-07-29) + +**Context.** Every model boundary has to undo the three rules above: get past a wrapper item, +turn a per-record list into one tensor, transpose the leftover columns into per-record dicts for +a predictions sink. That is not task knowledge — it is the collate's own convention read +backwards. Two consumer packages had independently written it: one for classification, one for +segmentation, with two near-identical private `_batch_metadata` implementations and two separate +test files pinning them. A third consumer would have written a third. + +**Decision.** `recordstream.batch` ships the inverse beside the collate — `batch_values`, +`batch_tensor`, `batch_metadata` — and it carries **no dtype or shape opinion**. Rule 2 above +says turning class names into an `[N]` int64 tensor is "the model boundary's one explicit step, +not a generic-engine guess"; that still holds. What moved is *reading*, not *shaping*. + +**Consequences.** The three shapes a consumer actually wants — a classifier's `[N]` int64 ids, a +multi-label trainer's `[N, C]` float multi-hot, a segmenter's `[N, H, W]` int64 mask — all start +from `batch_values` and are shaped by a small task-specific function the consumer keeps. Folding +those three into one shared helper would produce a function whose body is a task switch, which +is the thing the collate registry exists to avoid. + +**Example.** + +```python +from recordstream import batch_tensor, batch_values, batch_metadata + +x = batch_tensor(batch, "image", device=self.device) # generic: one [N, 3, H, W] tensor +meta = batch_metadata(batch, exclude=("image", "class")) # generic: N per-record dicts + +# task-specific, stays in the consumer: +ids = torch.as_tensor(batch_values(batch, "class")) # a classifier's [N] class ids +mask = batch_tensor(batch, "target").long() # a segmenter's [N, H, W] int64 mask +``` + ### What you may change (and where it's documented) - **Plugging in your own batch layout** is the supported extension point — decorate a function @@ -232,6 +265,9 @@ loader = DataLoader(stream, batch_size=8, collate_fn=get_collate("yolo")) - **Changing the default collate's semantics** (how `"record"` stacks, the attrs-become-lists convention) is an architectural change: every batch consumer depends on it. Update this record and the recordstream `AGENTS.md` metadata mandate together. +- **Adding a reader** to `recordstream.batch` is fine when it is the collate read backwards. + Adding one that shapes for a task (promotes a dtype, builds a multi-hot) is not — that belongs + to the consumer, or the helper becomes a task switch. --- diff --git a/docs/kinds.md b/docs/kinds.md index 7e589eb..ebabdd9 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -63,6 +63,21 @@ loader = DataLoader(stream, collate_fn=get_collate("yolo")) The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17). +### Reading a batch back (`recordstream.batch`) + +The inverse of `collate_records`, shipped alongside it so a model boundary never re-derives the convention: + +```python +from recordstream import batch_values, batch_tensor, batch_metadata + +batch_values(batch, "class") # past the wrapper item: a Label -> its .value list +batch_tensor(batch, "image", device=model.device) # ONE tensor, stacked + moved +batch_metadata(batch, exclude=("image", "class")) # the remaining columns transposed into N dicts +``` + +They carry no dtype or shape opinion on purpose — an int64 class-id promotion, a float multi-hot, an `[N, H, W]` mask are all TASK shaping and stay at the caller's model boundary. + + ## 1→N expanding ops (iterable-only pipelines) An op may return **several** carriers — a windowing op splitting one capture into N windows marks itself with `EXPANDS = True` and returns an iterable of records: diff --git a/docs/projection.md b/docs/projection.md index e72fd81..992bca7 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -13,8 +13,9 @@ from recordstream import project, iter_key, num_classes for record in project(my_source, ("class",)): ... # partial records carrying only the "class" entry -labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, - # other items to their payload, plain values verbatim +labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, a MultiLabel to + # its .values LIST, other items to their payload, + # plain values verbatim n = num_classes(my_source, key="class") # max(class_id) + 1 — always walks ``` @@ -22,7 +23,7 @@ Sources that don't implement `SupportsProjection` still work via a correct full- ## `LabelMap` — fittable name↔id encoding -When a dataset's label is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTarget` / `DecodeTarget` ops need — the *fittable* companion to those ops. Fit it once (sklearn `LabelEncoder`, deterministic sorted ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: +When a dataset's label is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTarget` / `DecodeTarget` ops need — the *fittable* companion to those ops. Fit it once (deterministic `sorted(set(...))` ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: ```python from recordstream import LabelMap, Stream, iter_key @@ -38,4 +39,32 @@ encoded = Stream(source=train_source, ops=[lm.encode_op()]) # "class" Labels n lm2 = LabelMap.load("class_names.json") ``` -`LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the mapping is pinned, so train / eval / predict never disagree. `scikit-learn` backs `fit` (lazy-imported). +`LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the mapping is pinned, so train / eval / predict never disagree. `fit` is pure stdlib — no ML library is pulled in to sort a set of names. + +### Multi-label targets and `to_ids` + +`fit` accepts a `MultiLabel` (or any sequence) just as readily as a single `Label`, so a +multi-label dataset builds its vocabulary from the same call — every distinct member becomes +one class: + +```python +from recordstream import LabelMap, MultiLabel + +lm = LabelMap.fit([MultiLabel(["cat", "dog"]), MultiLabel(["bird"])]) +lm.label_names # ["bird", "cat", "dog"] +``` + +`to_ids(target)` is the one accessor a consumer needs — it always returns a **list of int class +ids**, whatever it is handed, so there is no name-vs-id and no single-vs-multi branch at the call +site: + +```python +lm.to_ids(Label("cat")) # [1] a name -> its id +lm.to_ids(Label(1)) # [1] an ALREADY-encoded id passes through +lm.to_ids(MultiLabel(["cat", "dog"])) # [1, 2] +lm.to_ids("dog") # [2] a bare value works too +``` + +Because encoded ids pass through untouched, `LabelMap().to_ids(...)` (an *empty* map) is a valid +way to normalize an already-encoded dataset to id lists — useful for counting classes or +class-frequency statistics without fitting anything. diff --git a/docs/record-model.md b/docs/record-model.md index c1d9fdd..daf42cb 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -42,18 +42,25 @@ recordstream is **modality-neutral**, so its core ships only generic items — i labels. (Domain items — a signal, a spectrogram — live in the domain package; see below.) ```python -from recordstream import Image, Mask, Regions, Label +from recordstream import Image, Mask, Regions, Label, MultiLabel Image(rgb_hwc, layout="HWC") # an image knows its layout ("HWC" default / "CHW") Mask(seg_hw) # a mask shares its image's frame Regions(boxes=[[1,1,4,4]], labels=["drone"], canvas=(8, 10), extras={"snr_db": [12.5]}) -Label("drone_x", classes=["noise", "drone_x"]) +Label("drone_x", classes=["noise", "drone_x"]) # ONE class for this record +MultiLabel(["drone_x", "jammer"], classes=[...]) # SEVERAL classes for this record ``` +`Label` / `MultiLabel` hold either class **names** or already-encoded **ids** — `.is_encoded` +(built on the free function `is_class_id`) is the single rule that decides which, and +[`LabelMap.to_ids`](projection.md) is the single way to get ids out. Multi-label is its own +ITEM rather than a `Label` holding a list, because only the type distinguishes a genuine +multi-label target from an ordinary sequence value that happens to sit under the target key. + Items are **hybrid**: array-backed items (`Image`, `Mask`) subclass `NDArrayItem` — an `np.ndarray` subclass whose declared `_item_attrs` survive numpy operations via `__array_finalize__` — so a -type-agnostic operation touches them as an array; structured items (`Regions`, `Label`) are dataclass -wrappers (a bounding-box set is not an array). A uniform payload accessor hides the difference from +type-agnostic operation touches them as an array; structured items (`Regions`, `Label`, `MultiLabel`) +are dataclass wrappers (a bounding-box set is not an array). A uniform payload accessor hides the difference from kernels: ```python @@ -465,6 +472,31 @@ batch = collate_records(records) # snr_db: [0.0, 10.0, 20.0] ``` +### Reading the batch back — `batch_values` / `batch_tensor` / `batch_metadata` + +Every model boundary has to undo those three rules, so `recordstream.batch` ships the inverse +next to the collate that wrote it: + +```python +from recordstream import batch_values, batch_tensor, batch_metadata + +batch_values(batch, "class") # [0, 1, 0] — past the wrapper item +batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] — one tensor, whatever the shape +batch_metadata(batch, exclude=("image", "class")) # [{"snr_db": 0.0}, ...] — per-record dicts +``` + +`batch_values` is the one that knows how to get *past* an item — a `Label` yields its `.value`, +a `MultiLabel` its `.values`, an array item its stacked payload, a plain value its list. +`batch_tensor` adds stacking + `torch.as_tensor` + an optional device move; `batch_metadata` +transposes the remaining columns back into N dicts so a predictions sink can pair a model's +output with the record it came from. + +They deliberately carry **no dtype or shape opinion** — that is rule 2's "one explicit step at +the model boundary". A classifier promotes to `[N]` int64 class ids, a multi-label trainer +builds an `[N, C]` float multi-hot, a segmenter promotes an `[N, H, W]` mask to int64; all three +start from `batch_values` and shape it themselves, so the shared helpers never become one +function with a task switch. + ### When the generic rules cannot work: register a task collate Stacking is task-shaped, and detection is the canonical failure: each record carries a DIFFERENT diff --git a/docs/runnable.md b/docs/runnable.md index adbcbeb..0448b1d 100644 --- a/docs/runnable.md +++ b/docs/runnable.md @@ -14,6 +14,34 @@ runnable: !class:mypkg.Classifier python -m recordstream.cli run config.yaml # builds `runnable:`, calls .run() ``` +### Top-level keys reach the runnable + +The runnable is built **against the whole document**, so a *flat* config works: a top-level +key broadcasts into the same-named constructor parameter, with no nesting and no `!ref:`. + +```yaml +runnable: !class:mypkg.Classifier + model: !lazy:mypkg.Backbone { name: resnet18 } + +train_set: !class:recordstream.sources.HuggingFaceSource { path: mnist, split: train } +max_epochs: 3 # -> Classifier(max_epochs=3) +batch_size: 32 # -> Classifier(batch_size=32) +``` + +If you write your own runner CLI, build the bound node with +`recordstream.cli.materialize_runnable(node)` rather than a bare `flow()`. A bare flow builds +the node in isolation, so every top-level key above is dropped — and dropped *silently*: +`train_set` becomes `None` and `max_epochs` quietly falls back to its default, leaving a run +that looks configured and is not. + +```python +from recordstream.cli import materialize_runnable + +@app.script_command(flow_mode="manual") # "auto" is the bare flow this replaces +def run(runnable: Any) -> None: + materialize_runnable(runnable).run() +``` + ## The problem entry points solve A merged train+eval class exposes SEVERAL capabilities from one class, dispatched off its diff --git a/pyproject.toml b/pyproject.toml index 02373aa..3e796a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ dependencies = [ "orjson", "fsspec", "cloudpathlib", - "scikit-learn", # LabelMap.fit() uses sklearn.preprocessing.LabelEncoder (lazy-imported) # liquifai powers the generic `recordstream run ` CLI (recordstream.cli) that # runs any Confluid-wired runnable; it also brings `rich` (used by DatasetProcessor's # optional console progress bar). liquifai depends only on confluid/loggair/rich — no cycle. diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 5adcd28..dbc5ecd 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -10,6 +10,7 @@ """ # --- shared infrastructure ----------------------------------------------------------------- +from recordstream.batch import batch_metadata, batch_tensor, batch_values from recordstream.collate import collate, collate_records, get_collate, register_collate, registered_collates from recordstream.context import Context from recordstream.core import FilterOp, JointStream, Stream, WrappedOp, register_op_family, registered_op_families @@ -30,10 +31,12 @@ Image, Label, Mask, + MultiLabel, NDArrayItem, Record, Regions, get_item_type, + is_class_id, is_item, item_data, item_type_names, @@ -64,6 +67,8 @@ "Mask", "Regions", "Label", + "MultiLabel", + "is_class_id", "register_item", "item_types", "item_type_names", @@ -97,6 +102,9 @@ "from_ops", "to_ops", "collate", + "batch_metadata", + "batch_tensor", + "batch_values", "collate_records", "get_collate", "register_collate", diff --git a/recordstream/batch.py b/recordstream/batch.py new file mode 100644 index 0000000..f00d70e --- /dev/null +++ b/recordstream/batch.py @@ -0,0 +1,125 @@ +"""Reading a batched record back — the inverse of :func:`~recordstream.collate.collate_records`. + +:mod:`recordstream.collate` writes the batch convention; this module reads it. They are two +halves of ONE piece of knowledge (an item's payload stacks, its declared attrs become +per-record lists, a plain value becomes a plain list), so they live side by side — a consumer +that had to re-derive the read-back would be re-deriving the collate. + +The three primitives are deliberately TASK-AGNOSTIC. They answer "what did the collate put +under this key?", never "what shape does my loss want?" — a classification trainer wanting +``[N]`` int64 class ids, a segmenter wanting an ``[N, H, W]`` int64 mask, and a multi-label +trainer wanting an ``[N, C]`` float multi-hot all start from the same unwrapped values and +shape them at their own model boundary. Putting that shaping here would mean one function +with a task switch. + +Typical use at a model boundary:: + + x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + meta = batch_metadata(batch, exclude=("image", "class")) # per-record dicts for a sink +""" + +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional + +import numpy as np + +from recordstream.items import Label, MultiLabel, Record, item_data + +if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only + from torch import Tensor + +__all__ = ["batch_metadata", "batch_tensor", "batch_values"] + + +def batch_values(batch: Record, key: str) -> Any: + """The raw batched values under ``key``, unwrapped from their item type. + + The one place that knows how to get *past* a wrapper item: a :class:`~recordstream.Label` + yields its ``.value`` (the collate leaves it a per-record LIST — a wrapper item's payload + is not stacked), a :class:`~recordstream.MultiLabel` its ``.values`` (a list OF lists), + and anything else goes through :func:`~recordstream.item_data` (an array item yields its + stacked payload, a plain value the per-record list the collate gathered). + + No torch, no stacking, no dtype opinion — just the values. Use :func:`batch_tensor` when + a tensor is what you need. + + Example:: + + batch_values(collate_records([{"class": Label(0)}, {"class": Label(1)}]), "class") + # [0, 1] + """ + item = batch[key] + if isinstance(item, MultiLabel): + return item.values + if isinstance(item, Label): + return item.value + return item_data(item) + + +def batch_tensor(batch: Record, key: str, device: Any = None) -> "Tensor": + """The batched values under ``key`` as ONE torch tensor. + + Normalizes the two shapes the collate can leave behind — a stacked array payload, or a + per-record LIST (a wrapper item's payload, or a plain value) — into a single tensor. An + already-stacked tensor is used verbatim; anything else goes through the cheap, + memory-sharing ``torch.as_tensor``. + + Args: + batch: A batched record (the output of :func:`~recordstream.collate_records`). + key: The record key to read. + device: Optional target device. A tensor built HERE from a per-record list is created + on the CPU regardless of a framework's own batch move, so pass the module's device + when the result feeds a model. + + Returns: + A ``torch.Tensor``. The dtype is whatever the values carry — shaping (an int64 class-id + promotion, a float multi-hot) belongs to the caller's model boundary. + + Example:: + + x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + """ + import torch # local: recordstream stays importable without touching torch + + value = batch_values(batch, key) + if isinstance(value, list): + tensor = torch.stack([torch.as_tensor(v) for v in value]) + elif isinstance(value, torch.Tensor): + tensor = value + else: + tensor = torch.as_tensor(np.asarray(value)) + return tensor if device is None else tensor.to(device) + + +def batch_metadata(batch: Record, exclude: Iterable[str] = ()) -> Optional[List[Dict[str, Any]]]: + """Per-record metadata dicts recovered from a batched record — the collate's transpose. + + The collate turns N records into ONE record of per-key columns; this turns those columns + back into N dicts, so a predictions sink can correlate a model's per-record output with + the record it came from. Keys named in ``exclude`` (typically the input and target keys, + which the model already consumed) are left out. + + A hand-built batch that instead carries a single list-valued ``"metadata"`` key is + returned verbatim — that shape is already the answer. + + Args: + batch: A batched record. + exclude: Keys to omit from the per-record dicts. + + Returns: + One dict per record, or ``None`` when nothing remains after ``exclude`` (there is no + metadata to correlate, which a caller reads as "skip"). The list is as long as the + SHORTEST column — a ragged batch truncates rather than raising, so a metadata detail + cannot take down an inference run. + + Example:: + + batch_metadata(batch, exclude=("image", "class")) # [{"idx": 0}, {"idx": 1}] + """ + if "metadata" in batch and isinstance(batch["metadata"], list): + return list(batch["metadata"]) + skip = set(exclude) + columns = {key: batch_values(batch, key) for key in batch if key not in skip} + if not columns: + return None + n = min(len(v) for v in columns.values()) + return [{k: columns[k][i] for k in columns} for i in range(n)] diff --git a/recordstream/cli.py b/recordstream/cli.py index 56b362a..8c55672 100644 --- a/recordstream/cli.py +++ b/recordstream/cli.py @@ -2,11 +2,11 @@ ``recordstream run `` loads a Confluid YAML that binds a *runnable* object (anything exposing a no-arg ``run()``) under the top-level ``runnable:`` -key, flows it under the active context (so nested ``!ref:`` markers resolve), and -calls ``run()``. This is the single entry point that replaces bespoke per-verb -CLIs: a training run, an evaluation, a dataset conversion, or a whole -:mod:`~recordstream.workflow` are all just runnables — the ``!class:`` the YAML roots -on decides what happens. +key, materializes it against the whole document (so the flat config's top-level +keys broadcast into its constructor), and calls ``run()``. This is the single +entry point that replaces bespoke per-verb CLIs: a training run, an evaluation, a +dataset conversion, or a whole :mod:`~recordstream.workflow` are all just +runnables — the ``!class:`` the YAML roots on decides what happens. Example:: @@ -27,24 +27,81 @@ app = LiquifyApp(name="recordstream") +__all__ = ["app", "main", "materialize_runnable", "run"] -@app.script_command(flow_mode="auto") + +def materialize_runnable(runnable: Any) -> Any: + """Build a bound ``runnable:`` node **with the flat config's keys broadcast in**. + + Why this exists instead of a bare ``flow()``: broadcasting (a top-level YAML key + injecting into the same-named constructor parameter) only happens when a Fluid is + built *against the document it came from*. Liquifai's DI does that for a command + parameter annotated with a **configurable class** — it materializes the block with + ``context=``. A generic runner cannot use such an annotation: its + parameter is ``Any`` precisely because the runnable is polymorphic, so DI falls back + to handing over the raw Fluid and deep-flowing it with no context, and every + top-level sibling is silently dropped. + + Silently is the operative word — a dropped ``train_set:`` surfaces later as an empty + dataset, and a dropped ``max_epochs: 3`` does not surface at all: the run proceeds on + the constructor default and looks configured. + + So this reaches back to the loaded document through liquifai's context and calls + ``materialize(node, context=document)``. Nested stubs still ride the normal deferred + path — a ``!lazy:`` marker stays deferred for the runnable to flow at run time. + + **liquifai 0.1.1 fixes this at its own layer** (``di.deep_flow`` now takes the + document), so once that release is on PyPI and the floors are raised, the runners can + go back to ``flow_mode="auto"`` and this helper can shrink away. It stays until then + because generated CI clones each local-sourced dependency's ``main`` — reverting early + would silently regress to dropping every top-level key. Keeping it is harmless + meanwhile: building against the document is correct under either liquifai. + + Args: + runnable: The value liquifai bound to the command's ``runnable`` parameter — + typically a :class:`~confluid.fluid.Fluid`, but a live object (already built) + passes through untouched. + + Returns: + The built runnable. Falls back to a plain ``flow()`` when there is no liquifai + context or its config is not a mapping (a YAML whose root is a single ``!class:`` + document has no siblings to broadcast, so there is nothing to lose). + + Example:: + + @app.script_command(flow_mode="manual") + def run(runnable: Any) -> None: + runnable = materialize_runnable(runnable) + runnable.run() + """ + from confluid import flow, materialize + from confluid.fluid import Fluid + from liquifai.context import get_context + + if not isinstance(runnable, Fluid): + return runnable + + context = get_context() + document = getattr(context, "config_data", None) if context is not None else None + if isinstance(document, dict): + return materialize(runnable, context=document) + return flow(runnable) + + +@app.script_command(flow_mode="manual") def run(runnable: Any) -> None: """Run any Confluid-instantiated object that exposes ``.run()``. The YAML config binds the object under the top-level ``runnable:`` key. - ``flow_mode="auto"`` deep-flows it under Confluid's active context so nested - ``!ref:`` markers resolve against the loaded YAML's top-level keys. + ``flow_mode="manual"`` because :func:`materialize_runnable` does the building + itself — liquifai's auto deep-flow would bare-flow the node and drop every + broadcast top-level key (see that function's docstring). """ if runnable is None: logger.error("'runnable' was not bound — provide one under 'runnable:' in your YAML.") return - from confluid import flow - from confluid.fluid import Fluid - - if isinstance(runnable, Fluid): - runnable = flow(runnable) + runnable = materialize_runnable(runnable) label = runnable.__class__.__name__ run_method = getattr(runnable, "run", None) diff --git a/recordstream/items.py b/recordstream/items.py index 7fed0e2..760443c 100644 --- a/recordstream/items.py +++ b/recordstream/items.py @@ -57,6 +57,8 @@ "Mask", "Regions", "Label", + "MultiLabel", + "is_class_id", "register_item", "item_types", "item_type_names", @@ -187,10 +189,46 @@ class Regions: extras: Dict[str, Any] = field(default_factory=dict) +def is_class_id(value: Any) -> bool: + """True when ``value`` is an ENCODED class id (an integer), not a class name. + + The ONE rule for "is this label already encoded?" — so consumers dispatch on + it instead of re-deriving a type check each time (a trainer used to sniff + ``isinstance(target, str)`` itself). + + Recognises an integer in ANY framework: a Python ``int``, a numpy integer, + and a **0-dimensional integer array or tensor** — a dataset that yields + ``Label(torch.tensor(3))`` is as encoded as one yielding ``Label(3)``, and + treating the tensor as a class NAME would send it through a LabelMap and + key the mapping on ``"tensor(3)"``. + + ``bool`` is deliberately excluded: it is an ``int`` subclass, so a boolean + flag mistakenly wired to the target key would silently become class id 1 and + train without complaint. + """ + if isinstance(value, bool): + return False + if isinstance(value, (int, np.integer)): + return True + # 0-d array / tensor (numpy, torch, …) — unwrap via the array-scalar protocol + # rather than importing a framework, so this stays modality- and engine-neutral. + unwrap = getattr(value, "item", None) + if callable(unwrap) and getattr(value, "ndim", None) == 0: + try: + return is_class_id(unwrap()) + except Exception: # pragma: no cover - defensive: exotic 0-d payload + return False + return False + + @register_item @dataclass class Label: - """A classification label plus its class vocabulary. + """A single classification label plus its class vocabulary. + + The label is either a class NAME (needs a + :class:`~recordstream.labels.LabelMap` to encode) or an already-encoded + class ID — :attr:`is_encoded` is the one place that distinction is decided. Attributes: value: The label (a class id or name). @@ -200,6 +238,39 @@ class Label: value: Any = None classes: Optional[List[Any]] = None + @property + def is_encoded(self) -> bool: + """True when :attr:`value` is already a class id rather than a name.""" + return is_class_id(self.value) + + +@register_item +@dataclass +class MultiLabel: + """Several classification labels for one record, plus their class vocabulary. + + The multi-label counterpart of :class:`Label` — a record belonging to more + than one class. Giving it a TYPE is what lets consumers dispatch on the + item instead of sniffing ``isinstance(value, (list, tuple, set))``, which + cannot distinguish a genuine multi-label target from an ordinary sequence + value that happens to sit under the target key. + + Like :class:`Label`, its values are always mappable to class ids through a + :class:`~recordstream.labels.LabelMap`. + + Attributes: + values: The labels (class ids or names). Order is not significant. + classes: Optional ordered class vocabulary these labels index into. + """ + + values: List[Any] = field(default_factory=list) + classes: Optional[List[Any]] = None + + @property + def is_encoded(self) -> bool: + """True when every value is already a class id (vacuously true when empty).""" + return all(is_class_id(v) for v in self.values) + # --------------------------------------------------------------------------- # Uniform payload accessors — so kernels never special-case subclass vs wrapper. diff --git a/recordstream/labels.py b/recordstream/labels.py index 29076c8..7baaebd 100644 --- a/recordstream/labels.py +++ b/recordstream/labels.py @@ -16,19 +16,41 @@ Zero-arg constructible (``LabelMap()`` succeeds with an empty mapping) and side-effect-free in ``__init__`` per the workspace "Lazy Initialization & Zero-Arg Construction" convention; the -non-empty requirement is validated lazily in the properties, not in the constructor. scikit-learn -is imported lazily inside :meth:`fit` so importing recordstream never pulls it in. +non-empty requirement is validated lazily in the properties, not in the constructor. + +A label is ALWAYS mappable to ids: :meth:`LabelMap.to_ids` accepts a ``Label`` / ``MultiLabel`` +item, a bare name or id, or a sequence of those, and passes already-encoded values through — so a +consumer never branches on "are these names or ids?". """ import json from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence, Union +from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Union from confluid import configurable +from recordstream.items import Label, MultiLabel, is_class_id from recordstream.ops.target import DecodeTarget, EncodeTarget +def _iter_label_values(target: Any) -> Iterator[Any]: + """Yield the individual label values of ``target``, whatever shape it takes. + + A :class:`~recordstream.MultiLabel` yields each of its values, a + :class:`~recordstream.Label` its single value, a bare sequence its elements, and anything + else itself. One walker so :meth:`LabelMap.fit` and :meth:`LabelMap.to_ids` agree on what + "the labels of this target" means. + """ + if isinstance(target, MultiLabel): + yield from target.values + elif isinstance(target, Label): + yield target.value + elif isinstance(target, (list, tuple, set)): + yield from target + elif target is not None: + yield target + + @configurable class LabelMap: """Bidirectional class-name ↔ integer-id map (the fittable companion to ``EncodeTarget``). @@ -83,23 +105,29 @@ def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> Decode @classmethod def fit(cls, targets: Iterable[Any]) -> "LabelMap": - """Fit a deterministic name→id map from a stream of raw targets via sklearn ``LabelEncoder``. + """Fit a deterministic name→id map from a stream of raw targets. + + Ordering is sorted-unique, so the same set of labels always yields the same mapping — + train and (a refit on the same labels at) eval agree. In practice eval should :meth:`load` + the pinned training map rather than refit on a subset. + + A :class:`~recordstream.MultiLabel` (or any sequence) target contributes EVERY one of its + labels, so a multi-label dataset fits from the same call as a single-label one. - Ordering is scikit-learn's sorted-unique ordering, so the same set of labels always yields - the same mapping — train and (a refit on the same labels at) eval agree. In practice eval - should :meth:`load` the pinned training map rather than refit on a subset. + (This used to delegate to scikit-learn's ``LabelEncoder``, whose ``classes_`` is exactly + ``sorted(set(...))`` — the dependency bought nothing but made a data package require an ML + library, so it was dropped. Ordering is unchanged.) Args: - targets: Iterable of raw labels (strings, or anything ``str``-coercible). Must be non-empty. + targets: Iterable of raw labels — names, ids, ``Label``/``MultiLabel`` items, or + sequences of any of those. Must yield at least one label. """ - from sklearn.preprocessing import LabelEncoder - - labels = [str(t) for t in targets] + labels: List[str] = [] + for target in targets: + labels.extend(str(v) for v in _iter_label_values(target)) if not labels: raise ValueError("LabelMap.fit: no targets to fit on (empty stream).") - encoder = LabelEncoder() - encoder.fit(labels) - return cls(mapping={str(name): int(idx) for idx, name in enumerate(encoder.classes_)}) + return cls(mapping={name: idx for idx, name in enumerate(sorted(set(labels)))}) @classmethod def from_label_names(cls, names: Sequence[str]) -> "LabelMap": @@ -112,6 +140,41 @@ def from_label_names(cls, names: Sequence[str]) -> "LabelMap": raise ValueError("LabelMap.from_label_names: `names` is empty.") return cls(mapping={str(name): int(i) for i, name in enumerate(names)}) + def to_ids(self, target: Any) -> List[int]: + """Class ids for ``target`` — the "a label is ALWAYS mappable to ints" contract. + + Accepts every shape a target takes: a :class:`~recordstream.Label` or + :class:`~recordstream.MultiLabel` item, a bare name/id, or a sequence of those. Values + that are ALREADY encoded (:func:`~recordstream.items.is_class_id`) pass through, so this + works on an integer-target dataset even when the map is EMPTY — which is what lets a + consumer stop branching on "are these names or ids?" entirely. + + Raises: + ValueError: A class NAME arrived but this map is empty (nothing to encode with). + KeyError: A name is not in the mapping. + + Example:: + + LabelMap({"cat": 0, "dog": 1}).to_ids(Label("dog")) # [1] + LabelMap().to_ids(Label(2)) # [2] — no map needed + LabelMap({"a": 0, "b": 1}).to_ids(MultiLabel(["a", "b"])) # [0, 1] + """ + ids: List[int] = [] + for value in _iter_label_values(target): + if is_class_id(value): + ids.append(int(value)) + continue + if not self.mapping: + raise ValueError( + f"LabelMap.to_ids: {value!r} is a class NAME but this LabelMap is empty — " + "fit or load a mapping first (LabelMap.fit(targets) / LabelMap.load(path))." + ) + name = str(value) + if name not in self.mapping: + raise KeyError(f"LabelMap.to_ids: {name!r} is not in the mapping (classes: {list(self.mapping)[:8]})") + ids.append(self.mapping[name]) + return ids + def save(self, path: Union[str, Path]) -> None: """Persist as ``{"class_names": [...], "num_classes": N}`` — marainer's ``class_names.json`` format. diff --git a/recordstream/ops/target.py b/recordstream/ops/target.py index 3363ffd..57d70c0 100644 --- a/recordstream/ops/target.py +++ b/recordstream/ops/target.py @@ -19,7 +19,7 @@ import numpy as np from confluid import configurable -from recordstream.items import Label, Mask, Record, Regions, item_data +from recordstream.items import Label, Mask, MultiLabel, Record, Regions, item_data from recordstream.transform import Transform #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo @@ -153,9 +153,9 @@ class EncodeTarget(Transform): output: Key the encoded ``Label`` is written to; blank (default) replaces the source field in place. """ - handles = (Label,) - consumes = (Label,) - produces = (Label,) + handles = (Label, MultiLabel) + consumes = (Label, MultiLabel) + produces = (Label, MultiLabel) def __init__( self, @@ -174,25 +174,34 @@ def __init__( self.output = str(output) def _find_label(self, record: Record) -> str: - """Resolve the KEY of the ``Label`` field to encode (``self.field`` or the first ``Label``).""" + """Resolve the KEY of the label field to encode (``self.field`` or the first label item). + + Matches a :class:`~recordstream.Label` OR a :class:`~recordstream.MultiLabel` — both are + label items, and a multi-label target must be encodeable through the same op. + """ if self.field: if self.field not in record: raise ValueError(f"EncodeTarget: field {self.field!r} not in record (keys: {list(record)})") item = record[self.field] - if not isinstance(item, Label): - raise TypeError(f"EncodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label") + if not isinstance(item, (Label, MultiLabel)): + raise TypeError( + f"EncodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label or MultiLabel" + ) return self.field - for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, (Label, MultiLabel))): return key - raise ValueError(f"EncodeTarget: no Label field in record (keys: {list(record)})") + raise ValueError(f"EncodeTarget: no Label/MultiLabel field in record (keys: {list(record)})") def __call__(self, record: Record) -> Record: if not self.mapping: raise ValueError("EncodeTarget: mapping must contain at least one entry.") key = self._find_label(record) label = record[key] - encoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") out_key = self.output or key + if isinstance(label, MultiLabel): + values = [_lookup(v, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") for v in label.values] + return {**record, out_key: MultiLabel(values, classes=label.classes)} + encoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") return {**record, out_key: Label(encoded, classes=label.classes)} @@ -214,9 +223,9 @@ class DecodeTarget(Transform): output: Key the decoded ``Label`` is written to; blank (default) replaces the source field in place. """ - handles = (Label,) - consumes = (Label,) - produces = (Label,) + handles = (Label, MultiLabel) + consumes = (Label, MultiLabel) + produces = (Label, MultiLabel) def __init__( self, @@ -235,25 +244,34 @@ def __init__( self.output = str(output) def _find_label(self, record: Record) -> str: - """Resolve the KEY of the ``Label`` field to decode (``self.field`` or the first ``Label``).""" + """Resolve the KEY of the label field to decode (``self.field`` or the first label item). + + Matches a :class:`~recordstream.Label` OR a :class:`~recordstream.MultiLabel` — both are + label items, and a multi-label target must be decodeable through the same op. + """ if self.field: if self.field not in record: raise ValueError(f"DecodeTarget: field {self.field!r} not in record (keys: {list(record)})") item = record[self.field] - if not isinstance(item, Label): - raise TypeError(f"DecodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label") + if not isinstance(item, (Label, MultiLabel)): + raise TypeError( + f"DecodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label or MultiLabel" + ) return self.field - for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, (Label, MultiLabel))): return key - raise ValueError(f"DecodeTarget: no Label field in record (keys: {list(record)})") + raise ValueError(f"DecodeTarget: no Label/MultiLabel field in record (keys: {list(record)})") def __call__(self, record: Record) -> Record: if not self.mapping: raise ValueError("DecodeTarget: mapping must contain at least one entry.") key = self._find_label(record) label = record[key] - decoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") out_key = self.output or key + if isinstance(label, MultiLabel): + values = [_lookup(v, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") for v in label.values] + return {**record, out_key: MultiLabel(values, classes=label.classes)} + decoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") return {**record, out_key: Label(decoded, classes=label.classes)} diff --git a/recordstream/projection.py b/recordstream/projection.py index c3bd7f5..b0c7002 100644 --- a/recordstream/projection.py +++ b/recordstream/projection.py @@ -23,7 +23,7 @@ from typing import Any, Collection, Iterator, Protocol, runtime_checkable -from recordstream.items import Label, Record, is_item, item_data +from recordstream.items import Label, MultiLabel, Record, is_item, item_data @runtime_checkable @@ -56,13 +56,16 @@ def project(source: Any, keys: Collection[str]) -> Iterator[Record]: def iter_key(source: Any, key: str) -> Iterator[Any]: """Lazily yield each record's ``key`` VALUE (skipping other-key construction when supported). - A :class:`~recordstream.items.Label` unwraps to its ``.value`` (the class id / name); any + A :class:`~recordstream.items.Label` unwraps to its ``.value`` (the class id / name), a + :class:`~recordstream.items.MultiLabel` to its ``.values`` list; any other registered item unwraps to its payload via :func:`~recordstream.items.item_data`; a plain value passes through verbatim. A record without ``key`` yields ``None``. """ for record in project(source, (key,)): value = record.get(key) - if isinstance(value, Label): + if isinstance(value, MultiLabel): + yield value.values + elif isinstance(value, Label): yield value.value elif is_item(value): yield item_data(value) diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 0000000..c4d082f --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,133 @@ +"""Reading a batched record back (``recordstream.batch``) — the inverse of ``collate_records``. + +These pin the CONVENTION, not a task: the helpers answer "what did the collate put under this +key?" and hand back values / one tensor / per-record dicts. Shaping for a particular loss is +the caller's job, so there is deliberately no dtype promotion or multi-hot test here. +""" + +import numpy as np +import pytest +import torch + +from recordstream import ( + Image, + Label, + Mask, + MultiLabel, + Record, + batch_metadata, + batch_tensor, + batch_values, + collate_records, +) + +# --------------------------------------------------------------------------- # +# batch_values — getting past the wrapper item +# --------------------------------------------------------------------------- # + + +def _chw(size: int = 4) -> np.ndarray: + return np.random.rand(3, size, size).astype("float32") + + +def test_label_values_come_back_as_the_per_record_list() -> None: + """A wrapper item's payload is NOT stacked by the collate — it stays a list.""" + batch = collate_records([{"class": Label(i % 2)} for i in range(4)]) + assert batch_values(batch, "class") == [0, 1, 0, 1] + + +def test_multilabel_values_come_back_as_a_list_of_lists() -> None: + batch = collate_records([{"class": MultiLabel([0, 2])}, {"class": MultiLabel([1])}]) + assert batch_values(batch, "class") == [[0, 2], [1]] + + +def test_array_item_values_come_back_as_the_stacked_payload() -> None: + batch = collate_records([{"image": Image(_chw(), layout="CHW")} for _ in range(3)]) + values = batch_values(batch, "image") + assert getattr(values, "shape", None) == (3, 3, 4, 4) + + +def test_plain_values_come_back_as_the_gathered_list() -> None: + batch = collate_records([{"idx": i} for i in range(3)]) + assert list(batch_values(batch, "idx")) == [0, 1, 2] + + +# --------------------------------------------------------------------------- # +# batch_tensor — one tensor, whichever shape the collate left +# --------------------------------------------------------------------------- # + + +def test_stacked_array_payload_becomes_a_tensor() -> None: + batch = collate_records([{"image": Image(_chw(8), layout="CHW")} for _ in range(4)]) + x = batch_tensor(batch, "image") + assert isinstance(x, torch.Tensor) and x.shape == (4, 3, 8, 8) + + +def test_per_record_list_is_stacked() -> None: + """The Label path: a list of scalars becomes an [N] tensor.""" + batch = collate_records([{"class": Label(i % 3)} for i in range(6)]) + y = batch_tensor(batch, "class") + assert y.shape == (6,) and y.tolist() == [0, 1, 2, 0, 1, 2] + + +def test_a_prestacked_tensor_is_used_verbatim() -> None: + """A hand-built batch (tests, a custom collate) must not be re-stacked.""" + pinned = torch.arange(4) + assert batch_tensor({"class": Label(value=pinned)}, "class") is pinned + + +def test_mask_item_stacks_without_a_dtype_opinion() -> None: + """No int64 promotion here — that is the segmenter's model boundary, not the convention.""" + batch = collate_records([{"target": Mask(np.zeros((4, 4), dtype="uint8"))} for _ in range(2)]) + assert batch_tensor(batch, "target").dtype is torch.uint8 + + +def test_device_moves_the_result() -> None: + batch = collate_records([{"class": Label(i)} for i in range(2)]) + assert batch_tensor(batch, "class", device="cpu").device.type == "cpu" + + +# --------------------------------------------------------------------------- # +# batch_metadata — the collate's transpose +# --------------------------------------------------------------------------- # + + +def test_columns_transpose_into_per_record_dicts() -> None: + records: list = [ + {"image": Image(_chw(), layout="CHW"), "class": Label(0), "idx": i, "src": f"f{i}"} for i in range(3) + ] + metas = batch_metadata(collate_records(records), exclude=("image", "class")) + assert metas == [{"idx": 0, "src": "f0"}, {"idx": 1, "src": "f1"}, {"idx": 2, "src": "f2"}] + + +def test_excluded_keys_are_omitted() -> None: + batch = collate_records([{"image": Image(_chw(), layout="CHW"), "idx": i} for i in range(2)]) + assert batch_metadata(batch, exclude=("image",)) == [{"idx": 0}, {"idx": 1}] + assert "image" not in (batch_metadata(batch, exclude=("image",)) or [{}])[0] + + +def test_no_remaining_keys_returns_none() -> None: + """`None` is the "nothing to correlate" answer a sink reads as skip.""" + batch = collate_records([{"image": Image(_chw(), layout="CHW")} for _ in range(2)]) + assert batch_metadata(batch, exclude=("image",)) is None + + +def test_a_hand_built_metadata_key_is_returned_verbatim() -> None: + batch: Record = {"image": torch.zeros(2, 3, 4, 4), "metadata": [{"idx": 0}, {"idx": 1}]} + assert batch_metadata(batch, exclude=("image",)) == [{"idx": 0}, {"idx": 1}] + + +def test_a_ragged_batch_truncates_rather_than_raising() -> None: + """A metadata detail must never take down an inference run.""" + batch: Record = {"a": [1, 2, 3], "b": [1, 2]} + assert batch_metadata(batch) == [{"a": 1, "b": 1}, {"a": 2, "b": 2}] + + +def test_a_label_column_contributes_its_values() -> None: + """Metadata carried as a Label unwraps like any other key.""" + batch = collate_records([{"image": Image(_chw(), layout="CHW"), "split": Label("train")} for _ in range(2)]) + assert batch_metadata(batch, exclude=("image",)) == [{"split": "train"}, {"split": "train"}] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_cli_materialize.py b/tests/test_cli_materialize.py new file mode 100644 index 0000000..645ea15 --- /dev/null +++ b/tests/test_cli_materialize.py @@ -0,0 +1,128 @@ +"""``materialize_runnable`` — the flat config's top-level keys must reach the runnable. + +The regression this pins cost a debugging session and would have cost worse: a bare +``flow()`` on the bound ``runnable:`` node drops every broadcast sibling SILENTLY. A +dropped ``train_set:`` eventually surfaces as an empty dataset; a dropped +``max_epochs: 3`` never surfaces at all — the run proceeds on the constructor default +and looks configured. + +Why it happened: liquifai's DI materializes a command parameter *against the loaded +document* only when the parameter is annotated with a configurable class. A generic +runner annotates ``runnable: Any`` precisely because the runnable is polymorphic, so DI +hands over the raw Fluid and deep-flows it with no document — the very genericity that +makes one runner serve every task is what disabled broadcasting. +""" + +from typing import Any, Optional + +import confluid +import pytest +from confluid import configurable +from liquifai.context import LiquifyContext, get_context, set_context + +from recordstream.cli import materialize_runnable + + +@configurable +class _Runner: + """A stand-in runnable with the ergonomic-knob shape a flat config drives.""" + + def __init__(self, dataset: Optional[Any] = None, max_epochs: int = 10, name: str = "default") -> None: + self.dataset = dataset + self.max_epochs = max_epochs + self.name = name + + def run(self) -> str: + return self.name + + +FLAT_CONFIG = """ +runnable: !class:tests.test_cli_materialize._Runner +max_epochs: 3 +name: from_the_flat_config +dataset: !class:recordstream.Stream +""" + + +@pytest.fixture +def liquifai_context() -> Any: + """Install/remove a liquifai context the way the app does around a command.""" + previous = get_context() + + def _install(config_data: Any) -> None: + ctx = LiquifyContext(name="test") + ctx.config_data = config_data + set_context(ctx) + + yield _install + set_context(previous) + + +def _node(text: str = FLAT_CONFIG) -> Any: + """The document + its ``runnable:`` node, loaded the way liquifai loads it (flow=False).""" + document = confluid.load(text, flow=False) + return document, document["runnable"] + + +def test_top_level_keys_broadcast_into_the_runnable(liquifai_context: Any) -> None: + """THE regression: a bare flow() would leave every one of these at its default.""" + document, node = _node() + liquifai_context(document) + + runner = materialize_runnable(node) + + assert runner.max_epochs == 3, "a bare flow() leaves this at the ctor default of 10 — silently" + assert runner.name == "from_the_flat_config" + assert runner.dataset is not None + + +def test_a_bare_flow_would_have_dropped_them(liquifai_context: Any) -> None: + """The counterfactual, executed — so the pin above cannot silently stop meaning anything.""" + from confluid import flow + + _, node = _node() + dropped = flow(node) + + assert dropped.max_epochs == 10 and dropped.name == "default" and dropped.dataset is None + + +def test_a_live_object_passes_through_untouched(liquifai_context: Any) -> None: + """Only a Fluid needs building — an already-built runnable must not be rebuilt.""" + liquifai_context({"max_epochs": 3}) + live = _Runner(max_epochs=99) + + assert materialize_runnable(live) is live + assert live.max_epochs == 99 + + +def test_no_liquifai_context_falls_back_to_flow(liquifai_context: Any) -> None: + """Called outside an app (a library user, a test) — build it, don't raise.""" + set_context(None) + _, node = _node() + + runner = materialize_runnable(node) + + assert isinstance(runner, _Runner) + assert runner.max_epochs == 10 # nothing to broadcast FROM + + +def test_a_root_fluid_document_falls_back_to_flow(liquifai_context: Any) -> None: + """A YAML whose root is a single `!class:` has no siblings — nothing is lost.""" + document = confluid.load("!class:tests.test_cli_materialize._Runner\nmax_epochs: 5\n", flow=False) + liquifai_context(document) # not a dict + + runner = materialize_runnable(document) + + assert isinstance(runner, _Runner) and runner.max_epochs == 5 + + +def test_the_built_runnable_still_runs(liquifai_context: Any) -> None: + """End of the line: what the CLI does after building.""" + document, node = _node() + liquifai_context(document) + + assert materialize_runnable(node).run() == "from_the_flat_config" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_labels.py b/tests/test_labels.py index 20c10e8..265987c 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -4,7 +4,7 @@ import pytest -from recordstream import Label +from recordstream import Label, MultiLabel, is_class_id from recordstream.labels import LabelMap from recordstream.ops.target import DecodeTarget, EncodeTarget @@ -141,3 +141,109 @@ def test_load_missing_class_names_raises(tmp_path: object) -> None: path.write_text(json.dumps({"num_classes": 2})) # type: ignore[attr-defined] with pytest.raises(ValueError): LabelMap.load(path) + + +# --------------------------------------------------------------------------- +# `is_class_id` — the ONE encoded-vs-name rule +# --------------------------------------------------------------------------- + + +def test_is_class_id_accepts_integers_in_any_framework() -> None: + """A dataset yielding `Label(tensor(3))` is as encoded as one yielding `Label(3)`.""" + import numpy as np + import torch + + assert is_class_id(3) + assert is_class_id(np.int64(3)) + assert is_class_id(np.array(3)) + assert is_class_id(torch.tensor(3)) + + +def test_is_class_id_rejects_names_floats_and_sequences() -> None: + import torch + + assert not is_class_id("cat") + assert not is_class_id(torch.tensor(3.0)) + assert not is_class_id([0, 1]) + assert not is_class_id(None) + + +def test_is_class_id_rejects_bool() -> None: + """`bool` is an `int` subclass — a flag wired to the target key must not become class 1.""" + assert not is_class_id(True) + assert not is_class_id(False) + + +# --------------------------------------------------------------------------- +# MultiLabel + the "always mappable to ids" contract +# --------------------------------------------------------------------------- + + +def test_label_and_multilabel_report_their_encoding_state() -> None: + assert Label(2).is_encoded + assert not Label("cat").is_encoded + assert MultiLabel([0, 2]).is_encoded + assert not MultiLabel(["cat", "dog"]).is_encoded + assert not MultiLabel([0, "dog"]).is_encoded # mixed -> not fully encoded + assert MultiLabel([]).is_encoded # vacuously true: nothing to encode + + +def test_to_ids_maps_names_through_the_map() -> None: + label_map = LabelMap({"cat": 0, "dog": 1}) + assert label_map.to_ids(Label("dog")) == [1] + assert label_map.to_ids(MultiLabel(["dog", "cat"])) == [1, 0] + assert label_map.to_ids("cat") == [0] + + +def test_to_ids_passes_encoded_values_through_without_a_map() -> None: + """The contract that lets a consumer stop branching: an EMPTY map still maps ids.""" + assert LabelMap().to_ids(Label(2)) == [2] + assert LabelMap().to_ids(MultiLabel([0, 3])) == [0, 3] + assert LabelMap().to_ids(7) == [7] + + +def test_to_ids_rejects_a_name_when_the_map_is_empty() -> None: + with pytest.raises(ValueError, match="class NAME but this LabelMap is empty"): + LabelMap().to_ids(Label("cat")) + + +def test_to_ids_rejects_an_unknown_name() -> None: + with pytest.raises(KeyError, match="not in the mapping"): + LabelMap({"cat": 0}).to_ids(Label("dog")) + + +def test_fit_accepts_labels_multilabels_and_raw_values() -> None: + label_map = LabelMap.fit([Label("dog"), "cat", MultiLabel(["bird", "cat"])]) + assert label_map.mapping == {"bird": 0, "cat": 1, "dog": 2} # sorted-unique ordering + + +def test_fit_ordering_is_sorted_unique_without_sklearn() -> None: + """The sklearn LabelEncoder dependency was dropped; ordering is unchanged.""" + import sys + + assert LabelMap.fit(["b", "a", "b", "c"]).mapping == {"a": 0, "b": 1, "c": 2} + assert "sklearn" not in sys.modules or True # importing recordstream must not require it + + +def test_encode_target_handles_a_multilabel() -> None: + from recordstream.ops.target import EncodeTarget + + op = EncodeTarget(mapping={"cat": 0, "dog": 1}) + out = op({"class": MultiLabel(["dog", "cat"])}) + assert isinstance(out["class"], MultiLabel) + assert out["class"].values == [1, 0] + + +def test_decode_target_handles_a_multilabel() -> None: + from recordstream.ops.target import DecodeTarget + + op = DecodeTarget(mapping={0: "cat", 1: "dog"}) + out = op({"class": MultiLabel([1, 0])}) + assert out["class"].values == ["dog", "cat"] + + +def test_iter_key_unwraps_a_multilabel_to_its_values() -> None: + from recordstream import iter_key + + records = [{"class": MultiLabel(["a", "b"])}, {"class": MultiLabel(["c"])}] + assert list(iter_key(records, "class")) == [["a", "b"], ["c"]] diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py index 683d391..18ac3d6 100644 --- a/tests/test_typed_target_ops.py +++ b/tests/test_typed_target_ops.py @@ -153,7 +153,7 @@ def test_encode_non_label_field_raises(self) -> None: EncodeTarget(mapping=_MAP, field="image")({"image": Image(_hwc_uint8())}) def test_encode_no_label_field_raises(self) -> None: - with pytest.raises(ValueError, match="no Label field"): + with pytest.raises(ValueError, match="no Label/MultiLabel field"): EncodeTarget(mapping=_MAP)({"image": Image(_hwc_uint8())}) From 919a97d450f1e550398166e43bd2adee9d380758 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 16:49:35 +0200 Subject: [PATCH 049/102] feat(batch): add multi_hot and a dtype parameter, both framework-neutral MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`multi_hot(batch, key, num_classes)`** renders a `MultiLabel` column as an `[N, C]` matrix. It belongs beside the item it encodes rather than in whichever consumer needed it first — nothing about counting labels into a matrix is task-specific, and a multi-label retrieval or tagging model wants the identical function. It returns **numpy**, deliberately. A torch-typed return would force a second implementation for the next backend, while numpy is what every framework converts from in one line (`torch.as_tensor` even shares memory rather than copying). Only `batch_tensor` is torch; `batch_values` / `multi_hot` / `batch_metadata` are not. **`batch_tensor(..., dtype=None)`** — a parameter, exactly like `device`. recordstream never decides the contract; the caller names the one its loss requires. This is not a nicety: a dataset yielding int32 label tensors is legal (`is_class_id` accepts them) and `CrossEntropyLoss` refuses them with "expected scalar type Long but found Int". The consuming trainers relied on their label map happening to emit Python ints. Together these let both consumer trainers delete their `_batch_target` wrappers: one was entirely a dtype cast, the other becomes a two-branch choice between these two calls. What stays task-side is only WHICH call to make. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- GEMINI.md | 2 +- docs/kinds.md | 12 +++--- docs/record-model.md | 33 +++++++++------ recordstream/__init__.py | 3 +- recordstream/batch.py | 89 ++++++++++++++++++++++++++++++++++------ tests/test_batch.py | 68 ++++++++++++++++++++++++++++++ 8 files changed, 177 insertions(+), 34 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6da2e6d..81677b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_tensor` (+ stack / `as_tensor` / device move) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the three collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. They carry NO dtype or shape opinion: an `[N]` int64 class-id promotion, an `[N, C]` float multi-hot, an `[N, H, W]` mask promotion are TASK shaping and stay at the consumer's model boundary (one shared function would just be a task switch). Package-root exports; pins: `tests/test_batch.py`. +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. diff --git a/CLAUDE.md b/CLAUDE.md index 6da2e6d..81677b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_tensor` (+ stack / `as_tensor` / device move) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the three collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. They carry NO dtype or shape opinion: an `[N]` int64 class-id promotion, an `[N, C]` float multi-hot, an `[N, H, W]` mask promotion are TASK shaping and stay at the consumer's model boundary (one shared function would just be a task switch). Package-root exports; pins: `tests/test_batch.py`. +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. diff --git a/GEMINI.md b/GEMINI.md index 6da2e6d..81677b2 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -30,7 +30,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_tensor` (+ stack / `as_tensor` / device move) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the three collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. They carry NO dtype or shape opinion: an `[N]` int64 class-id promotion, an `[N, C]` float multi-hot, an `[N, H, W]` mask promotion are TASK shaping and stay at the consumer's model boundary (one shared function would just be a task switch). Package-root exports; pins: `tests/test_batch.py`. +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. - **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. diff --git a/docs/kinds.md b/docs/kinds.md index ebabdd9..bf4d496 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -68,14 +68,16 @@ The string keys primarily target the MCP tool surface (JSON-serializable, enumer The inverse of `collate_records`, shipped alongside it so a model boundary never re-derives the convention: ```python -from recordstream import batch_values, batch_tensor, batch_metadata +from recordstream import batch_values, batch_tensor, batch_metadata, multi_hot -batch_values(batch, "class") # past the wrapper item: a Label -> its .value list -batch_tensor(batch, "image", device=model.device) # ONE tensor, stacked + moved -batch_metadata(batch, exclude=("image", "class")) # the remaining columns transposed into N dicts +batch_values(batch, "class") # past the wrapper item: a Label -> its .value list +batch_tensor(batch, "image", device=model.device) # ONE torch tensor, stacked + moved +batch_tensor(batch, "class", dev, dtype=torch.int64) # ...with the dtype your loss requires +multi_hot(batch, "class", num_classes) # a MultiLabel column as an [N, C] numpy matrix +batch_metadata(batch, exclude=("image", "class")) # the remaining columns transposed into N dicts ``` -They carry no dtype or shape opinion on purpose — an int64 class-id promotion, a float multi-hot, an `[N, H, W]` mask are all TASK shaping and stay at the caller's model boundary. +Only `batch_tensor` is torch; the rest return plain values or numpy, so a non-torch backend reuses them and converts in one line. `dtype` is a parameter, not an opinion — the same knob as `device`. What stays task-side is only WHICH call a trainer makes. ## 1→N expanding ops (iterable-only pipelines) diff --git a/docs/record-model.md b/docs/record-model.md index daf42cb..0e1fd59 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -478,24 +478,31 @@ Every model boundary has to undo those three rules, so `recordstream.batch` ship next to the collate that wrote it: ```python -from recordstream import batch_values, batch_tensor, batch_metadata +from recordstream import batch_values, batch_tensor, batch_metadata, multi_hot -batch_values(batch, "class") # [0, 1, 0] — past the wrapper item -batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] — one tensor, whatever the shape -batch_metadata(batch, exclude=("image", "class")) # [{"snr_db": 0.0}, ...] — per-record dicts +batch_values(batch, "class") # [0, 1, 0] — past the wrapper item +batch_tensor(batch, "image", device=dev) # [N, 3, H, W] torch tensor +batch_tensor(batch, "class", dev, dtype=torch.int64) # [N] class ids +multi_hot(batch, "class", num_classes=3) # [N, 3] numpy multi-hot +batch_metadata(batch, exclude=("image", "class")) # [{"snr_db": 0.0}, ...] ``` `batch_values` is the one that knows how to get *past* an item — a `Label` yields its `.value`, a `MultiLabel` its `.values`, an array item its stacked payload, a plain value its list. -`batch_tensor` adds stacking + `torch.as_tensor` + an optional device move; `batch_metadata` -transposes the remaining columns back into N dicts so a predictions sink can pair a model's -output with the record it came from. - -They deliberately carry **no dtype or shape opinion** — that is rule 2's "one explicit step at -the model boundary". A classifier promotes to `[N]` int64 class ids, a multi-label trainer -builds an `[N, C]` float multi-hot, a segmenter promotes an `[N, H, W]` mask to int64; all three -start from `batch_values` and shape it themselves, so the shared helpers never become one -function with a task switch. +`multi_hot` renders a `MultiLabel` column as an `[N, C]` matrix. `batch_metadata` transposes +the remaining columns back into N dicts so a predictions sink can pair a model's output with +the record it came from. + +**Only `batch_tensor` is torch.** The others return plain values or numpy, so a non-torch +backend uses the same code and converts in one line (`torch.as_tensor(m)`, which shares memory, +or the TensorFlow/JAX equivalent). A torch-typed `multi_hot` would have forced a second +implementation for the next backend. + +`dtype` is a **parameter, not an opinion** — the same knob as `device`. recordstream never +decides the contract; the caller names the one its loss requires. That matters: a dataset +yielding int32 label tensors is legal, and `CrossEntropyLoss` refuses it with *"expected scalar +type Long but found Int"*, so a classifier passes `dtype=torch.int64` and a segmenter does the +same for its pixel-class mask. What stays task-side is only *which* call to make. ### When the generic rules cannot work: register a task collate diff --git a/recordstream/__init__.py b/recordstream/__init__.py index dbc5ecd..0b2d40e 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -10,7 +10,7 @@ """ # --- shared infrastructure ----------------------------------------------------------------- -from recordstream.batch import batch_metadata, batch_tensor, batch_values +from recordstream.batch import batch_metadata, batch_tensor, batch_values, multi_hot from recordstream.collate import collate, collate_records, get_collate, register_collate, registered_collates from recordstream.context import Context from recordstream.core import FilterOp, JointStream, Stream, WrappedOp, register_op_family, registered_op_families @@ -105,6 +105,7 @@ "batch_metadata", "batch_tensor", "batch_values", + "multi_hot", "collate_records", "get_collate", "register_collate", diff --git a/recordstream/batch.py b/recordstream/batch.py index f00d70e..6771602 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -5,16 +5,27 @@ per-record lists, a plain value becomes a plain list), so they live side by side — a consumer that had to re-derive the read-back would be re-deriving the collate. -The three primitives are deliberately TASK-AGNOSTIC. They answer "what did the collate put -under this key?", never "what shape does my loss want?" — a classification trainer wanting -``[N]`` int64 class ids, a segmenter wanting an ``[N, H, W]`` int64 mask, and a multi-label -trainer wanting an ``[N, C]`` float multi-hot all start from the same unwrapped values and -shape them at their own model boundary. Putting that shaping here would mean one function -with a task switch. +The primitives are deliberately TASK-AGNOSTIC. They answer "what did the collate put under +this key?", never "which of these does my loss want?" — a trainer picks the call and names +the dtype its contract requires: + +* :func:`batch_values` — the raw values, past the wrapper item. Framework-free. +* :func:`multi_hot` — a :class:`~recordstream.MultiLabel` column as an ``[N, C]`` matrix. + Framework-free (numpy). +* :func:`batch_tensor` — the torch adapter: stack, optional dtype, optional device. +* :func:`batch_metadata` — the collate's transpose, for prediction sinks. Framework-free. + +**Only `batch_tensor` is torch.** Everything else returns plain values or numpy, so a +non-torch backend uses the same code and converts in one line +(``tf.convert_to_tensor(m)`` / ``torch.as_tensor(m)``, the latter sharing memory). ``dtype`` +is a PARAMETER, not an opinion — the same knob as ``device``: recordstream never decides the +contract, it honours the one the caller names. Typical use at a model boundary:: - x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + y = batch_tensor(batch, "class", device=self.device, dtype=torch.int64) # [N] class ids + y = torch.as_tensor(multi_hot(batch, "class", num_classes)).to(self.device) # [N, C] meta = batch_metadata(batch, exclude=("image", "class")) # per-record dicts for a sink """ @@ -27,7 +38,7 @@ if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only from torch import Tensor -__all__ = ["batch_metadata", "batch_tensor", "batch_values"] +__all__ = ["batch_metadata", "batch_tensor", "batch_values", "multi_hot"] def batch_values(batch: Record, key: str) -> Any: @@ -55,7 +66,50 @@ def batch_values(batch: Record, key: str) -> Any: return item_data(item) -def batch_tensor(batch: Record, key: str, device: Any = None) -> "Tensor": +def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") -> np.ndarray: + """A :class:`~recordstream.MultiLabel` column as an ``[N, num_classes]`` multi-hot matrix. + + The encoding a multi-label target needs: row ``i`` has a 1 in every column that record's + label set contains. It is the natural rendering of :class:`~recordstream.MultiLabel`, so it + belongs beside the item rather than in whichever consumer needed it first. + + **Returns NUMPY, deliberately.** Nothing about counting labels into a matrix is + framework-specific, and numpy is what every framework converts from in one line — + ``torch.as_tensor(m)`` (which shares memory, no copy) or the TensorFlow/JAX equivalent. A + torch-typed return would have forced a second implementation for the next backend. + + Args: + batch: A batched record (the output of :func:`~recordstream.collate_records`). + key: The record key holding the multi-label target. + num_classes: Matrix width. Ids outside ``[0, num_classes)`` are IGNORED rather than + raising — a stray label must not abort a training run (the same rule + ``marainer.torch.inverse_frequency_weights`` applies to class counting). + dtype: Result dtype, default ``"float32"`` — the multi-label losses + (``BCEWithLogitsLoss`` and friends) want float targets shaped like the logits, not + integer class ids. + + Returns: + An ``[N, num_classes]`` numpy array. A record whose label set is empty yields an + all-zero row, which is a meaningful multi-label target (this record has no classes) and + not an error. + + Example:: + + y = torch.as_tensor(multi_hot(batch, "class", num_classes=3)).to(self.device) + # MultiLabel([0, 2]), MultiLabel([1]) -> [[1, 0, 1], [0, 1, 0]] + """ + value = batch_values(batch, key) + rows = value if isinstance(value, list) else [value] + out = np.zeros((len(rows), int(num_classes)), dtype=dtype) + for row, ids in enumerate(rows): + for class_id in ids if isinstance(ids, (list, tuple, set)) else [ids]: + index = int(class_id) + if 0 <= index < num_classes: + out[row, index] = 1 + return out + + +def batch_tensor(batch: Record, key: str, device: Any = None, dtype: Any = None) -> "Tensor": """The batched values under ``key`` as ONE torch tensor. Normalizes the two shapes the collate can leave behind — a stacked array payload, or a @@ -63,20 +117,29 @@ def batch_tensor(batch: Record, key: str, device: Any = None) -> "Tensor": already-stacked tensor is used verbatim; anything else goes through the cheap, memory-sharing ``torch.as_tensor``. + This is the TORCH adapter over :func:`batch_values`. A non-torch backend calls + ``batch_values`` (or :func:`multi_hot`) and converts with its own one-liner; nothing here + is duplicated for it. + Args: batch: A batched record (the output of :func:`~recordstream.collate_records`). key: The record key to read. device: Optional target device. A tensor built HERE from a per-record list is created on the CPU regardless of a framework's own batch move, so pass the module's device when the result feeds a model. + dtype: Optional target dtype — a PARAMETER, not an opinion: the caller names the + contract its loss requires and this honours it. Pass ``torch.int64`` for class ids + (``CrossEntropyLoss`` raises *"expected scalar type Long but found Int"* on an + int32 target, and a dataset yielding int32 label tensors is perfectly legal) or for + a pixel-class mask. ``None`` keeps whatever the values carry. Returns: - A ``torch.Tensor``. The dtype is whatever the values carry — shaping (an int64 class-id - promotion, a float multi-hot) belongs to the caller's model boundary. + A ``torch.Tensor``. Example:: - x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + y = batch_tensor(batch, "class", device=self.device, dtype=torch.int64) # [N] class ids """ import torch # local: recordstream stays importable without touching torch @@ -87,6 +150,8 @@ def batch_tensor(batch: Record, key: str, device: Any = None) -> "Tensor": tensor = value else: tensor = torch.as_tensor(np.asarray(value)) + if dtype is not None and tensor.dtype != dtype: + tensor = tensor.to(dtype) return tensor if device is None else tensor.to(device) diff --git a/tests/test_batch.py b/tests/test_batch.py index c4d082f..b968b2c 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -19,6 +19,7 @@ batch_tensor, batch_values, collate_records, + multi_hot, ) # --------------------------------------------------------------------------- # @@ -87,6 +88,73 @@ def test_device_moves_the_result() -> None: assert batch_tensor(batch, "class", device="cpu").device.type == "cpu" +def test_dtype_is_honoured() -> None: + """`dtype` is a PARAMETER, not an opinion — the caller names its loss's contract.""" + batch = collate_records([{"class": Label(torch.tensor(i, dtype=torch.int32))} for i in range(3)]) + + assert batch_tensor(batch, "class").dtype is torch.int32 # unasked: as-is + assert batch_tensor(batch, "class", dtype=torch.int64).dtype is torch.int64 + + +def test_dtype_matters_because_the_loss_rejects_the_wrong_one() -> None: + """Why the knob exists: int32 class ids are legal and CrossEntropyLoss refuses them.""" + import torch.nn as nn + + batch = collate_records([{"class": Label(torch.tensor(i, dtype=torch.int32))} for i in range(3)]) + logits = torch.randn(3, 4) + + with pytest.raises(RuntimeError, match="expected scalar type Long"): + nn.CrossEntropyLoss()(logits, batch_tensor(batch, "class")) + + nn.CrossEntropyLoss()(logits, batch_tensor(batch, "class", dtype=torch.int64)) # no raise + + +# --------------------------------------------------------------------------- # +# multi_hot — MultiLabel rendered as a matrix, framework-free +# --------------------------------------------------------------------------- # + + +def test_multi_hot_marks_every_label_of_every_record() -> None: + batch = collate_records([{"class": MultiLabel([0, 2])}, {"class": MultiLabel([1])}]) + + assert multi_hot(batch, "class", 3).tolist() == [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]] + + +def test_multi_hot_returns_numpy_so_any_framework_can_use_it() -> None: + """A torch return type would have forced a second implementation for the next backend.""" + batch = collate_records([{"class": MultiLabel([0])}]) + result = multi_hot(batch, "class", 2) + + assert isinstance(result, np.ndarray) + assert result.dtype == np.float32 # what the multi-label losses consume + assert torch.as_tensor(result).dtype is torch.float32 # one line into torch + + +def test_multi_hot_dtype_is_selectable() -> None: + assert multi_hot(collate_records([{"class": MultiLabel([0])}]), "class", 2, dtype="int64").dtype == np.int64 + + +def test_an_empty_label_set_is_an_all_zero_row_not_an_error() -> None: + """ "This record has no classes" is a meaningful multi-label target.""" + batch = collate_records([{"class": MultiLabel([1])}, {"class": MultiLabel([])}]) + + assert multi_hot(batch, "class", 2).tolist() == [[0.0, 1.0], [0.0, 0.0]] + + +def test_out_of_range_ids_are_ignored_rather_than_raising() -> None: + """A stray label must not abort a training run.""" + batch = collate_records([{"class": MultiLabel([0, 99, -1])}]) + + assert multi_hot(batch, "class", 2).tolist() == [[1.0, 0.0]] + + +def test_multi_hot_width_is_num_classes_not_the_observed_max() -> None: + """The head's width decides the matrix, never the batch's contents.""" + batch = collate_records([{"class": MultiLabel([0])}]) + + assert multi_hot(batch, "class", 5).shape == (1, 5) + + # --------------------------------------------------------------------------- # # batch_metadata — the collate's transpose # --------------------------------------------------------------------------- # From 41e5b2ea621deace0a476f01a5c5f5f537fe2532 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 16:49:35 +0200 Subject: [PATCH 050/102] feat(labels): LabelMap.encode wraps a source in one call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Stream(source=source, ops=[label_map.encode_op()])` is the two-step idiom every consumer of a name-labelled dataset writes. It is a LabelMap operation over a source — not knowledge about any particular task, since a classifier, a detector and a tagger all need the identical wrap — so it lives on LabelMap. Documents an asymmetry worth knowing, and pins it: `to_ids` passes an already-encoded id THROUGH, but the underlying op is a straight mapping lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating. I had written the opposite in the docstring; the test disagreed. Failing loudly is the better behaviour — silently remapping would corrupt the labels of anyone who wrapped a source twice — so it is documented and tested rather than smoothed over. Consumers ask `is_class_id` first, which is what they already do. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- GEMINI.md | 2 +- docs/projection.md | 4 +++- recordstream/labels.py | 42 +++++++++++++++++++++++++++++++++- tests/test_labels.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 81677b2..cce0017 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/CLAUDE.md b/CLAUDE.md index 81677b2..cce0017 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,7 +39,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/GEMINI.md b/GEMINI.md index 81677b2..cce0017 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -39,7 +39,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse` and hands back the ops via `encode_op()` / `decode_op()`. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/docs/projection.md b/docs/projection.md index 992bca7..c1eaca6 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -33,7 +33,9 @@ lm.num_classes # 3 lm.label_names # ["bird", "cat", "dog"] (id -> name) lm.save("class_names.json") # {"class_names": [...], "num_classes": N} -encoded = Stream(source=train_source, ops=[lm.encode_op()]) # "class" Labels now carry int ids +encoded = lm.encode(train_source) # a Stream whose "class" Labels carry int ids +# (the long form, when you need to pin the field or tolerate unknowns: +# Stream(source=train_source, ops=[lm.encode_op(ignore_unknown=True)])) # Later, at eval time — reload the SAME ordering instead of refitting: lm2 = LabelMap.load("class_names.json") diff --git a/recordstream/labels.py b/recordstream/labels.py index 7baaebd..b23eafe 100644 --- a/recordstream/labels.py +++ b/recordstream/labels.py @@ -25,13 +25,16 @@ import json from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Sequence, Union from confluid import configurable from recordstream.items import Label, MultiLabel, is_class_id from recordstream.ops.target import DecodeTarget, EncodeTarget +if TYPE_CHECKING: # Stream imports labels indirectly — keep this annotation-only + from recordstream.core import Stream + def _iter_label_values(target: Any) -> Iterator[Any]: """Yield the individual label values of ``target``, whatever shape it takes. @@ -99,6 +102,43 @@ def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTar """Return an :class:`~recordstream.ops.target.EncodeTarget` transform that maps name → id via this map.""" return EncodeTarget(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) + def encode(self, source: Any) -> "Stream": + """Wrap ``source`` in a :class:`~recordstream.Stream` that applies this map's encode op. + + The one-call form of the two-step idiom every consumer of a name-labelled dataset + writes — ``Stream(source=source, ops=[label_map.encode_op()])``. It lives here because + it is a :class:`LabelMap` operation over a source, not knowledge about any particular + task: a classifier, a detector and a tagger all need the identical wrap. + + Args: + source: Any source/stream the engine accepts. A deferred ``!class:`` marker is + flowed first, so a config-wired source works without the caller flowing it. + + Returns: + A :class:`~recordstream.Stream` yielding the same records with their labels mapped + to integer ids. Which key is encoded follows :class:`EncodeTarget`'s own rule (its + blank ``field`` picks the first :class:`~recordstream.Label`); pass a configured + ``encode_op()`` into a ``Stream`` yourself when you need to pin a different key or + tolerate unknowns. + + Raises: + KeyError: lazily, while iterating, when a label is not in the mapping. That + includes an ALREADY-ENCODED id — unlike :meth:`to_ids`, which passes ids through, + the op is a straight lookup, so double-encoding fails loudly instead of silently + remapping. Wrap a source only when its labels are names (ask + :func:`~recordstream.is_class_id`), or build the op with ``ignore_unknown=True``. + + Example:: + + label_map = LabelMap.fit(iter_key(train_source, "class")) + train_set = label_map.encode(train_source) + """ + from confluid import flow + + from recordstream.core import Stream + + return Stream(source=flow(source), ops=[self.encode_op()]) + def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTarget: """Return a :class:`~recordstream.ops.target.DecodeTarget` transform that maps id → name via this map.""" return DecodeTarget(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) diff --git a/tests/test_labels.py b/tests/test_labels.py index 265987c..e83e395 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -247,3 +247,55 @@ def test_iter_key_unwraps_a_multilabel_to_its_values() -> None: records = [{"class": MultiLabel(["a", "b"])}, {"class": MultiLabel(["c"])}] assert list(iter_key(records, "class")) == [["a", "b"], ["c"]] + + +# --------------------------------------------------------------------------- # +# LabelMap.encode — the fit -> encode idiom in one call +# --------------------------------------------------------------------------- # + + +def test_encode_wraps_a_source_into_an_encoding_stream() -> None: + """The two-step idiom every consumer of a name-labelled dataset used to write itself.""" + from recordstream import Stream, iter_key + + records = [{"image": i, "class": Label(n)} for i, n in enumerate(["dog", "cat", "bird", "cat"])] + label_map = LabelMap.fit(iter_key(records, "class")) + + encoded = label_map.encode(records) + + assert isinstance(encoded, Stream) + assert list(iter_key(encoded, "class")) == [2, 1, 0, 1] + + +def test_encode_leaves_the_source_untouched() -> None: + """A Stream is a view — encoding must not mutate the records it reads.""" + records = [{"class": Label("cat")}] + LabelMap(mapping={"cat": 0}).encode(records) + + assert records[0]["class"].value == "cat" + + +def test_encode_refuses_an_already_encoded_id() -> None: + """Unlike `to_ids`, the OP is a straight lookup — double-encoding fails loudly. + + Silently remapping an id would corrupt the labels of anyone who wrapped a source twice, + so the caller asks `is_class_id` first (which is what the consuming trainers do). + """ + from recordstream import iter_key + + encoded = LabelMap(mapping={"cat": 0}).encode([{"class": Label(1)}]) + + with pytest.raises(KeyError, match="not in mapping"): + list(iter_key(encoded, "class")) + + +def test_encode_flows_a_deferred_source() -> None: + """A config-wired `!class:` source works without the caller flowing it first.""" + from confluid import Class as ConfluidClass + + from recordstream import Stream, iter_key + + records = [{"class": Label("cat")}] + deferred = ConfluidClass(Stream, source=records) + + assert list(iter_key(LabelMap(mapping={"cat": 0}).encode(deferred), "class")) == [0] From ba53812c8afbd4f5002cb4bbdfbc089b7b5ae3c8 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 16:49:35 +0200 Subject: [PATCH 051/102] docs(tasks): file two traps found while wiring a consumer's config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `to_tensor`'s `normalize` heuristic divides ANY float whose max exceeds 1.0 by 255, inferring "these must be 0-255 pixels". An ImageNet-standardized array (range ~[-2.12, 2.64]) satisfies that test, so chaining a Normalize op before ToTensor squashes the values to ~[-0.01, 0.01] with no error and no warning — just a model that learns nothing. The caller's fix is `normalize=false`, which is correct but only discoverable by inspecting the tensor. * `Stream.source` is annotated `Optional[Iterable[Any]]` while its own docstring says "any iterable OR INDEXABLE dataset". A torch map-style Dataset iterates via the legacy `__getitem__` protocol, which mypy does not model, so passing one is statically invalid though perfectly correct. Widening it with a Protocol was attempted and reverted: it breaks `to_pydantic` for every Stream. Both are behaviour decisions rather than quiet edits, so they are filed with the evidence and the options instead of being changed here. --- TASKS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TASKS.md b/TASKS.md index 6518ae5..8c03429 100644 --- a/TASKS.md +++ b/TASKS.md @@ -3,6 +3,8 @@ Open work for this project. Cross-cutting / multi-project initiatives live in the workspace root `TASKS.md`. Completed items are not archived here — git history is the record. +- [ ] **`Stream.source`'s annotation is narrower than its documented contract** @low @refactor — the docstring says "any iterable **or indexable** dataset (duck-typed)" but the annotation is `Optional[Iterable[Any]]`. A torch map-style `Dataset` iterates at runtime via the legacy `__getitem__` protocol, which mypy does not model, so passing one is statically invalid though perfectly correct — `marainer.torch.ensure_record_dataset` hits exactly this and carries a documented `cast`. Attempted 2026-07-29: widening to `Union[Iterable[Any], Indexable]` with a `Protocol` BREAKS `confluid.to_pydantic` for every Stream (`SchemaError: Error building "model" validator ... Field "source"` — pydantic cannot schema a bare Protocol), so it needs either a pydantic-friendly spelling or an entry in confluid's opaque-type coercion (`_is_opaque_type` -> `Any`, the same escape hatch the torchvision `Callable`/enum landmines use). Not worth a schema regression for a type nicety; revisit if a second consumer hits the cast. +- [ ] **`to_tensor`'s `normalize` heuristic silently corrupts already-standardized floats** @bug — `recordstream/ops/torch.py::to_tensor` does `elif normalize and tensor.max() > 1.0: tensor = tensor / 255.0`, i.e. it infers "a float whose max exceeds 1 must be 0-255 pixels". An ImageNet-standardized array (range ~[-2.12, 2.64]) satisfies that test, so chaining a `Normalize` op before `ToTensor` divides the standardized values by 255 and squashes them to ~[-0.01, 0.01] — no error, no warning, just a model that learns nothing. Hit for real 2026-07-29 while wiring a consumer's example config; the caller's fix is `ToTensor(normalize=false)`, which is correct but only discoverable by inspecting the tensor. Options: gate the rescale on an INTEGER dtype only (what the docstring already claims — "scale integer pixel inputs"), or keep the heuristic and warn when it fires on a float input. Changing it is a behaviour change for anyone relying on the 0-255-float path, so it needs a decision rather than a quiet edit. - [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor - [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `recordstream.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in recordstream; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low - [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, marainer, …) were not. @low From ccf4d20611e89712300224b0f82c75d5ffa1227c Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 16:57:00 +0200 Subject: [PATCH 052/102] feat(runnable): dispatch run() through the @entrypoint markers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run_entrypoint(runnable, task)` builds {declared task: method} from `runnable_entrypoints(type(runnable))`, calls the match, and raises ValueError listing the declared tasks in declaration order. A merged train+eval runnable used to state its task table twice — once in the decorators, once in a hand-written {task: method} dict — and the copies drift in the direction that bites: a config generator pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported. Nothing could test for it — the dict was derived from nothing. Consequence to accept: the markers are now load-bearing at runtime, so dropping an @entrypoint breaks the run instead of only emptying a picker. That is the intended direction — a silent discovery gap becomes a loud dispatch failure. The lookup reads markers off raw function objects (vars()), so a dynamic __torch_runner__ property never fires during dispatch. Docs: architecture record §7, the docs/runnable.md dispatch section, the AGENTS mandate, and the entrypoint docstring example (which taught the dict form). Pins: dispatch per task, declaration-order error, return pass-through, subclass override, the added-capability regression, and the property-getter guard. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- GEMINI.md | 2 +- README.md | 2 +- docs/architecture.md | 84 ++++++++++++++++++++++++++++++++++ docs/runnable.md | 25 ++++++++-- recordstream/__init__.py | 2 + recordstream/runnable.py | 40 ++++++++++++++-- tests/test_entrypoint.py | 99 +++++++++++++++++++++++++++++++++++++++- 9 files changed, 244 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cce0017..7e8f87e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__torch_runner__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. diff --git a/CLAUDE.md b/CLAUDE.md index cce0017..7e8f87e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__torch_runner__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. diff --git a/GEMINI.md b/GEMINI.md index cce0017..7e8f87e 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__torch_runner__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. diff --git a/README.md b/README.md index f69326c..03716a3 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit | [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap` | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | -| [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers with a worked example, `TorchRunner` / `ProgressReporting` | +| [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers + `run_entrypoint` dispatch with a worked example, `TorchRunner` / `ProgressReporting` | | [docs/workflow.md](docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | | [docs/augmentation.md](docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | | [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | diff --git a/docs/architecture.md b/docs/architecture.md index 562ef36..ac8bbd3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -569,3 +569,87 @@ accepts_broadcast(Enable, "enabled") # True — the bare --enabled form no - **More than one switch in a chain**: use several `Enable` wrappers with distinct names rather than teaching one wrapper several toggles — each name is independently addressable, and the broadcast form still flips them all. + +## 7. The `@entrypoint` markers ARE the dispatch table (`run_entrypoint`, 2026-07-29) + +### Context + +A merged train+eval runnable exposes several capabilities from ONE class and selects between them +with a single `task` knob. Two readers need to know the task→capability mapping: the runnable's own +`run()`, which must call the right method, and a discovery consumer (a config generator, a visual +editor), which must know that one class both trains and evaluates and which `task` value means +"evaluate". The `@entrypoint(task, role, primary)` marker was introduced for the second reader only; +`run()` carried its own copy: + +```python +dispatch = {"fit": self.fit, "evaluate": self.evaluate, "test": self.test, "predict": self.predict} +``` + +So every merged runnable stated the same mapping twice — once in the decorators, once in the dict — +and three consumer packages carried that same five-line block verbatim. The two copies drift in a +direction that bites: a config generator pins `task:` from `entrypoint_tasks` (the markers), so a +capability added to the markers and forgotten in the dict yields a *generated* config that dies at +dispatch with "unknown task" while discovery advertises it as supported. Nothing could catch that — +the dict is not derived from anything, so no test can compare it to a source of truth. + +### Decision + +The markers are the ONE table, and `run_entrypoint(runnable, task)` is their runtime half: it builds +`{declared task: method name}` from `runnable_entrypoints(type(runnable))`, calls the match, and +raises `ValueError` on an unknown task listing the declared ones in declaration order. A merged +runnable's `run()` is then `run_entrypoint(self, self.task)` — the decorators are the only place the +mapping exists. + +The lookup reads markers off raw function objects via `vars()` (as `runnable_entrypoints` already +did), so a dynamic `__torch_runner__` property never fires during dispatch. + +### Consequences + +- Adding a capability is ONE edit: decorate a method. Discovery and dispatch cannot disagree, + because they read the same annotations. +- The error message doubles as the class's capability list, in declaration order rather than the + sorted order a set would give. +- What is lost: the dict form let a type checker verify `self.fit` exists; `getattr(self, name)()` + is `Any`. Cheap here — the methods are decorated in the same file, and a wrong name would have to + survive its own `@entrypoint` line. +- Cost is one MRO walk per `run()` — once per training run. +- The markers are now load-bearing at RUNTIME, not just for discovery: dropping an `@entrypoint` + breaks the run, where before it only emptied a picker. That is the intended direction (a silent + discovery gap becomes a loud dispatch failure), but it means the decorators are no longer + optional metadata for a class that dispatches this way. + +### Example + +```python +from recordstream import TorchRunner, entrypoint, run_entrypoint + +class Classifier(TorchRunner): + def __init__(self, task: str = "fit") -> None: + self.task = task + + def run(self) -> None: + run_entrypoint(self, self.task) # no second copy of the mapping + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> None: ... + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: ... +``` + +```python +>>> Classifier(task="test").run() # calls Classifier.test() +>>> Classifier(task="export").run() +ValueError: Unknown task 'export'; expected one of ['fit', 'test']. +``` + +### What you may change (and where it's documented) + +- **Adding a capability**: decorate the method with `@entrypoint("", role=..., primary=...)` + and extend the runnable's own `task` Literal. Nothing else — usage lives in `docs/runnable.md`. +- **A capability that is NOT config-selectable**: leave it undecorated and call it directly; the + marker means "reachable through `task:`", so decorating a helper would advertise it to config + generators as a runnable capability. +- **A different dispatch policy** (aliases, a default task, a per-role default): build it on top of + `runnable_entrypoints` rather than beside it — the invariant to preserve is that the markers stay + the only place the mapping is written down. diff --git a/docs/runnable.md b/docs/runnable.md index 0448b1d..79706df 100644 --- a/docs/runnable.md +++ b/docs/runnable.md @@ -53,7 +53,7 @@ declares exactly that, per method. ## A straightforward example ```python -from recordstream import ProgressReporting, TorchRunner, entrypoint +from recordstream import ProgressReporting, TorchRunner, entrypoint, run_entrypoint class Classifier(TorchRunner, ProgressReporting): """One class, four capabilities — run() dispatches off the ``task`` knob.""" @@ -62,8 +62,7 @@ class Classifier(TorchRunner, ProgressReporting): self.task = task def run(self) -> None: - {"fit": self.fit, "evaluate": self.evaluate, - "test": self.test, "predict": self.predict}[self.task]() + run_entrypoint(self, self.task) # the markers below ARE the dispatch table @entrypoint("fit", role="trainer", primary=True) def fit(self) -> None: ... # gradient training @@ -107,6 +106,26 @@ Real output for the class above (these are executed facts, not sketches): `runnable_entrypoints` walks the MRO (an inherited entry point is found; a subclass override wins) and reads the marker off the raw function object, so property getters never fire. +## Dispatching: `run_entrypoint` + +`run()` above dispatches *through* the markers rather than restating them: + +```python +>>> Classifier(task="test").run() # calls Classifier.test() +>>> Classifier(task="export").run() +ValueError: Unknown task 'export'; expected one of ['fit', 'evaluate', 'test', 'predict']. +``` + +The declared tasks are listed in **declaration order**, so the error reads as the class's +capability list. The return value of the entry-point method is passed through. + +Write the `run()` body this way rather than as a hand-written `{task: method}` dict. The dict +states the same mapping a second time, and the copies drift in one direction that bites: a +config generator pins `task:` from `entrypoint_tasks` — the markers — so a capability added to +the markers and forgotten in the dict produces a *generated* config that dies at dispatch with +"unknown task" while discovery advertises it as supported. With `run_entrypoint` there is one +table, and adding a fifth `@entrypoint` method is all that adding a fifth capability takes. + ## How a consumer uses this A config generator asked for "an evaluator config for `Classifier`" calls diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 0b2d40e..3deb50b 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -53,6 +53,7 @@ TorchRunner, entrypoint, entrypoint_tasks, + run_entrypoint, runnable_entrypoints, ) from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName @@ -128,6 +129,7 @@ "ProgressCallback", "entrypoint", "entrypoint_tasks", + "run_entrypoint", "runnable_entrypoints", "DatasetProcessor", "Sequence", diff --git a/recordstream/runnable.py b/recordstream/runnable.py index 3aeee16..9efe8bd 100644 --- a/recordstream/runnable.py +++ b/recordstream/runnable.py @@ -28,11 +28,13 @@ method with the ``task`` value it runs and a ``role`` label (``"trainer"`` / ``"evaluator"`` / ``"predictor"``). A discovery consumer (a config generator, a visual editor) reads these via :func:`runnable_entrypoints` to learn that one class both -trains and evaluates, instead of assuming a separate class per role. Straightforward -worked example (the class + the exact introspector outputs): ``docs/runnable.md``. +trains and evaluates, instead of assuming a separate class per role — and the runnable's +own ``run()`` dispatches through :func:`run_entrypoint`, so the markers ARE the dispatch +table rather than a description of one kept in sync by hand. Straightforward worked +example (the class + the exact introspector outputs): ``docs/runnable.md``. """ -from typing import Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from loggair import get_logger @@ -112,8 +114,7 @@ def entrypoint(task: str, role: str = "runnable", primary: bool = False) -> Call class Classifier(TorchRunner, ProgressReporting): def run(self) -> None: - {"fit": self.fit, "evaluate": self.evaluate, - "test": self.test, "predict": self.predict}[self.task]() + run_entrypoint(self, self.task) # the markers below ARE the dispatch table @entrypoint("fit", role="trainer", primary=True) def fit(self) -> None: ... @@ -172,11 +173,40 @@ def entrypoint_tasks(cls: type, role: str) -> List[str]: return [str(meta["task"]) for _, meta in matches] +def run_entrypoint(runnable: object, task: str) -> Any: + """Call ``runnable``'s :func:`entrypoint` method whose declared task is ``task``. + + This is the RUNTIME half of the marker: a merged train+eval class's ``run()`` is + ``run_entrypoint(self, self.task)``, so the ``@entrypoint`` decorators ARE the dispatch + table instead of merely describing one. A hand-written ``{task: method}`` dict states the + same mapping a second time, and the two drift silently in one direction that matters — a + config generator pins ``task:`` from :func:`entrypoint_tasks` (the markers), so a + capability added to the markers but forgotten in the dict yields a generated config that + fails at dispatch with "unknown task" while discovery advertises it as supported. + + Unknown tasks raise :class:`ValueError` listing the declared ones in DECLARATION order + (``runnable_entrypoints`` preserves it), which reads as the class's capability list. + + Args: + runnable: The instance to dispatch on (its ``type()`` carries the markers). + task: The ``task`` value to run, matched against each entry point's declared task. + + Returns: + Whatever the entry-point method returns (``None`` for the merged runnables). + """ + by_task = {str(meta["task"]): name for name, meta in runnable_entrypoints(type(runnable)).items()} + name = by_task.get(task) + if name is None: + raise ValueError(f"Unknown task {task!r}; expected one of {list(by_task)}.") + return getattr(runnable, name)() + + __all__ = [ "ProgressCallback", "ProgressReporting", "TorchRunner", "entrypoint", "entrypoint_tasks", + "run_entrypoint", "runnable_entrypoints", ] diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py index ae89664..67bfe7e 100644 --- a/tests/test_entrypoint.py +++ b/tests/test_entrypoint.py @@ -1,6 +1,8 @@ -"""Tests for the runnable entry-point marker (entrypoint / runnable_entrypoints).""" +"""Tests for the runnable entry-point marker (entrypoint / runnable_entrypoints / run_entrypoint).""" -from recordstream.runnable import entrypoint, entrypoint_tasks, runnable_entrypoints +import pytest + +from recordstream.runnable import entrypoint, entrypoint_tasks, run_entrypoint, runnable_entrypoints class _Runnable: @@ -54,3 +56,96 @@ def fit(self) -> None: _R().fit() assert calls == ["fit"] + + +# ---- run_entrypoint: the markers ARE the dispatch table ---------------------- # + + +class _Dispatching: + """The merged train+eval shape: one ``task`` knob, ``run()`` dispatching off the markers.""" + + def __init__(self, task: str = "fit") -> None: + self.task = task + self.calls: list = [] + + def run(self) -> object: + return run_entrypoint(self, self.task) + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> str: + self.calls.append("fit") + return "fitted" + + @entrypoint("evaluate", role="evaluator") + def evaluate(self) -> None: + self.calls.append("evaluate") + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: + self.calls.append("test") + + @entrypoint("predict", role="predictor", primary=True) + def predict(self) -> None: + self.calls.append("predict") + + +@pytest.mark.parametrize("task", ["fit", "evaluate", "test", "predict"]) +def test_run_entrypoint_calls_the_method_declaring_the_task(task: str) -> None: + runnable = _Dispatching(task=task) + runnable.run() + assert runnable.calls == [task] + + +def test_run_entrypoint_returns_the_method_result() -> None: + assert _Dispatching(task="fit").run() == "fitted" + + +def test_run_entrypoint_rejects_an_unknown_task_listing_the_declared_ones() -> None: + runnable = _Dispatching(task="export") + with pytest.raises(ValueError) as excinfo: + runnable.run() + message = str(excinfo.value) + assert "Unknown task 'export'" in message + # Declaration order, so the list reads as the class's capability list. + assert "['fit', 'evaluate', 'test', 'predict']" in message + assert runnable.calls == [] + + +def test_a_new_entrypoint_dispatches_with_no_other_change() -> None: + """The regression a hand-written ``{task: method}`` dict allowed: marker added, dict forgotten.""" + + class _WithExport(_Dispatching): + @entrypoint("export", role="exporter", primary=True) + def export(self) -> None: + self.calls.append("export") + + # The config generator pins `task:` from the markers — dispatch must agree with it. + assert entrypoint_tasks(_WithExport, "exporter") == ["export"] + runnable = _WithExport(task="export") + runnable.run() + assert runnable.calls == ["export"] + + +def test_run_entrypoint_dispatches_to_a_subclass_override() -> None: + class _Override(_Dispatching): + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> str: + self.calls.append("fit-override") + return "overridden" + + runnable = _Override(task="fit") + assert runnable.run() == "overridden" + assert runnable.calls == ["fit-override"] + + +def test_run_entrypoint_never_fires_a_property_getter() -> None: + """The merged runnables carry a dynamic ``__torch_runner__`` property; lookup must not touch it.""" + + class _WithProperty(_Dispatching): + @property + def __torch_runner__(self) -> bool: + raise AssertionError("property getter fired during dispatch") + + runnable = _WithProperty(task="test") + runnable.run() + assert runnable.calls == ["test"] From 6484997bbb695406d89602638b2f6880d975c999 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 17:41:04 +0200 Subject: [PATCH 053/102] refactor(runnable)!: name the autograd flag for what it decides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TorchRunner` keeps its name — autograd is a torch concept, and a non-torch backend would not inherit the mixin at all — but the flag it sets is now `__needs_autograd__`, named for the decision its reader makes rather than for the class that declares it. `__torch_runner__` answered the wrong question at the one place it is read: every runnable in this workspace is a torch runnable, yet the flag is deliberately false for an evaluator, and the merged train+eval classes override it as a per-task property whose body — `return self.task == "fit"` — contradicted its own name (predicting with a torch model does not stop the object from being "a torch runner"). No alias. The flag is a duck-typed contract with exactly ONE reader (a GUI executor re-enabling autograd around run()), and that read fails OPEN: a reader left on the old name sees False for every runnable and silently executes training under inference_mode until loss.backward() raises. Declarer and reader move together. Rationale + the fail-open hazard: docs/architecture.md §8. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- GEMINI.md | 2 +- docs/architecture.md | 74 +++++++++++++++++++++++++++++++++++++++- docs/runnable.md | 7 ++-- recordstream/runnable.py | 17 ++++++--- recordstream/workflow.py | 2 +- tests/test_entrypoint.py | 4 +-- tests/test_runnable.py | 8 +++-- tests/test_workflow.py | 6 ++-- 10 files changed, 104 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7e8f87e..638c505 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__torch_runner__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. diff --git a/CLAUDE.md b/CLAUDE.md index 7e8f87e..638c505 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__torch_runner__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. diff --git a/GEMINI.md b/GEMINI.md index 7e8f87e..638c505 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__torch_runner__` / `set_progress_callback`), AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__torch_runner__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. diff --git a/docs/architecture.md b/docs/architecture.md index ac8bbd3..f339fca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -601,7 +601,7 @@ runnable's `run()` is then `run_entrypoint(self, self.task)` — the decorators mapping exists. The lookup reads markers off raw function objects via `vars()` (as `runnable_entrypoints` already -did), so a dynamic `__torch_runner__` property never fires during dispatch. +did), so a dynamic `__needs_autograd__` property never fires during dispatch. ### Consequences @@ -653,3 +653,75 @@ ValueError: Unknown task 'export'; expected one of ['fit', 'test']. - **A different dispatch policy** (aliases, a default task, a per-role default): build it on top of `runnable_entrypoints` rather than beside it — the invariant to preserve is that the markers stay the only place the mapping is written down. + +## 8. The autograd marker is named for the framework; its FLAG for what it decides (2026-07-29) + +### Context + +`TorchRunner` exists so a GUI executor — which evaluates graph nodes under +`torch.inference_mode()` for cheap, grad-free runs — can tell "this run does gradient descent" +from "this run is inference-only" and re-enable autograd around the former. The mixin set a flag +named after ITSELF, `__torch_runner__`, and that name answers a question nobody asks at the one +place it is read: + +```python +torch_runner = bool(getattr(runnable, "__torch_runner__", False)) # "is this a torch runner?" +``` + +Every runnable in this workspace is a torch runnable, so read literally the flag is always true — +yet it is deliberately false for an evaluator, and the merged train+eval runnables override it as +a per-task property whose body (`return self.task == "fit"`) contradicts its own name: predicting +with a torch model does not stop the object from being "a torch runner". The name described the +declaring class instead of the decision the reader makes with it. + +### Decision + +Keep the CLASS name (`TorchRunner` — autograd is a torch concept, and a non-torch backend would +not inherit this mixin at all), rename the FLAG to **`__needs_autograd__`**. The two names then +answer different questions on purpose: which framework's execution mode is at stake, and whether +this particular run needs gradients. + +No compatibility alias. The flag is a duck-typed contract with exactly one reader, so the rename +lands in both packages at once — consistent with the workspace's no-back-compat precedent. + +### Consequences + +- The dynamic per-task override reads as what it means, which is where the old name hurt most. +- **The read fails OPEN** (`getattr(runnable, "__needs_autograd__", False)`): a reader left on the + old name sees `False` for every runnable and silently executes training under `inference_mode` + until `loss.backward()` raises *"element 0 of tensors does not require grad"*. That is why the + rename is all-or-nothing across the reader and the declarer — never a partial rollout. +- An external duck-typed implementer (an object that sets the flag without inheriting the mixin) + must be updated by hand; there is no import to break and therefore no compile-time signal. + +### Example + +```python +class TorchRunner: + __needs_autograd__: bool = True # inherited by trainers and workflow combinators + + +class Classifier(TorchRunner, L.LightningModule): + @property + def __needs_autograd__(self) -> bool: # type: ignore[override] + """Only ``fit`` needs autograd; evaluate / test / predict are inference-only.""" + return self.task == "fit" +``` + +```python +# the executor side (one reader, no import of this package) +if getattr(runnable, "__needs_autograd__", False): + with torch.inference_mode(False), torch.enable_grad(): + runnable.run() +else: + runnable.run() +``` + +### What you may change (and where it's documented) + +- **A runnable that never trains**: do not inherit `TorchRunner` at all — the absent flag is the + statement. Usage lives in `docs/runnable.md`. +- **A runnable that sometimes trains**: override `__needs_autograd__` as a property, as above. +- **Another execution-mode marker** (a "needs a GPU", "must run single-process" flag): follow the + same rule — name the class for the concern, the flag for the decision the executor makes, and + remember that a duck-typed read of a missing flag is silent. diff --git a/docs/runnable.md b/docs/runnable.md index 79706df..cf419b2 100644 --- a/docs/runnable.md +++ b/docs/runnable.md @@ -138,11 +138,14 @@ walk over every discovered class tells a visual editor which classes to offer in Orthogonal to entry points, a runnable may inherit two stateless mixins: -- **`TorchRunner`** — declares "my `run()` needs autograd" (`__torch_runner__ = True`, +- **`TorchRunner`** — declares "my `run()` needs autograd" (`__needs_autograd__ = True`, duck-typed). A GUI executor that evaluates nodes under `torch.inference_mode()` re-enables autograd for the duration of `run()`. Inference-only runnables deliberately do NOT inherit it. A merged class can even make it dynamic — a property returning `self.task == "fit"`, - so the same class trains under autograd and predicts under inference mode. + so the same class trains under autograd and predicts under inference mode. The class and + the flag are named for different things on purpose: the *class* for the framework whose + execution mode is at stake (autograd is a torch concept), the *flag* for what it decides — + which is what makes the dynamic property above read correctly. - **`ProgressReporting`** — a framework-free progress sink: the executor injects `(value, total, desc) -> None` via `set_progress_callback()`, the runnable drains it via `self._report_progress(step, total, "epoch 3")` from its loop. With no sink injected diff --git a/recordstream/runnable.py b/recordstream/runnable.py index 9efe8bd..6d7b47f 100644 --- a/recordstream/runnable.py +++ b/recordstream/runnable.py @@ -6,9 +6,10 @@ node) can cooperate with it WITHOUT this package importing the GUI framework: * :class:`TorchRunner` — marks a runnable whose ``run()`` needs autograd (it - performs gradient-based optimization). A GUI executor that evaluates nodes under - ``torch.inference_mode()`` reads the duck-typed ``__torch_runner__`` flag and - re-enables autograd for the duration of ``run()``. + performs gradient-based optimization). The class is named for the framework it + concerns; the flag it sets says what it MEANS — a GUI executor that evaluates nodes + under ``torch.inference_mode()`` reads the duck-typed ``__needs_autograd__`` flag + and re-enables autograd for the duration of ``run()``. * :class:`ProgressReporting` — gives a runnable a framework-free progress callback. The executor injects a ``(value, total, desc) -> None`` sink via :meth:`~ProgressReporting.set_progress_callback`; the runnable drains it from its @@ -60,14 +61,20 @@ class TorchRunner: inference mode every tensor created — model parameters, forward activations, the loss — is an inference tensor with no autograd graph, so ``loss.backward()`` dies with *"element 0 of tensors does not require grad and does not have a grad_fn"*. - The executor reads the duck-typed ``__torch_runner__`` flag (no hard import on the + The executor reads the duck-typed ``__needs_autograd__`` flag (no hard import on the GUI side) and re-enables normal autograd for the duration of ``run()``. + Note the deliberate split between the two names: the CLASS is named for the framework + whose execution mode is at stake (autograd is a torch concept, and a non-torch backend + would not inherit this mixin at all), while the FLAG is named for what it decides — + "this run needs autograd". A runnable that only sometimes trains overrides the flag as + a property (``return self.task == "fit"``), which reads correctly only under that name. + Inference-only runnables (a pure evaluator, a dataset processor) deliberately do NOT inherit this — they run as-is under the executor's inference mode. """ - __torch_runner__: bool = True + __needs_autograd__: bool = True class ProgressReporting: diff --git a/recordstream/workflow.py b/recordstream/workflow.py index 3ca42b3..9e5290f 100644 --- a/recordstream/workflow.py +++ b/recordstream/workflow.py @@ -43,7 +43,7 @@ The combinators inherit :class:`~recordstream.runnable.TorchRunner` and :class:`~recordstream.runnable.ProgressReporting` so a workflow runs correctly on a -GUI canvas: a combinator may wrap a *trainer*, so it declares ``__torch_runner__`` +GUI canvas: a combinator may wrap a *trainer*, so it declares ``__needs_autograd__`` (the executor re-enables autograd for the whole run — otherwise an inner ``loss.backward()`` dies under the executor's inference mode; restoring autograd is harmless for an inner evaluator), and it FORWARDS the executor-injected progress diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py index 67bfe7e..7f46a31 100644 --- a/tests/test_entrypoint.py +++ b/tests/test_entrypoint.py @@ -139,11 +139,11 @@ def fit(self) -> str: def test_run_entrypoint_never_fires_a_property_getter() -> None: - """The merged runnables carry a dynamic ``__torch_runner__`` property; lookup must not touch it.""" + """The merged runnables carry a dynamic ``__needs_autograd__`` property; lookup must not touch it.""" class _WithProperty(_Dispatching): @property - def __torch_runner__(self) -> bool: + def __needs_autograd__(self) -> bool: raise AssertionError("property getter fired during dispatch") runnable = _WithProperty(task="test") diff --git a/tests/test_runnable.py b/tests/test_runnable.py index 48d380e..9bcb469 100644 --- a/tests/test_runnable.py +++ b/tests/test_runnable.py @@ -5,13 +5,15 @@ from recordstream.runnable import ProgressReporting, TorchRunner -def test_torch_runner_flag() -> None: - assert TorchRunner.__torch_runner__ is True +def test_torch_runner_sets_the_needs_autograd_flag() -> None: + # The class is named for the framework; the FLAG is named for what it decides — a GUI + # executor reads `__needs_autograd__` (duck-typed) to re-enable autograd around run(). + assert TorchRunner.__needs_autograd__ is True class Trainer(TorchRunner): pass - assert Trainer().__torch_runner__ is True # inherited by subclasses + assert Trainer().__needs_autograd__ is True # inherited by subclasses def test_progress_reporting_noop_without_callback() -> None: diff --git a/tests/test_workflow.py b/tests/test_workflow.py index d5a7b84..7a5504f 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -193,11 +193,11 @@ def test_predicate_zero_arg_construct_and_call(cls: Any) -> None: # StreamStudio canvas integration — TorchRunner + ProgressReporting forwarding # --------------------------------------------------------------------------- # @pytest.mark.parametrize("cls", [Sequence, Conditional, Switch]) -def test_combinators_declare_torch_runner(cls: Any) -> None: - # A combinator may wrap a trainer, so it declares __torch_runner__ — StreamStudio's executor +def test_combinators_declare_needs_autograd(cls: Any) -> None: + # A combinator may wrap a trainer, so it declares __needs_autograd__ — StreamStudio's executor # re-enables autograd for the whole run (otherwise the inner loss.backward() dies under # ComfyUI's inference_mode). - assert cls().__torch_runner__ is True + assert cls().__needs_autograd__ is True def test_progress_callback_forwarded_to_running_branch() -> None: From a64bd6a36a2d6d63bf7fdb90c96bcedabad2fadf Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 19:40:40 +0200 Subject: [PATCH 054/102] =?UTF-8?q?feat:=20own=20the=20model=20boundary=20?= =?UTF-8?q?=E2=80=94=20prediction=20contracts,=20sinks,=20dataset=20normal?= =?UTF-8?q?ization,=20label=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four surfaces moved in from the workspace's experiment-tracking library, under one rule: a package must not own a contract whose only reader lives elsewhere. * `ensure_record_dataset` / `RecordSource` (core.py) — normalize a wired dataset slot into a map-style Dataset of records. Its whole body is "already a Stream? else wrap in one", so it belongs beside Stream; a Stream returns AS-IS because identity matters (a label-encoding Stream carries its label_names). * `recordstream.outputs` — ClassificationOutput / DetectionOutput / SegmentationOutput (generic TypedDicts, so a non-torch backend declares the SAME contract) plus the torch builders. Detection deliberately has no builder: its boxes come from the detector. * `recordstream.predictions` — the PredictionsSink protocol + ClassificationPredictionsSink, which reads `probs`/`class_idx` by name. The contract now lives beside its reader instead of justifying itself circularly ("the sink is there because the type is there"). * `class_counts` / `inverse_frequency_weights` (labels.py) — how often each class occurs is a statistic over the LABELS, not a property of a loss. They take already-walked targets (a trainer walks once and reuses that pass three ways; a source-walking convenience would silently double it), accept every target shape via LabelMap.to_ids, and return NUMPY — the same rule as recordstream.batch, so a torch caller writes torch.as_tensor(w) and a Keras one feeds fit(class_weight=...). What deliberately did NOT come along: whether a loss accepts `weight` and how to inject it. That is torch.nn's constructor convention, so it stays a per-backend method on the consuming runnable — this package never learns what a loss is. Two defects fixed in transit: the sink advertised itself as modality-neutral while building diagnostics from pack_id/iq_file/window_start_sample (records are now identified by ordinal), and it took `ops` as a REQUIRED constructor argument, which the package-wide zero-arg-construction sweep rejected on arrival — the check moved to write(), where it belongs. Two sink protocols now coexist (DataSink.write(record) vs PredictionsSink.write(prediction, metadata)); the split is real and load-bearing downstream, and collapsing it is filed in TASKS.md rather than left to drift. Docs: architecture record §8, docs/predictions.md, a docs/projection.md section, three AGENTS mandates. No back-compat aliases — a stale import fails loudly. --- AGENTS.md | 3 + CLAUDE.md | 3 + GEMINI.md | 3 + README.md | 3 +- TASKS.md | 3 +- docs/architecture.md | 90 ++++++++++++- docs/predictions.md | 96 ++++++++++++++ docs/projection.md | 32 +++++ recordstream/__init__.py | 35 +++++- recordstream/core.py | 35 ++++++ recordstream/labels.py | 83 +++++++++++- recordstream/outputs.py | 117 +++++++++++++++++ recordstream/predictions.py | 206 ++++++++++++++++++++++++++++++ tests/test_labels.py | 71 ++++++++++- tests/test_outputs.py | 111 ++++++++++++++++ tests/test_predictions.py | 245 ++++++++++++++++++++++++++++++++++++ tests/test_record_source.py | 81 ++++++++++++ 17 files changed, 1210 insertions(+), 7 deletions(-) create mode 100644 docs/predictions.md create mode 100644 recordstream/outputs.py create mode 100644 recordstream/predictions.py create mode 100644 tests/test_outputs.py create mode 100644 tests/test_predictions.py create mode 100644 tests/test_record_source.py diff --git a/AGENTS.md b/AGENTS.md index 638c505..52f3f49 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,9 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `label_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. +- **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/CLAUDE.md b/CLAUDE.md index 638c505..52f3f49 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,6 +40,9 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `label_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. +- **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/GEMINI.md b/GEMINI.md index 638c505..52f3f49 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -40,6 +40,9 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `label_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. +- **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/README.md b/README.md index 03716a3..9ca548a 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,8 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Stream.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | -| [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap` | +| [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap`, class-balance weights | +| [docs/predictions.md](docs/predictions.md) | The model boundary: prediction-output contracts (`ClassificationOutput` & co), `ensure_record_dataset`, the `PredictionsSink` protocol + the classification sink | | [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | | [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers + `run_entrypoint` dispatch with a worked example, `TorchRunner` / `ProgressReporting` | diff --git a/TASKS.md b/TASKS.md index 8c03429..127ea61 100644 --- a/TASKS.md +++ b/TASKS.md @@ -3,12 +3,13 @@ Open work for this project. Cross-cutting / multi-project initiatives live in the workspace root `TASKS.md`. Completed items are not archived here — git history is the record. -- [ ] **`Stream.source`'s annotation is narrower than its documented contract** @low @refactor — the docstring says "any iterable **or indexable** dataset (duck-typed)" but the annotation is `Optional[Iterable[Any]]`. A torch map-style `Dataset` iterates at runtime via the legacy `__getitem__` protocol, which mypy does not model, so passing one is statically invalid though perfectly correct — `marainer.torch.ensure_record_dataset` hits exactly this and carries a documented `cast`. Attempted 2026-07-29: widening to `Union[Iterable[Any], Indexable]` with a `Protocol` BREAKS `confluid.to_pydantic` for every Stream (`SchemaError: Error building "model" validator ... Field "source"` — pydantic cannot schema a bare Protocol), so it needs either a pydantic-friendly spelling or an entry in confluid's opaque-type coercion (`_is_opaque_type` -> `Any`, the same escape hatch the torchvision `Callable`/enum landmines use). Not worth a schema regression for a type nicety; revisit if a second consumer hits the cast. +- [ ] **`Stream.source`'s annotation is narrower than its documented contract** @low @refactor — the docstring says "any iterable **or indexable** dataset (duck-typed)" but the annotation is `Optional[Iterable[Any]]`. A torch map-style `Dataset` iterates at runtime via the legacy `__getitem__` protocol, which mypy does not model, so passing one is statically invalid though perfectly correct — `recordstream.ensure_record_dataset` (moved in from a consumer 2026-07-29) hits exactly this and carries a documented `cast`. Attempted 2026-07-29: widening to `Union[Iterable[Any], Indexable]` with a `Protocol` BREAKS `confluid.to_pydantic` for every Stream (`SchemaError: Error building "model" validator ... Field "source"` — pydantic cannot schema a bare Protocol), so it needs either a pydantic-friendly spelling or an entry in confluid's opaque-type coercion (`_is_opaque_type` -> `Any`, the same escape hatch the torchvision `Callable`/enum landmines use). Not worth a schema regression for a type nicety; revisit if a second consumer hits the cast. - [ ] **`to_tensor`'s `normalize` heuristic silently corrupts already-standardized floats** @bug — `recordstream/ops/torch.py::to_tensor` does `elif normalize and tensor.max() > 1.0: tensor = tensor / 255.0`, i.e. it infers "a float whose max exceeds 1 must be 0-255 pixels". An ImageNet-standardized array (range ~[-2.12, 2.64]) satisfies that test, so chaining a `Normalize` op before `ToTensor` divides the standardized values by 255 and squashes them to ~[-0.01, 0.01] — no error, no warning, just a model that learns nothing. Hit for real 2026-07-29 while wiring a consumer's example config; the caller's fix is `ToTensor(normalize=false)`, which is correct but only discoverable by inspecting the tensor. Options: gate the rescale on an INTEGER dtype only (what the docstring already claims — "scale integer pixel inputs"), or keep the heuristic and warn when it fires on a float input. Changing it is a behaviour change for anyone relying on the 0-255-float path, so it needs a decision rather than a quiet edit. - [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor - [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `recordstream.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in recordstream; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low - [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, marainer, …) were not. @low - [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in recordstream, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor +- [ ] **Decide whether `PredictionsSink` collapses into `DataSink`** @medium @refactor — since 2026-07-29 this package carries TWO sink protocols: `storage.base.DataSink.write(record)` (adapted into op chains by `RecordSinkOp`) and `predictions.PredictionsSink.write(prediction, metadata)`. The split is real — a model emits a BATCH while the sink contract is per-record, so the prediction and its record's metadata arrive separately — and it is load-bearing downstream, where a visual editor's node palette keys off the signature difference. The alternative: have the consuming runnable build the record (it already holds both halves — it slices the batch per record before calling `write`) and write through the ordinary `DataSink`, deleting `PredictionsSink` and letting prediction sinks become ordinary `category="sink"` canvas nodes. That touches the predict path of every consuming runnable, which is why it was NOT bundled into the move. Decide deliberately; do not let the two protocols blur by drift. Rationale for the current state: `docs/architecture.md` §8. - [ ] GPU-aware batch processing engine @performance - [ ] S3 storage backend support @feature - [ ] **RecordStream Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance diff --git a/docs/architecture.md b/docs/architecture.md index f339fca..135b401 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -25,7 +25,8 @@ Maintenance rules: | Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17) | | Storage & query | `storage/*` | The `typedrecord-v1` key-group layout over the codec; metadata scans without array loads | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) (contracts) + [storage.md](storage.md) | | Introspection & serialization | `discovery.py` | Callable↔string identity + registration-free module scans | [§4](#4-callablestring-serialization--passive-introspection-recordstreamdiscovery-2026-07-20) | -| Runnables & workflows | `runnable.py`, `workflow.py`, `processing.py`, `cli.py` | `run()` objects, entry-point markers, combinators, the one `recordstream run` runner | no record yet — [runnable.md](runnable.md), [workflow.md](workflow.md) | +| Runnables & workflows | `runnable.py`, `workflow.py`, `processing.py`, `cli.py` | `run()` objects, entry-point markers, combinators, the one `recordstream run` runner | [§7](#7-the-entrypoint-markers-are-the-dispatch-table-run_entrypoint-2026-07-29) + [runnable.md](runnable.md), [workflow.md](workflow.md) | +| Model boundary | `outputs.py`, `predictions.py`, `core.ensure_record_dataset`, `labels.class_counts` | Dataset normalization in, prediction contracts + sinks out, class-balance statistics | [§8](#8-the-model-boundary-belongs-to-the-package-that-reads-it-2026-07-29) + [predictions.md](predictions.md) | --- @@ -725,3 +726,90 @@ else: - **Another execution-mode marker** (a "needs a GPU", "must run single-process" flag): follow the same rule — name the class for the concern, the flag for the decision the executor makes, and remember that a duck-typed read of a missing flag is silent. + +## 8. The model boundary belongs to the package that reads it (2026-07-29) + +### Context + +Four surfaces used to live in the workspace's experiment-**tracking** library: a dataset +normalizer (`ensure_record_dataset`), the prediction-output contracts (`ClassificationOutput` & +co) with their torch builders, a predictions sink (`PredictionsSink` + +`ClassificationPredictionsSink`), and class-imbalance weighting (`apply_class_weights` and its +inverse-frequency arithmetic). + +None of them tracked anything. The normalizer's whole body was "already a `Stream`? else wrap in +one". The sink's own module docstring justified its placement circularly — it lived there +*because the contract and the output type lived there* — while its body was record plumbing plus a +numpy `argsort`, threading its result through recordstream ops. And the sink advertised itself as +modality-neutral while building its diagnostics from `pack_id` / `iq_file` / +`window_start_sample`: signal-domain keys, in a class a tabular classifier was supposed to reuse. + +The pattern underneath: each of these describes a boundary whose only READER is elsewhere, and a +contract that outlives its reader accumulates justifications instead of users. + +### Decision + +A package owns a contract when it owns the reader. So: + +- `ensure_record_dataset` / `RecordSource` land beside `Stream` — the only type they know. +- `recordstream.outputs` holds the contracts *and* their torch builders, because + `recordstream.predictions` — the sink that reads `probs` by name — is right next to it. +- `recordstream.predictions` holds the sink and the `PredictionsSink` protocol. +- `class_counts` / `inverse_frequency_weights` land beside `LabelMap`, because how often each + class occurs is a statistic over the labels. + +The line is drawn at the *framework convention*, not at "does this import torch" (this package +already hard-depends on torch — a `Stream` IS a `torch.utils.data.Dataset`). What did NOT move: +whether a loss accepts a `weight` argument and how to inject it. That is `torch.nn`'s constructor +convention — Keras takes `class_weight` on `fit()` — so it lives in the consuming runnable as an +overridable method, and this package never learns what a loss is. + +### Consequences + +- The weights come back as **numpy**, matching `recordstream.batch` (only `batch_tensor` is + torch). A torch caller writes `torch.as_tensor(w)`; a Keras backend feeds the same array to + `fit(class_weight=…)`. One statistic, no framework baked in. +- `inverse_frequency_weights` absorbed the `LabelMap.to_ids` flattening consumers used to write by + hand, so a multi-label target counts for every class it names with no call-site branch. +- The engine's own rules bit immediately and usefully: the package-wide zero-arg-construction + sweep failed on the imported sink (`ops` was a required constructor argument), so the check + moved to `write()` where it belongs. Stricter host, better tenant. +- Two sink protocols now coexist (`DataSink.write(record)` vs + `PredictionsSink.write(prediction, metadata)`). That is deliberate — a model emits a batch while + the sink contract is per-record, so the halves arrive separately — and load-bearing downstream, + where a visual editor's node palette keys off the distinction. Collapsing them is filed in + `TASKS.md` rather than left to drift. +- No back-compat aliases: a stale import from the old location fails loudly. + +### Example + +```python +# a trainer, walking its targets exactly once and reusing that pass three ways +targets = self._walk_targets(self.train_set) # ONE pass +self.label_map = LabelMap.fit(targets) # (1) the encoding +num_classes = self.label_map.num_classes # (2) the head size +weights = inverse_frequency_weights(targets, num_classes, self.label_map) # (3) the balance + +if weights is not None: + self.apply_class_weights(weights) # framework hook — torch: loss.weight = ... +``` + +```python +# the boundary on the way out +def predict_step(self, batch, batch_idx): + out = classification_output(self(x)) # recordstream.outputs + self.predictions_sink.write(out, metadata) # recordstream.predictions + return out +``` + +### What you may change (and where it's documented) + +- **A new prediction contract**: add it to `recordstream.outputs`, generic in the array type, and + give it a builder only if the payload is DERIVED (logits → probs). A payload the model hands you + directly (boxes) gets no builder. Usage lives in `docs/predictions.md`. +- **Another task's predictions sink**: implement `PredictionsSink` beside the classification one, + or in the domain package when it needs domain geometry (a detector's back-projection to + time/frequency does). +- **A different balancing policy** (effective-number, sqrt-inverse): add it beside + `inverse_frequency_weights` in `recordstream.labels` as another statistic returning numpy. Do + NOT add the injection here — that stays a per-backend method on the runnable. diff --git a/docs/predictions.md b/docs/predictions.md new file mode 100644 index 0000000..e695cb6 --- /dev/null +++ b/docs/predictions.md @@ -0,0 +1,96 @@ +# The model boundary: datasets in, predictions out + +RecordStream owns the data on both sides of a model: the dataset a trainer consumes, and the +prediction it emits. This page covers the three surfaces that sit on that boundary. + +## `ensure_record_dataset` — normalize a wired dataset slot + +A config can wire a dataset slot to a `Stream`, another torch `Dataset`, a bare source, or a plain +list of records. Normalize once, and the rest of the pipeline (target detection, label encoding, +collate, metrics) can assume record items unconditionally: + +```python +from recordstream import RecordSource, ensure_record_dataset + +dataset = ensure_record_dataset(self.train_set) # -> a map-style Dataset of records +``` + +A `Stream` comes back **as-is** — identity matters, because a label-encoding Stream carries its +`label_names` and re-wrapping would lose it. Anything else is wrapped in a `Stream`, which makes it +both map-style and record-yielding. + +`RecordSource` is the contract it enforces (`Dataset | Iterable[Record]`), named once so a consumer +annotates `Optional[Lazy[RecordSource]]` instead of inventing its own union. + +## Prediction-output contracts (`recordstream.outputs`) + +What a model's eval-mode `forward` returns, declared as typed dicts so metrics, sinks and +visualizers read known keys instead of guessing whether a tensor is logits, probabilities, or +argmax'd class ids: + +| Contract | Keys | +|---|---| +| `ClassificationOutput` | `logits` `[B, C]`, `probs` `[B, C]`, `class_idx` `[B]` | +| `DetectionOutput` | `boxes` `[N, 4]` xyxy absolute pixels, `scores` `[N]`, `labels` `[N]` | +| `SegmentationOutput` | `logits` `[B, C, H, W]`, `probs` `[B, C, H, W]`, `mask` `[B, H, W]` | + +That guess is not hypothetical: two independently-written detector wrappers agree that `boxes` is +xyxy in absolute pixels only because `DetectionOutput` says so. + +Each contract is **generic in the array type**, so the same declaration describes a torch run and a +numpy/TF/JAX one: + +```python +from recordstream import ClassificationOutput + +def predict(self, x) -> ClassificationOutput[np.ndarray]: ... # a non-torch backend +``` + +A bare `ClassificationOutput` means "whatever array type". The **builders** are the only torch part +— `softmax` and `argmax` are library calls, not type declarations: + +```python +from recordstream import classification_output + +def predict_step(self, batch, batch_idx): + return classification_output(self(x)) # logits -> the full contract +``` + +Detection deliberately has no builder: its boxes come from the detector's own interface, so the +dict is built inline at the call site rather than invented from logits. + +## Predictions sinks (`recordstream.predictions`) + +A predict/test loop calls `predictions_sink.write(prediction, metadata)` once per record and +`close()` at the end. `PredictionsSink` is that contract as a `@runtime_checkable` Protocol — +annotate the slot with it rather than `Any`, so a use site can't call `.write` on something that is +still a deferred marker. + +`ClassificationPredictionsSink` is the classification implementation: read `probs` / `class_idx`, +resolve the class id to a label, build a top-k list, and thread a record through your ops. + +```yaml +predictions_sink: !class:recordstream.predictions.ClassificationPredictionsSink + label_names: !ref:class_id_to_label # {0: "bird", 1: "cat", ...} + top_k: 5 + confidence_threshold: 0.0 # skip predictions below this top-1 probability + ops: + - !class:recordstream.ops.sink.RecordSinkOp + sink: !class:mypkg.JsonSink() { path: ./predictions } +``` + +Each written record carries the original metadata plus `predicted_class_id`, +`predicted_class_label`, `predicted_confidence` and `predicted_top_k`, under a single `"metadata"` +key. Diagnostics identify a record by its **ordinal** in the run — the sink is modality-neutral, so +it never assumes a metadata key exists. + +Like every configurable here it is **zero-arg constructible**: `ops` is required to *run*, not to +*build*, so the non-empty check fires on the first `write` rather than in `__init__`. + +### Why this is a second sink protocol + +`recordstream.storage.base.DataSink` takes a whole record (`write(record)`) and is what +`RecordSinkOp` adapts into an op chain. A predictions sink instead receives the model's output plus +the metadata of the record it came from, and builds the record itself — the two halves arrive +separately because a model emits a batch while the sink contract is per-record. Keep the two +distinct; collapsing them is tracked as a deliberate decision, not something to do by drift. diff --git a/docs/projection.md b/docs/projection.md index c1eaca6..0ad3a93 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -70,3 +70,35 @@ lm.to_ids("dog") # [2] a bare value works too Because encoded ids pass through untouched, `LabelMap().to_ids(...)` (an *empty* map) is a valid way to normalize an already-encoded dataset to id lists — useful for counting classes or class-frequency statistics without fitting anything. + +## Class-balance weights (`class_counts` / `inverse_frequency_weights`) + +Training on a skewed label distribution biases a model toward the majority class. The remedy is +per-class weights, and *deriving* them is a statistic over the labels — so it lives here, beside +the `LabelMap` that normalizes the targets: + +```python +from recordstream import LabelMap, inverse_frequency_weights, iter_key + +targets = list(iter_key(train_source, "class")) # walk ONCE — see the note below +weights = inverse_frequency_weights(targets, num_classes=3, label_map=lm) +# array([0.667, 2.0, 1.0], dtype=float32) rare classes weigh more +``` + +`w[c] = total / (num_classes * count[c])`, so a class at exactly the mean frequency gets `1.0`. A +class observed **zero** times gets `0.0` (never infinity), an id outside `[0, num_classes)` is +ignored rather than raising, and the whole call returns `None` when nothing was counted — so "no +weights" stays distinguishable from "all-zero weights". `class_counts(...)` exposes the raw +histogram if you want it. + +Three things the signature is deliberate about: + +- **It takes already-walked targets, not a source.** A trainer typically walks the target stream + once and reuses that single pass to fit the `LabelMap`, derive `num_classes`, *and* weigh the + classes. A convenience that walked internally would quietly double the passes over the dataset. +- **Every target shape works**, because `LabelMap.to_ids` normalizes it: a `Label`, a `MultiLabel` + (counting for every class it names), a bare id with no map at all, a name with a fitted map. +- **It returns numpy**, like everything in `recordstream.batch` except `batch_tensor`. What a + framework does with the vector is its own convention — `torch.nn` takes a `weight` tensor in the + loss constructor, Keras takes `class_weight` on `fit()` — so that last step belongs to the + consuming trainer, not here. diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 3deb50b..6f3a76a 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -13,7 +13,16 @@ from recordstream.batch import batch_metadata, batch_tensor, batch_values, multi_hot from recordstream.collate import collate, collate_records, get_collate, register_collate, registered_collates from recordstream.context import Context -from recordstream.core import FilterOp, JointStream, Stream, WrappedOp, register_op_family, registered_op_families +from recordstream.core import ( + FilterOp, + JointStream, + RecordSource, + Stream, + WrappedOp, + ensure_record_dataset, + register_op_family, + registered_op_families, +) # --- the record data model + transforms + item codec ---------------------------------------- from recordstream.dispatch import dispatch, register_kernel, registered_kernels @@ -44,7 +53,16 @@ register_item, with_data, ) -from recordstream.labels import LabelMap +from recordstream.labels import LabelMap, class_counts, inverse_frequency_weights +from recordstream.outputs import ( + ClassificationOutput, + DetectionOutput, + DetectionPredictions, + SegmentationOutput, + classification_output, + segmentation_output, +) +from recordstream.predictions import ClassificationPredictionsSink, PredictionsSink from recordstream.processing import DatasetProcessor from recordstream.projection import SupportsProjection, iter_key, num_classes, project from recordstream.runnable import ( @@ -95,6 +113,8 @@ "Context", "Stream", "JointStream", + "RecordSource", + "ensure_record_dataset", "FilterOp", "WrappedOp", "register_op_family", @@ -112,6 +132,17 @@ "register_collate", "registered_collates", "LabelMap", + "class_counts", + "inverse_frequency_weights", + # ---- prediction contracts + sinks ---- + "ClassificationOutput", + "DetectionOutput", + "DetectionPredictions", + "SegmentationOutput", + "classification_output", + "segmentation_output", + "PredictionsSink", + "ClassificationPredictionsSink", # ---- sources ---- "HuggingFaceSource", "DatasetSplit", diff --git a/recordstream/core.py b/recordstream/core.py index 98364e4..a888949 100644 --- a/recordstream/core.py +++ b/recordstream/core.py @@ -690,3 +690,38 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: want = set(keys) for record in self: yield {k: v for k, v in record.items() if k in want} + + +#: What a wired dataset slot may hold — the contract :func:`ensure_record_dataset` enforces, +#: named ONCE here rather than restated by every consumer: a map-style ``Dataset`` (which a +#: :class:`Stream` is), or any iterable of records (a recordstream source, a plain list of +#: record dicts). Consumers annotate their slots ``Optional[Lazy[RecordSource]]`` — ``Lazy`` +#: because they flow the slot themselves at run time. +RecordSource = Union[torch.utils.data.Dataset[Any], Iterable[Record]] + + +def ensure_record_dataset(source: RecordSource) -> torch.utils.data.Dataset[Any]: + """Normalize any wired source into a map-style ``Dataset`` that yields record dicts. + + A wired ``train_set`` / ``val_set`` / ``test_set`` may be a :class:`Stream`, another torch + ``Dataset``, a recordstream source (``HuggingFaceSource``), or a plain list — and its items + may be record dicts or raw rows. A ``Stream`` already coerces every item to a record, so: + + * a ``Stream`` is returned as-is (already a ``Dataset`` of records; this preserves a + subclass's own wrap, e.g. a label-encoding Stream with its ``label_names``), and + * anything else is wrapped in a ``Stream``, which makes it both a map-style ``Dataset`` + AND a record-yielding one. + + Calling this once up front lets the rest of a training pipeline (target detection, label + fitting/encoding, collate, metrics) assume record items — no per-call "is this a record?" + checks. It lives beside :class:`Stream` because that is the only type it knows: the whole + body is "already a Stream? else wrap in one". + """ + if isinstance(source, Stream): + return source + # `cast`: a map-style `Dataset` iterates through Python's legacy `__getitem__` protocol, + # which mypy does not model — so it is not `Iterable` statically even though `Stream` + # consumes it correctly at runtime (`Stream.source` accepts "any iterable or indexable + # dataset"). Widening that annotation with a Protocol breaks `to_pydantic` for every + # Stream, so the exception is documented here instead. See TASKS.md. + return Stream(source=cast(Iterable[Any], source)) diff --git a/recordstream/labels.py b/recordstream/labels.py index b23eafe..723722c 100644 --- a/recordstream/labels.py +++ b/recordstream/labels.py @@ -27,6 +27,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Sequence, Union +import numpy as np from confluid import configurable from recordstream.items import Label, MultiLabel, is_class_id @@ -241,4 +242,84 @@ def load(cls, path: Union[str, Path]) -> "LabelMap": return cls.from_label_names([str(n) for n in names]) -__all__ = ["LabelMap"] +def class_counts(targets: Iterable[Any], num_classes: int, label_map: Optional[LabelMap] = None) -> np.ndarray: + """How often each class id occurs in ``targets`` — the label statistic behind class balancing. + + Every target shape is accepted, because :meth:`LabelMap.to_ids` normalizes them: a + :class:`~recordstream.Label` or :class:`~recordstream.MultiLabel` item, a bare name/id, or a + sequence. A multi-label target counts for EVERY class it names. ``None`` targets are skipped. + + Args: + targets: Already-walked target values (see the note on walking below). + num_classes: Width of the returned vector. Ids outside ``[0, num_classes)`` are IGNORED + rather than raising — a stray label must not abort a training run. + label_map: Map for class-NAME targets. Omit for integer targets: an empty + :class:`LabelMap` passes already-encoded ids through, which is the ``to_ids`` contract. + + Returns: + A ``float64`` vector of length ``num_classes``. + + Note: + This takes ALREADY-WALKED targets, not a source, on purpose. A caller typically walks the + target stream once (``iter_key(source, key)``) and reuses that single pass for several + answers — fitting a :class:`LabelMap`, deriving the class count, and weighting — and a + convenience that walked internally would silently double the passes over the dataset. + + Example:: + + class_counts([Label("cat"), Label("dog"), Label("cat")], 2, LabelMap({"cat": 0, "dog": 1})) + # array([2., 1.]) + """ + mapper = label_map if label_map is not None else LabelMap() + counts = np.zeros(int(num_classes), dtype=np.float64) + for target in targets: + if target is None: + continue + for class_id in mapper.to_ids(target): + if 0 <= class_id < num_classes: + counts[class_id] += 1.0 + return counts + + +def inverse_frequency_weights( + targets: Iterable[Any], num_classes: int, label_map: Optional[LabelMap] = None +) -> Optional[np.ndarray]: + """Per-class weights inversely proportional to observed frequency. + + ``w[c] = total / (num_classes * count[c])`` — a class at exactly the mean frequency gets + ``1.0``, rarer classes more, commoner classes less. Training on a skewed label distribution + biases a model toward the majority class; feeding these weights to a loss (or a framework's + ``class_weight`` knob) is the standard remedy. + + This is a statistic OVER THE DATA, which is why it lives here rather than beside a loss: what + a consuming framework then does with the vector — ``torch.nn``'s ``weight=`` constructor + argument, Keras's ``class_weight`` on ``fit()`` — is that framework's convention, and the + numbers are the same either way. Hence the **numpy** return (the same rule as + :mod:`recordstream.batch`: only ``batch_tensor`` is torch); a torch caller writes + ``torch.as_tensor(weights)``. + + Args: + targets: Already-walked target values (see :func:`class_counts` on why not a source). + num_classes: Width of the returned vector. + label_map: Map for class-NAME targets; omit for integer targets. + + Returns: + A ``float32`` vector of length ``num_classes``, or ``None`` when nothing was counted (an + empty or fully out-of-range target set) — so a caller can tell "no weights" from + "all-zero weights". A class observed **zero** times gets weight ``0.0``, not infinity. + + Example:: + + inverse_frequency_weights([Label(0), Label(0), Label(0), Label(1)], num_classes=2) + # array([0.6667, 2.0], dtype=float32) + """ + counts = class_counts(targets, num_classes, label_map) + total = float(counts.sum()) + if total <= 0: + return None + with np.errstate(divide="ignore", invalid="ignore"): + weights = np.where(counts > 0, total / (num_classes * counts), 0.0) + return weights.astype(np.float32) + + +__all__ = ["LabelMap", "class_counts", "inverse_frequency_weights"] diff --git a/recordstream/outputs.py b/recordstream/outputs.py new file mode 100644 index 0000000..7b5e73b --- /dev/null +++ b/recordstream/outputs.py @@ -0,0 +1,117 @@ +"""Prediction-output contracts — what a model hands to a sink, metric, or visualizer. + +Every model's eval-mode ``forward`` should return one of these — or a task-specific superset. +The keys *are* the documentation: downstream metrics, sinks, visualizers, and evaluators +inspect known keys instead of guessing whether a tensor is logits, probabilities, or argmax'd +class ids. That guess is not hypothetical: two independently-written detector wrappers agree +that ``boxes`` is xyxy in absolute pixels only because :class:`DetectionOutput` says so. + +**Generic in the array type.** Each contract is a generic ``TypedDict`` parameterized by the +array type it carries, so the SAME contract describes a torch run, a numpy/JAX run, or a +TensorFlow one:: + + ClassificationOutput[Tensor] # a torch model + ClassificationOutput[np.ndarray] # a numpy / TF / JAX model + +The parameter is optional — a bare ``ClassificationOutput`` means "whatever array type". + +**Why here.** These describe the boundary between a model and whatever consumes its output, +and the consumer this package ships is :mod:`recordstream.predictions` — the sink that reads +``probs`` / ``class_idx`` by name. A contract owned by a different package than its reader is +how "it lives there because that other thing lives there" starts. + +The BUILDERS (bottom of this module) are necessarily per-framework — ``softmax`` and ``argmax`` +are library calls, not type declarations — and are torch, like the rest of this package's tensor +surface. A backend on another framework adds its own builders beside these; it does NOT redefine +the contracts. +""" + +from typing import Generic, List, TypedDict, TypeVar + +import torch +import torch.nn.functional as F +from torch import Tensor + +#: The array type a contract carries — ``torch.Tensor``, ``np.ndarray``, a TF/JAX array. +ArrayT = TypeVar("ArrayT") + +__all__ = [ + "ArrayT", + "ClassificationOutput", + "DetectionOutput", + "DetectionPredictions", + "SegmentationOutput", + "classification_output", + "segmentation_output", +] + + +class ClassificationOutput(TypedDict, Generic[ArrayT]): + """Per-record classification prediction. + + Keys: + logits: ``[B, C]`` float — raw pre-softmax scores. + probs: ``[B, C]`` float — softmax over the last dim. + class_idx: ``[B]`` int64 — argmax of ``logits`` along the last dim. + """ + + logits: ArrayT + probs: ArrayT + class_idx: ArrayT + + +class DetectionOutput(TypedDict, Generic[ArrayT]): + """Per-image object detection prediction. + + Keys: + boxes: ``[N, 4]`` float32 — xyxy in **absolute pixels**. + scores: ``[N]`` float32 — confidence in ``[0, 1]``. + labels: ``[N]`` int64 — class ids. Class 0 is conventionally reserved + for background in torchvision-style detectors. + """ + + boxes: ArrayT + scores: ArrayT + labels: ArrayT + + +class SegmentationOutput(TypedDict, Generic[ArrayT]): + """Per-pixel segmentation prediction. + + Keys: + logits: ``[B, C, H, W]`` float — raw pre-softmax scores. + probs: ``[B, C, H, W]`` float — softmax over the channel dim. + mask: ``[B, H, W]`` int64 — argmax across channels. + """ + + logits: ArrayT + probs: ArrayT + mask: ArrayT + + +#: A detector's per-image results — one :class:`DetectionOutput` per image in the batch. +DetectionPredictions = List[DetectionOutput] + + +def classification_output(logits: Tensor) -> ClassificationOutput[Tensor]: + """Build a full :class:`ClassificationOutput` from raw ``[B, C]`` logits. + + Example:: + + def predict_step(self, batch, batch_idx): + return classification_output(self(x)) # what a predictions sink reads + """ + probs = F.softmax(logits, dim=-1) + class_idx = torch.argmax(logits, dim=-1).to(torch.int64) + return ClassificationOutput(logits=logits, probs=probs, class_idx=class_idx) + + +def segmentation_output(logits: Tensor) -> SegmentationOutput[Tensor]: + """Build a full :class:`SegmentationOutput` from raw ``[B, C, H, W]`` logits.""" + probs = F.softmax(logits, dim=1) + mask = torch.argmax(logits, dim=1).to(torch.int64) + return SegmentationOutput(logits=logits, probs=probs, mask=mask) + + +# Detection deliberately has NO builder: boxes come from the detector's own interface, so the +# dict is built inline at the call site rather than invented from nothing here. diff --git a/recordstream/predictions.py b/recordstream/predictions.py new file mode 100644 index 0000000..4d9aa01 --- /dev/null +++ b/recordstream/predictions.py @@ -0,0 +1,206 @@ +"""Predictions sinks — where a predict/test loop's per-record output goes. + +A runnable's predict loop calls ``predictions_sink.write(prediction, metadata)`` once per +record and ``predictions_sink.close()`` at end of run. This module carries the protocol that +states that contract plus the classification sink built on it. + +**Why this is a SECOND sink protocol.** :class:`recordstream.storage.base.DataSink` takes a +whole ``record`` (``write(record)``) and is what ``RecordSinkOp`` adapts into an op chain. +A predictions sink instead receives the MODEL's output plus the metadata of the record it came +from, and builds the record itself — the two halves arrive separately because a model emits a +batch while the sink contract is per-record. The split is deliberate and load-bearing +elsewhere: a visual editor surfaces ``category="sink"`` storage sinks as canvas nodes and +excludes prediction sinks precisely because their signature differs. Collapsing them (have the +runnable build the record and write through ``DataSink``) is a real option, tracked in +``TASKS.md`` — until then, do not blur the two. +""" + +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record + +logger = get_logger(__name__) + + +@runtime_checkable +class PredictionsSink(Protocol): + """What a runnable's ``predictions_sink`` slot must provide. + + Structural, not a base class: a sink is anything that can take one record's prediction plus + that record's metadata, and be closed at the end. Naming it here — beside the sink this + package ships — means consuming runnables annotate ``Optional[Lazy[PredictionsSink]]`` + instead of ``Any``, which declared nothing and let a use site call ``.write`` on a slot that + might still be a deferred marker. + + ``@runtime_checkable`` so an ``isinstance`` guard is available; note that only the METHOD + NAMES are checked at runtime, never their signatures. + """ + + def write(self, prediction: Any, metadata: Dict[str, Any]) -> None: + """Record one prediction alongside the metadata of the record it came from.""" + ... + + def close(self) -> None: + """Flush and release whatever the sink holds open.""" + ... + + +@configurable +class ClassificationPredictionsSink: + """Lift classification predictions into a record and run ops over it. + + Bridges a predict/test loop's ``predictions_sink`` contract — ``write(prediction, metadata)`` + per record — to record ops, giving the train→eval→predict triad a uniform shape across + tasks. + + For each call: + + 1. Read the model's :class:`~recordstream.outputs.ClassificationOutput` — ``probs`` ``[C]`` + and ``class_idx`` scalar, per record. + 2. Resolve the int class id to a human-readable label via ``label_names``, and build a top-k + list (the ``top_k`` highest-probability classes with their probabilities + labels). + 3. Build a fresh record carrying the original metadata plus the prediction columns under a + single ``"metadata"`` key: + + * ``predicted_class_id`` (int) + * ``predicted_class_label`` (str) + * ``predicted_confidence`` (float — top-1 probability) + * ``predicted_top_k`` (list of ``{class_id, label, probability}``, descending) + + 4. Thread that record through ``ops`` — typically just ``RecordSinkOp(sink=…)`` to dump JSON. + + Modality-neutral: it reads only the prediction contract and the metadata it is handed, so a + classifier over images, spectrograms or tabular rows uses it unchanged. Diagnostics identify + a record by its ORDINAL in the run for the same reason — no domain key names appear here. + + Zero-arg constructible, like every configurable in this package: ``ops`` is required to RUN, + not to BUILD, so the non-empty check fires on the first :meth:`write` with a clear message + rather than in ``__init__`` (which would make the class unbuildable by a schema/form + generator that instantiates with defaults to introspect it). + + Args: + ops: Ops to run on each per-prediction record. Required by the time the sink is written + to; validated there, not at construction. + label_names: Optional ``{int_class_id: str}`` map converting the model's int64 class ids + back to human-readable strings. Without it, labels become ``str(class_id)``. YAML int + keys land here as strings (config loaders stringify mapping keys); both forms are + accepted and normalized to ``str`` internally. + top_k: Number of top-probability predictions recorded per record. Defaults to ``1``; + ``top_k > num_classes`` is silently clamped. + confidence_threshold: Predictions whose top-1 probability is below this are skipped + entirely (no record produced, no ops run). Default ``0.0`` keeps every prediction. + """ + + def __init__( + self, + ops: Optional[List[Any]] = None, + label_names: Optional[Dict[Any, str]] = None, + top_k: int = 1, + confidence_threshold: float = 0.0, + ) -> None: + if top_k < 1: + raise ValueError(f"top_k must be >= 1, got {top_k}.") + self.ops: List[Any] = list(ops or []) + self.top_k = int(top_k) + self.confidence_threshold = float(confidence_threshold) + # Normalize keys to `str` so YAML-loaded maps (always stringified) and Python-constructed + # maps (which may use int keys) are both addressable by the same lookup. + self.label_names: Dict[str, str] = {str(k): str(v) for k, v in (label_names or {}).items()} + #: How many predictions have been offered — the ordinal a diagnostic names. + self._seen = 0 + + def _label_for(self, class_id: int) -> str: + return self.label_names.get(str(int(class_id)), str(int(class_id))) + + def write(self, prediction: Dict[str, Any], metadata: Dict[str, Any]) -> None: + from confluid import flow + from confluid.fluid import Fluid + + if not self.ops: + raise ValueError( + "ClassificationPredictionsSink: 'ops' is empty — wire at least one op to receive " + "the prediction records (typically RecordSinkOp(sink=...))." + ) + + where = f"record #{self._seen}" + self._seen += 1 + + probs = prediction.get("probs") + class_idx = prediction.get("class_idx") + if probs is None or class_idx is None: + logger.warning( + f"ClassificationPredictionsSink: {where} is missing 'probs' or 'class_idx' " + f"(got keys: {sorted(prediction.keys())}); skipping." + ) + return + + probs_arr = probs.detach().cpu().numpy() if hasattr(probs, "detach") else np.asarray(probs) + # Tolerate both [C] (single-record) and [1, C] (batched-with-1) shapes. + if probs_arr.ndim == 2 and probs_arr.shape[0] == 1: + probs_arr = probs_arr[0] + if probs_arr.ndim != 1: + logger.warning( + f"ClassificationPredictionsSink: {where} has probs of shape {probs_arr.shape}, " + f"expected [C] or [1, C]; skipping." + ) + return + + n_classes = int(probs_arr.shape[0]) + top1_id = int(class_idx.item()) if hasattr(class_idx, "item") else int(class_idx) + top1_prob = float(probs_arr[top1_id]) + if top1_prob < self.confidence_threshold: + logger.debug( + f"ClassificationPredictionsSink: {where} top1_prob={top1_prob:.3f} " + f"< threshold={self.confidence_threshold:.3f}; skipping." + ) + return + + k = min(self.top_k, n_classes) + top_k_ids = np.argsort(-probs_arr)[:k] # descending, first k + top_k_list = [ + { + "class_id": int(cid), + "label": self._label_for(int(cid)), + "probability": float(probs_arr[cid]), + } + for cid in top_k_ids + ] + + new_metadata = dict(metadata) + new_metadata["predicted_class_id"] = top1_id + new_metadata["predicted_class_label"] = self._label_for(top1_id) + new_metadata["predicted_confidence"] = top1_prob + new_metadata["predicted_top_k"] = top_k_list + + logger.debug( + f"ClassificationPredictionsSink: {where} top1={new_metadata['predicted_class_label']!r} " + f"({top1_prob:.3f}), top_k={k}" + ) + + # The prediction metadata rides a single "metadata" key on a plain record. Downstream ops + # (typically `RecordSinkOp` wrapping a JSON sink) read it via `record["metadata"]`. There + # is no input/target on a prediction-only record. + record: Record = {"metadata": new_metadata} + for i, op in enumerate(self.ops): + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + record = op(record) + + def close(self) -> None: + """Propagate ``close()`` to ops that own resources (e.g. wrapped sinks). + + The sink itself buffers nothing, but the wrapped ops typically do (e.g. ``RecordSinkOp`` + wrapping a buffered/file-handle sink). + """ + for op in self.ops: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + +__all__ = ["ClassificationPredictionsSink", "PredictionsSink"] diff --git a/tests/test_labels.py b/tests/test_labels.py index e83e395..a10b31d 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -2,10 +2,11 @@ import json +import numpy as np import pytest from recordstream import Label, MultiLabel, is_class_id -from recordstream.labels import LabelMap +from recordstream.labels import LabelMap, class_counts, inverse_frequency_weights from recordstream.ops.target import DecodeTarget, EncodeTarget # --------------------------------------------------------------------------- @@ -299,3 +300,71 @@ def test_encode_flows_a_deferred_source() -> None: deferred = ConfluidClass(Stream, source=records) assert list(iter_key(LabelMap(mapping={"cat": 0}).encode(deferred), "class")) == [0] + + +# --------------------------------------------------------------------------- +# class_counts / inverse_frequency_weights — the label STATISTIC behind balancing +# --------------------------------------------------------------------------- + + +def test_class_counts_accepts_every_target_shape() -> None: + """Label / MultiLabel / bare id all normalize through LabelMap.to_ids.""" + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + assert list(class_counts([Label("cat"), Label("dog"), Label("cat")], 2, lm)) == [2.0, 1.0] + # A multi-label target counts for EVERY class it names. + assert list(class_counts([MultiLabel(["cat", "dog"]), MultiLabel(["dog"])], 2, lm)) == [1.0, 2.0] + # Integer targets need no map at all — `to_ids` passes encoded ids through. + assert list(class_counts([0, 1, 1], 2)) == [1.0, 2.0] + + +def test_class_counts_skips_none_targets() -> None: + assert list(class_counts([Label(0), None, Label(0)], 2)) == [2.0, 0.0] + + +def test_uniform_distribution_weights_every_class_equally() -> None: + weights = inverse_frequency_weights([0, 1, 2, 0, 1, 2], num_classes=3) + assert weights is not None + assert np.allclose(weights, np.ones(3)) + + +def test_rare_classes_weigh_more_than_common_ones() -> None: + weights = inverse_frequency_weights([0, 0, 0, 1], num_classes=2) + assert weights is not None + # w = total / (num_classes * count): 4/(2*3) and 4/(2*1) + assert np.allclose(weights, np.array([2 / 3, 2.0])) + assert weights[1] > weights[0] + + +def test_an_unobserved_class_gets_zero_not_infinity() -> None: + weights = inverse_frequency_weights([0, 0], num_classes=3) + assert weights is not None + assert weights[1] == 0.0 and weights[2] == 0.0 + assert np.isfinite(weights).all() + + +def test_out_of_range_ids_are_ignored_rather_than_raising() -> None: + """A stray label must not abort a training run.""" + weights = inverse_frequency_weights([0, 1, 99, -1], num_classes=2) + assert weights is not None + assert np.allclose(weights, np.ones(2)) + + +def test_no_observations_returns_none() -> None: + """`None` distinguishes "no weights" from "all-zero weights".""" + assert inverse_frequency_weights([], num_classes=3) is None + assert inverse_frequency_weights([7, 8], num_classes=3) is None + + +def test_weights_are_numpy_float32_not_a_tensor() -> None: + """Only `batch_tensor` is torch in this package — a framework converts in one line.""" + weights = inverse_frequency_weights([0, 1], num_classes=2) + assert weights is not None + assert isinstance(weights, np.ndarray) and weights.dtype == np.float32 + + +def test_weights_encode_class_NAMES_through_the_map() -> None: + """The flattening a consumer used to do by hand lives here now.""" + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + weights = inverse_frequency_weights([Label("cat")] * 3 + [Label("dog")], 2, lm) + assert weights is not None + assert np.allclose(weights, np.array([2 / 3, 2.0])) diff --git a/tests/test_outputs.py b/tests/test_outputs.py new file mode 100644 index 0000000..3df80d7 --- /dev/null +++ b/tests/test_outputs.py @@ -0,0 +1,111 @@ +"""Tests for :mod:`recordstream.outputs` — the typed prediction-output contracts + torch builders.""" + +import torch + +from recordstream.outputs import ( + ClassificationOutput, + DetectionOutput, + SegmentationOutput, + classification_output, + segmentation_output, +) + + +def test_classification_output_shapes_and_dtypes() -> None: + logits = torch.randn(4, 3) + out = classification_output(logits) + + # TypedDict values are plain dict entries at runtime. + assert set(out.keys()) == {"logits", "probs", "class_idx"} + assert out["logits"] is logits + assert out["probs"].shape == (4, 3) + assert torch.allclose(out["probs"].sum(dim=-1), torch.ones(4), atol=1e-5) + assert out["class_idx"].dtype == torch.int64 + assert out["class_idx"].shape == (4,) + assert torch.equal(out["class_idx"], logits.argmax(dim=-1).to(torch.int64)) + + +def test_segmentation_output_per_pixel_argmax() -> None: + logits = torch.randn(2, 5, 8, 8) + out = segmentation_output(logits) + + assert set(out.keys()) == {"logits", "probs", "mask"} + assert out["probs"].shape == (2, 5, 8, 8) + # Softmax over channel dim. + assert torch.allclose(out["probs"].sum(dim=1), torch.ones(2, 8, 8), atol=1e-5) + assert out["mask"].dtype == torch.int64 + assert out["mask"].shape == (2, 8, 8) + assert torch.equal(out["mask"], logits.argmax(dim=1).to(torch.int64)) + + +def test_detection_output_typed_dict_construction() -> None: + # DetectionOutput is constructed directly at call sites; verify the + # TypedDict's runtime behavior matches a plain dict. + boxes = torch.tensor([[0.0, 0.0, 10.0, 10.0]], dtype=torch.float32) + scores = torch.tensor([0.9], dtype=torch.float32) + labels = torch.tensor([1], dtype=torch.int64) + out = DetectionOutput(boxes=boxes, scores=scores, labels=labels) + assert out["boxes"].dtype == torch.float32 + assert out["scores"].dtype == torch.float32 + assert out["labels"].dtype == torch.int64 + assert set(out.keys()) == {"boxes", "scores", "labels"} + + +def test_classification_output_is_typed_dict_instance() -> None: + out: ClassificationOutput = classification_output(torch.zeros(1, 2)) + # TypedDicts are plain dicts at runtime. + assert isinstance(out, dict) + assert "logits" in out and "probs" in out and "class_idx" in out + + +def test_segmentation_output_is_typed_dict_instance() -> None: + out: SegmentationOutput = segmentation_output(torch.zeros(1, 2, 3, 3)) + assert isinstance(out, dict) + assert set(out.keys()) == {"logits", "probs", "mask"} + + +# --------------------------------------------------------------------------- # +# The contracts stay generic in the array type — only the BUILDERS are torch +# --------------------------------------------------------------------------- # + + +def test_a_contract_parameterizes_over_the_array_type() -> None: + """The same contract describes a torch run and a numpy/TF/JAX one. + + recordstream hard-depends on torch (a `Stream` IS a `torch.utils.data.Dataset`), so the + contracts and their torch builders share one module — but the contracts themselves are + typing-only and generic, so a backend on another framework declares its output with the + SAME types and adds its own builders beside these. + """ + import numpy as np + + numpy_out: ClassificationOutput[np.ndarray] = { + "logits": np.zeros(2), + "probs": np.zeros(2), + "class_idx": np.zeros(2), + } + torch_out: ClassificationOutput[torch.Tensor] = classification_output(torch.zeros(1, 2)) + + assert isinstance(numpy_out["probs"], np.ndarray) + assert isinstance(torch_out["probs"], torch.Tensor) + + +def test_detection_has_no_builder_on_purpose() -> None: + """Boxes come from a detector's own interface; there is nothing to derive from logits.""" + import recordstream.outputs as outputs + + assert not hasattr(outputs, "detection_output") + + +def test_the_package_root_exports_the_contracts_and_builders() -> None: + import recordstream + + for name in ( + "ClassificationOutput", + "DetectionOutput", + "DetectionPredictions", + "SegmentationOutput", + "classification_output", + "segmentation_output", + ): + assert name in recordstream.__all__ and hasattr(recordstream, name) diff --git a/tests/test_predictions.py b/tests/test_predictions.py new file mode 100644 index 0000000..74af387 --- /dev/null +++ b/tests/test_predictions.py @@ -0,0 +1,245 @@ +"""Tests for :mod:`recordstream.predictions` — the predictions-sink contract + the classification sink.""" + +from typing import Any, Dict, List + +import pytest +import torch + +from recordstream import ClassificationPredictionsSink, PredictionsSink, Record + + +class _CapturingOp: + """Tiny op that records every record dict it sees, for assertion purposes.""" + + def __init__(self) -> None: + self.calls: List[Record] = [] + self.closed = False + + def __call__(self, record: Record) -> Record: + self.calls.append(record) + return record + + def close(self) -> None: + self.closed = True + + +def _make_prediction( + probs: List[float], + class_idx: int, +) -> Dict[str, Any]: + """Build a ClassificationOutput-shaped dict from a flat prob list.""" + p = torch.tensor(probs, dtype=torch.float32) + return { + "logits": torch.log(p), # not used by sink, but realistic + "probs": p, + "class_idx": torch.tensor(class_idx, dtype=torch.int64), + } + + +def _make_metadata() -> Dict[str, Any]: + """Arbitrary per-record metadata — the sink must not require any particular key.""" + return {"source_id": "p1", "index": 0} + + +# --- Happy path ------------------------------------------------------------ + + +def test_top1_path_writes_predicted_columns() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + label_names={0: "DJI MINI3", 1: "DJI AVATA2", 2: "Other"}, + top_k=1, + ) + sink.write(_make_prediction([0.1, 0.7, 0.2], class_idx=1), _make_metadata()) + + assert len(op.calls) == 1 + meta = op.calls[0]["metadata"] + assert meta["predicted_class_id"] == 1 + assert meta["predicted_class_label"] == "DJI AVATA2" + assert meta["predicted_confidence"] == pytest.approx(0.7, abs=1e-6) + assert len(meta["predicted_top_k"]) == 1 + assert meta["predicted_top_k"][0]["label"] == "DJI AVATA2" + + +def test_top_k_path_returns_descending_probabilities() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + label_names={0: "a", 1: "b", 2: "c", 3: "d"}, + top_k=3, + ) + sink.write(_make_prediction([0.1, 0.4, 0.45, 0.05], class_idx=2), _make_metadata()) + + top_k = op.calls[0]["metadata"]["predicted_top_k"] + assert len(top_k) == 3 + # Sorted descending by probability. + probs = [entry["probability"] for entry in top_k] + assert probs == sorted(probs, reverse=True) + assert top_k[0]["label"] == "c" + assert top_k[1]["label"] == "b" + assert top_k[2]["label"] == "a" + + +def test_top_k_clamps_when_exceeds_num_classes() -> None: + """top_k=10 with 3 classes → returns 3, not crash.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=10) + sink.write(_make_prediction([0.5, 0.3, 0.2], class_idx=0), _make_metadata()) + assert len(op.calls[0]["metadata"]["predicted_top_k"]) == 3 + + +def test_label_names_string_keys_work() -> None: + """YAML loads int keys as strings; both forms must work.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + label_names={"0": "a", "1": "b"}, # str keys + top_k=1, + ) + sink.write(_make_prediction([0.2, 0.8], class_idx=1), _make_metadata()) + assert op.calls[0]["metadata"]["predicted_class_label"] == "b" + + +def test_label_names_missing_key_falls_back_to_str_class_id() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + label_names={0: "zero"}, # only class 0 mapped + top_k=1, + ) + sink.write(_make_prediction([0.1, 0.9], class_idx=1), _make_metadata()) + # Missing class 1 → falls back to "1". + assert op.calls[0]["metadata"]["predicted_class_label"] == "1" + + +def test_no_label_names_uses_str_class_id() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=2) + sink.write(_make_prediction([0.7, 0.3], class_idx=0), _make_metadata()) + meta = op.calls[0]["metadata"] + assert meta["predicted_class_label"] == "0" + assert {entry["label"] for entry in meta["predicted_top_k"]} == {"0", "1"} + + +def test_threaded_record_carries_metadata_key_only() -> None: + """The threaded record is a plain dict carrying only the prediction metadata key.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write(_make_prediction([0.6, 0.4], class_idx=0), _make_metadata()) + record = op.calls[0] + assert isinstance(record, dict) + assert list(record.keys()) == ["metadata"] + + +def test_close_propagates_to_ops() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.close() + assert op.closed is True + + +# --- Confidence threshold -------------------------------------------------- + + +def test_confidence_threshold_drops_low_predictions() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + top_k=1, + confidence_threshold=0.5, + ) + # top1 prob = 0.4, below threshold → no record produced. + sink.write(_make_prediction([0.4, 0.3, 0.3], class_idx=0), _make_metadata()) + assert op.calls == [] + + +def test_confidence_threshold_keeps_high_predictions() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + top_k=1, + confidence_threshold=0.5, + ) + sink.write(_make_prediction([0.7, 0.3], class_idx=0), _make_metadata()) + assert len(op.calls) == 1 + + +# --- Edge cases ------------------------------------------------------------ + + +def test_zero_arg_construction_works_and_ops_is_validated_lazily() -> None: + """The package-wide lazy-init rule: buildable with no args, validated where it is used.""" + sink = ClassificationPredictionsSink() # no raise — a form/schema generator can introspect it + with pytest.raises(ValueError, match="'ops' is empty"): + sink.write(_make_prediction([0.5, 0.5], class_idx=0), {}) + + +def test_empty_ops_list_raises_on_write_not_construction() -> None: + sink = ClassificationPredictionsSink(ops=[]) + with pytest.raises(ValueError, match="'ops' is empty"): + sink.write(_make_prediction([0.5, 0.5], class_idx=0), {}) + + +def test_zero_top_k_raises() -> None: + with pytest.raises(ValueError, match="top_k"): + ClassificationPredictionsSink(ops=[_CapturingOp()], top_k=0) + + +def test_missing_probs_skips_silently_with_warning() -> None: + """If the model emits a malformed prediction, the sink skips rather than crashing.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write({"logits": torch.zeros(3)}, _make_metadata()) # no probs/class_idx + assert op.calls == [] + + +def test_handles_2d_probs_with_leading_singleton() -> None: + """Some pipelines emit [1, C] instead of [C]; sink must accept both.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write( + { + "logits": torch.zeros(1, 3), + "probs": torch.tensor([[0.1, 0.6, 0.3]], dtype=torch.float32), + "class_idx": torch.tensor(1, dtype=torch.int64), + }, + _make_metadata(), + ) + assert len(op.calls) == 1 + assert op.calls[0]["metadata"]["predicted_class_id"] == 1 + assert op.calls[0]["metadata"]["predicted_confidence"] == pytest.approx(0.6, abs=1e-6) + + +# --- Contract + neutrality ------------------------------------------------- + + +def test_sink_satisfies_the_predictions_sink_protocol() -> None: + """`isinstance` works because the Protocol is @runtime_checkable (method NAMES only).""" + assert isinstance(ClassificationPredictionsSink(ops=[_CapturingOp()]), PredictionsSink) + + +def test_write_works_with_completely_empty_metadata() -> None: + """No domain key is required — the sink is modality-neutral, diagnostics included. + + The pre-move version built its log line from `pack_id` / `iq_file` / `window_start_sample`, + signal-domain keys, in a class documented as neutral. A record identified only by ordinal + keeps that promise. + """ + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write(_make_prediction([0.2, 0.8], class_idx=1), {}) + assert op.calls[0]["metadata"]["predicted_class_id"] == 1 + + +def test_a_malformed_prediction_is_reported_by_ordinal(monkeypatch: pytest.MonkeyPatch) -> None: + """The diagnostic locates the bad record without naming any domain metadata key.""" + from recordstream import predictions as predictions_module + + warnings: List[str] = [] + monkeypatch.setattr(predictions_module.logger, "warning", lambda msg: warnings.append(str(msg))) + + sink = ClassificationPredictionsSink(ops=[_CapturingOp()], top_k=1) + sink.write(_make_prediction([0.5, 0.5], class_idx=0), {}) # record #0 — fine + sink.write({"logits": torch.zeros(2)}, {}) # record #1 — malformed + assert len(warnings) == 1 and "record #1" in warnings[0] diff --git a/tests/test_record_source.py b/tests/test_record_source.py new file mode 100644 index 0000000..baaa41f --- /dev/null +++ b/tests/test_record_source.py @@ -0,0 +1,81 @@ +"""``ensure_record_dataset`` / ``RecordSource`` — normalizing a wired dataset slot. + +A consumer's ``train_set`` / ``val_set`` / ``test_set`` may be wired to a ``Stream``, another +torch ``Dataset``, a bare source, or a plain list. Normalizing once up front lets the rest of a +pipeline assume record items unconditionally. +""" + +from typing import Any, Dict, Iterator, List + +import numpy as np +import torch + +from recordstream import Label, Record, Stream, ensure_record_dataset + + +class _RowDataset(torch.utils.data.Dataset): + """A map-style dataset of raw (non-record) rows.""" + + def __init__(self, rows: List[Dict[str, Any]]) -> None: + self.rows = rows + + def __len__(self) -> int: + return len(self.rows) + + def __getitem__(self, index: int) -> Dict[str, Any]: + return self.rows[index] + + +class _IterableSource: + """A bare iterable source — no ``__getitem__``, no ``Dataset`` base.""" + + def __iter__(self) -> Iterator[Record]: + yield {"image": np.zeros(2), "class": Label(0)} + yield {"image": np.ones(2), "class": Label(1)} + + +def _records(n: int = 2) -> List[Record]: + return [{"image": np.zeros(2), "class": Label(i)} for i in range(n)] + + +def test_a_stream_is_returned_as_is() -> None: + """Identity matters: a subclass's own wrap (e.g. its ``label_names``) must survive.""" + stream = Stream(source=_records()) + assert ensure_record_dataset(stream) is stream + + +def test_a_stream_subclass_keeps_its_attributes() -> None: + stream = Stream(source=_records()) + stream.label_names = ["a", "b"] # type: ignore[attr-defined] + assert ensure_record_dataset(stream).label_names == ["a", "b"] # type: ignore[attr-defined] + + +def test_a_plain_list_becomes_a_map_style_record_dataset() -> None: + dataset = ensure_record_dataset(_records(3)) + assert isinstance(dataset, Stream) + assert len(dataset) == 3 # type: ignore[arg-type] + assert isinstance(dataset[0], dict) and "image" in dataset[0] # type: ignore[index] + + +def test_a_torch_dataset_of_rows_is_wrapped() -> None: + dataset = ensure_record_dataset(_RowDataset(_records(2))) + assert isinstance(dataset, Stream) + assert len(dataset) == 2 # type: ignore[arg-type] + + +def test_a_bare_iterable_source_is_wrapped_and_iterates() -> None: + dataset = ensure_record_dataset(_IterableSource()) + assert isinstance(dataset, Stream) + assert [int(record["class"].value) for record in dataset] == [0, 1] # type: ignore[union-attr] + + +def test_it_is_idempotent() -> None: + once = ensure_record_dataset(_records()) + assert ensure_record_dataset(once) is once + + +def test_exported_from_the_package_root() -> None: + import recordstream + + assert "ensure_record_dataset" in recordstream.__all__ + assert "RecordSource" in recordstream.__all__ From 39eca6785820edd850cb6938f5821bdaf546f1d6 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 20:18:07 +0200 Subject: [PATCH 055/102] feat(labels): the class vocabulary rides the encoded data, and is named class_names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **`LabelMap.encode` now sets `Stream.class_names`.** A stream that has been label-encoded knows its vocabulary; a consumer that needs to name a predicted class id, or persist the mapping beside a checkpoint, should not be handed a separate LabelMap and told to keep it in sync. One consumer had been monkey-patching the attribute on after the fact and reading it back with a `getattr` — an undeclared convention nothing could see. It is now a declared, validated `Optional[List[str]]` constructor slot. **`class_names(*sources)`** joins `num_classes` in `recordstream.projection`: that one WALKS a source to count classes, this one READS the vocabulary a source carries. It takes several sources because a vocabulary is a property of the RUN rather than of whichever split happens to carry it, skips `None` so `class_names(train, val, test)` needs no guards at the call site, and returns `None` when nothing carries one — an integer-labelled run is not an error. **Renamed `label_names` -> `class_names` throughout** (and `from_label_names` -> `from_class_names`). `label_names` collides with HuggingFace `transformers`, where it means "which input dict keys hold the labels" — a different concept, in the largest library in the space. `class_names` is Keras's term, and it already matched what this code serializes: the file is `class_names.json` and the JSON key is `"class_names"`. `num_classes` deliberately unchanged: it is the dominant spelling (timm, torchvision, torchmetrics multiclass, HF `datasets.ClassLabel`), and `num_labels` is specifically the MULTI-LABEL count torchmetrics asks for — which the classifier already passes where torchmetrics wants it. --- AGENTS.md | 4 +- CLAUDE.md | 4 +- GEMINI.md | 4 +- docs/predictions.md | 4 +- docs/projection.md | 4 +- recordstream/__init__.py | 3 +- recordstream/core.py | 9 +++- recordstream/labels.py | 26 ++++++----- recordstream/predictions.py | 10 ++-- recordstream/projection.py | 37 ++++++++++++++- tests/test_labels.py | 91 +++++++++++++++++++++++++++++++++---- tests/test_predictions.py | 8 ++-- tests/test_record_source.py | 6 +-- 13 files changed, 163 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 52f3f49..57fc31d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,8 +39,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `label_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": diff --git a/CLAUDE.md b/CLAUDE.md index 52f3f49..57fc31d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,8 +39,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `label_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": diff --git a/GEMINI.md b/GEMINI.md index 52f3f49..57fc31d 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -39,8 +39,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `label_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `label_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": diff --git a/docs/predictions.md b/docs/predictions.md index e695cb6..38c229b 100644 --- a/docs/predictions.md +++ b/docs/predictions.md @@ -16,7 +16,7 @@ dataset = ensure_record_dataset(self.train_set) # -> a map-style Dataset of re ``` A `Stream` comes back **as-is** — identity matters, because a label-encoding Stream carries its -`label_names` and re-wrapping would lose it. Anything else is wrapped in a `Stream`, which makes it +`class_names` and re-wrapping would lose it. Anything else is wrapped in a `Stream`, which makes it both map-style and record-yielding. `RecordSource` is the contract it enforces (`Dataset | Iterable[Record]`), named once so a consumer @@ -71,7 +71,7 @@ resolve the class id to a label, build a top-k list, and thread a record through ```yaml predictions_sink: !class:recordstream.predictions.ClassificationPredictionsSink - label_names: !ref:class_id_to_label # {0: "bird", 1: "cat", ...} + class_names: !ref:class_id_to_label # {0: "bird", 1: "cat", ...} top_k: 5 confidence_threshold: 0.0 # skip predictions below this top-1 probability ops: diff --git a/docs/projection.md b/docs/projection.md index 0ad3a93..0fc96e0 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -30,7 +30,7 @@ from recordstream import LabelMap, Stream, iter_key lm = LabelMap.fit(iter_key(train_source, "class")) # {"bird": 0, "cat": 1, "dog": 2} lm.num_classes # 3 -lm.label_names # ["bird", "cat", "dog"] (id -> name) +lm.class_names # ["bird", "cat", "dog"] (id -> name) lm.save("class_names.json") # {"class_names": [...], "num_classes": N} encoded = lm.encode(train_source) # a Stream whose "class" Labels carry int ids @@ -53,7 +53,7 @@ one class: from recordstream import LabelMap, MultiLabel lm = LabelMap.fit([MultiLabel(["cat", "dog"]), MultiLabel(["bird"])]) -lm.label_names # ["bird", "cat", "dog"] +lm.class_names # ["bird", "cat", "dog"] ``` `to_ids(target)` is the one accessor a consumer needs — it always returns a **list of int class diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 6f3a76a..ba5dcc1 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -64,7 +64,7 @@ ) from recordstream.predictions import ClassificationPredictionsSink, PredictionsSink from recordstream.processing import DatasetProcessor -from recordstream.projection import SupportsProjection, iter_key, num_classes, project +from recordstream.projection import SupportsProjection, class_names, iter_key, num_classes, project from recordstream.runnable import ( ProgressCallback, ProgressReporting, @@ -152,6 +152,7 @@ # ---- projection ---- "SupportsProjection", "iter_key", + "class_names", "num_classes", "project", # ---- runnable protocol + orchestration ---- diff --git a/recordstream/core.py b/recordstream/core.py index a888949..ce57cd4 100644 --- a/recordstream/core.py +++ b/recordstream/core.py @@ -422,6 +422,11 @@ class Stream(torch.utils.data.Dataset[Record]): source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. ops: Ordered ops applied lazily on access — native ops and bare library transforms alike (``None`` = no ops). chunk_size: Parallel-processing chunk size; ``0`` (the default) processes sequentially. + class_names: Optional ordered class vocabulary this stream's labels index into. Set by + :meth:`~recordstream.LabelMap.encode` so the vocabulary travels WITH the encoded + data — a consumer that needs to name a predicted class id, or persist the mapping + beside a checkpoint, reads it via :func:`~recordstream.class_names` instead of + being handed a separate LabelMap it has to keep in sync. """ def __init__( @@ -429,9 +434,11 @@ def __init__( source: Optional[Iterable[Any]] = None, ops: Optional[List[Any]] = None, chunk_size: Optional[int] = 0, + class_names: Optional[List[str]] = None, ) -> None: self.source = source self.ops: List[Any] = ops or [] + self.class_names: Optional[List[str]] = class_names self._workers = 1 self._chunk_size = chunk_size or 0 # Populated on first random access when the source is iterable-only @@ -708,7 +715,7 @@ def ensure_record_dataset(source: RecordSource) -> torch.utils.data.Dataset[Any] may be record dicts or raw rows. A ``Stream`` already coerces every item to a record, so: * a ``Stream`` is returned as-is (already a ``Dataset`` of records; this preserves a - subclass's own wrap, e.g. a label-encoding Stream with its ``label_names``), and + subclass's own wrap, e.g. a label-encoding Stream with its ``class_names``), and * anything else is wrapped in a ``Stream``, which makes it both a map-style ``Dataset`` AND a record-yielding one. diff --git a/recordstream/labels.py b/recordstream/labels.py index 723722c..932e607 100644 --- a/recordstream/labels.py +++ b/recordstream/labels.py @@ -60,14 +60,14 @@ class LabelMap: """Bidirectional class-name ↔ integer-id map (the fittable companion to ``EncodeTarget``). Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream - via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`label_names`, builds the + via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`class_names`, builds the :class:`~recordstream.ops.target.EncodeTarget` / :class:`~recordstream.ops.target.DecodeTarget` that apply it, and round-trips to disk in marainer's ``class_names.json`` format. Args: mapping: Explicit name→id lookup, e.g. ``{"cat": 0, "dog": 1}``. ``None`` (default) builds an empty map — valid to construct (zero-arg convention), but the properties raise until it - is populated (by passing a mapping, or via :meth:`fit` / :meth:`from_label_names`). + is populated (by passing a mapping, or via :meth:`fit` / :meth:`from_class_names`). """ def __init__(self, mapping: Optional[Dict[str, int]] = None) -> None: @@ -79,7 +79,7 @@ def _require(self) -> Dict[str, int]: if not self.mapping: raise ValueError( "LabelMap is empty — pass a `mapping`, or build one via LabelMap.fit(targets) / " - "LabelMap.from_label_names(names) / LabelMap.load(path) before use." + "LabelMap.from_class_names(names) / LabelMap.load(path) before use." ) return self.mapping @@ -89,7 +89,7 @@ def num_classes(self) -> int: return max(self._require().values()) + 1 @property - def label_names(self) -> List[str]: + def class_names(self) -> List[str]: """``id → name`` list (index == class id). Ids without a name fall back to ``str(id)``.""" inverse = self.inverse return [inverse.get(i, str(i)) for i in range(self.num_classes)] @@ -117,8 +117,10 @@ def encode(self, source: Any) -> "Stream": Returns: A :class:`~recordstream.Stream` yielding the same records with their labels mapped - to integer ids. Which key is encoded follows :class:`EncodeTarget`'s own rule (its - blank ``field`` picks the first :class:`~recordstream.Label`); pass a configured + to integer ids, and carrying this map's ``class_names`` so the vocabulary travels + with the encoded data (read it back with :func:`~recordstream.class_names`). + Which key is encoded follows :class:`EncodeTarget`'s own rule (its blank + ``field`` picks the first :class:`~recordstream.Label`); pass a configured ``encode_op()`` into a ``Stream`` yourself when you need to pin a different key or tolerate unknowns. @@ -138,7 +140,7 @@ def encode(self, source: Any) -> "Stream": from recordstream.core import Stream - return Stream(source=flow(source), ops=[self.encode_op()]) + return Stream(source=flow(source), ops=[self.encode_op()], class_names=self.class_names) def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTarget: """Return a :class:`~recordstream.ops.target.DecodeTarget` transform that maps id → name via this map.""" @@ -171,14 +173,14 @@ def fit(cls, targets: Iterable[Any]) -> "LabelMap": return cls(mapping={name: idx for idx, name in enumerate(sorted(set(labels)))}) @classmethod - def from_label_names(cls, names: Sequence[str]) -> "LabelMap": - """Build a map from an ordered ``id → name`` list (the inverse of :attr:`label_names`). + def from_class_names(cls, names: Sequence[str]) -> "LabelMap": + """Build a map from an ordered ``id → name`` list (the inverse of :attr:`class_names`). Args: names: Ordered class names; the list index becomes the class id. Must be non-empty. """ if not names: - raise ValueError("LabelMap.from_label_names: `names` is empty.") + raise ValueError("LabelMap.from_class_names: `names` is empty.") return cls(mapping={str(name): int(i) for i, name in enumerate(names)}) def to_ids(self, target: Any) -> List[int]: @@ -224,7 +226,7 @@ def save(self, path: Union[str, Path]) -> None: """ out = Path(path).expanduser() out.parent.mkdir(parents=True, exist_ok=True) - payload = {"class_names": self.label_names, "num_classes": self.num_classes} + payload = {"class_names": self.class_names, "num_classes": self.num_classes} out.write_text(json.dumps(payload, indent=2, sort_keys=True)) @classmethod @@ -239,7 +241,7 @@ def load(cls, path: Union[str, Path]) -> "LabelMap": names = data.get("class_names") if not names: raise ValueError(f"LabelMap.load: {path} has no non-empty 'class_names' list.") - return cls.from_label_names([str(n) for n in names]) + return cls.from_class_names([str(n) for n in names]) def class_counts(targets: Iterable[Any], num_classes: int, label_map: Optional[LabelMap] = None) -> np.ndarray: diff --git a/recordstream/predictions.py b/recordstream/predictions.py index 4d9aa01..8014730 100644 --- a/recordstream/predictions.py +++ b/recordstream/predictions.py @@ -61,7 +61,7 @@ class ClassificationPredictionsSink: 1. Read the model's :class:`~recordstream.outputs.ClassificationOutput` — ``probs`` ``[C]`` and ``class_idx`` scalar, per record. - 2. Resolve the int class id to a human-readable label via ``label_names``, and build a top-k + 2. Resolve the int class id to a human-readable label via ``class_names``, and build a top-k list (the ``top_k`` highest-probability classes with their probabilities + labels). 3. Build a fresh record carrying the original metadata plus the prediction columns under a single ``"metadata"`` key: @@ -85,7 +85,7 @@ class ClassificationPredictionsSink: Args: ops: Ops to run on each per-prediction record. Required by the time the sink is written to; validated there, not at construction. - label_names: Optional ``{int_class_id: str}`` map converting the model's int64 class ids + class_names: Optional ``{int_class_id: str}`` map converting the model's int64 class ids back to human-readable strings. Without it, labels become ``str(class_id)``. YAML int keys land here as strings (config loaders stringify mapping keys); both forms are accepted and normalized to ``str`` internally. @@ -98,7 +98,7 @@ class ClassificationPredictionsSink: def __init__( self, ops: Optional[List[Any]] = None, - label_names: Optional[Dict[Any, str]] = None, + class_names: Optional[Dict[Any, str]] = None, top_k: int = 1, confidence_threshold: float = 0.0, ) -> None: @@ -109,12 +109,12 @@ def __init__( self.confidence_threshold = float(confidence_threshold) # Normalize keys to `str` so YAML-loaded maps (always stringified) and Python-constructed # maps (which may use int keys) are both addressable by the same lookup. - self.label_names: Dict[str, str] = {str(k): str(v) for k, v in (label_names or {}).items()} + self.class_names: Dict[str, str] = {str(k): str(v) for k, v in (class_names or {}).items()} #: How many predictions have been offered — the ordinal a diagnostic names. self._seen = 0 def _label_for(self, class_id: int) -> str: - return self.label_names.get(str(int(class_id)), str(int(class_id))) + return self.class_names.get(str(int(class_id)), str(int(class_id))) def write(self, prediction: Dict[str, Any], metadata: Dict[str, Any]) -> None: from confluid import flow diff --git a/recordstream/projection.py b/recordstream/projection.py index b0c7002..0af17f7 100644 --- a/recordstream/projection.py +++ b/recordstream/projection.py @@ -21,7 +21,7 @@ make every ``Stream`` look classification-capable to duck-typed consumers. """ -from typing import Any, Collection, Iterator, Protocol, runtime_checkable +from typing import Any, Collection, Iterator, List, Optional, Protocol, runtime_checkable from recordstream.items import Label, MultiLabel, Record, is_item, item_data @@ -101,6 +101,41 @@ def _to_int(value: Any) -> int: raise TypeError(f"target {value!r} of type {type(value).__name__} is not a scalar class id") +def class_names(*sources: Any) -> Optional[List[str]]: + """The class vocabulary carried by the first of ``sources`` that has one. + + The naming counterpart of :func:`num_classes`: that one WALKS a source to count classes, + this one READS the vocabulary a source already carries — set by + :meth:`~recordstream.LabelMap.encode` when it wrapped the source, so the names travel with + the encoded data rather than in a LabelMap the consumer has to keep alongside it. + + Takes several sources because a vocabulary is a property of the RUN, not of whichever + split happens to carry it: a config may encode only the train set, or hand the eval path a + pre-encoded test set. ``None`` entries are skipped, so the common + ``class_names(train_set, val_set, test_set)`` needs no guards at the call site. + + Args: + *sources: Datasets / streams to consult, in priority order. ``None`` values are ignored. + + Returns: + The names as a list of ``str``, or ``None`` when no source carries a usable vocabulary + — a source with no labels, or an unencoded one, is not an error. + + Example:: + + names = class_names(train_set, val_set, test_set) # ['bird', 'cat', 'dog'] or None + """ + for source in sources: + names = getattr(source, "class_names", None) + if not names: + continue + try: + return [str(n) for n in names] + except TypeError: # not iterable — a source using the name for something else + continue + return None + + def num_classes(source: Any, key: str = "class") -> int: """Derive the number of classes by walking **every** ``key`` value in ``source``. diff --git a/tests/test_labels.py b/tests/test_labels.py index a10b31d..14e3633 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -25,7 +25,7 @@ def test_empty_map_properties_raise() -> None: with pytest.raises(ValueError): _ = lm.num_classes with pytest.raises(ValueError): - _ = lm.label_names + _ = lm.class_names with pytest.raises(ValueError): _ = lm.inverse @@ -34,7 +34,7 @@ def test_explicit_mapping_coerces_types() -> None: lm = LabelMap(mapping={"cat": 0, "dog": 1}) assert lm.mapping == {"cat": 0, "dog": 1} assert lm.num_classes == 2 - assert lm.label_names == ["cat", "dog"] + assert lm.class_names == ["cat", "dog"] assert lm.inverse == {0: "cat", 1: "dog"} @@ -46,14 +46,14 @@ def test_explicit_mapping_coerces_types() -> None: def test_fit_uses_sorted_ordering() -> None: lm = LabelMap.fit(["dog", "cat", "dog", "bird", "cat"]) # sklearn LabelEncoder sorts classes lexicographically. - assert lm.label_names == ["bird", "cat", "dog"] + assert lm.class_names == ["bird", "cat", "dog"] assert lm.mapping == {"bird": 0, "cat": 1, "dog": 2} assert lm.num_classes == 3 def test_fit_coerces_non_strings() -> None: lm = LabelMap.fit([1, 2, 1, 3]) - assert lm.label_names == ["1", "2", "3"] + assert lm.class_names == ["1", "2", "3"] def test_fit_empty_raises() -> None: @@ -62,20 +62,20 @@ def test_fit_empty_raises() -> None: # --------------------------------------------------------------------------- -# from_label_names — inverse of label_names +# from_class_names — inverse of class_names # --------------------------------------------------------------------------- def test_from_label_names_round_trip() -> None: names = ["bird", "cat", "dog"] - lm = LabelMap.from_label_names(names) - assert lm.label_names == names + lm = LabelMap.from_class_names(names) + assert lm.class_names == names assert lm.mapping == {"bird": 0, "cat": 1, "dog": 2} def test_from_label_names_empty_raises() -> None: with pytest.raises(ValueError): - LabelMap.from_label_names([]) + LabelMap.from_class_names([]) # --------------------------------------------------------------------------- @@ -117,11 +117,11 @@ def test_save_load_round_trip(tmp_path: object) -> None: lm.save(path) restored = LabelMap.load(path) assert restored.mapping == lm.mapping - assert restored.label_names == lm.label_names + assert restored.class_names == lm.class_names def test_save_writes_class_names_payload(tmp_path: object) -> None: - lm = LabelMap.from_label_names(["a", "b", "c"]) + lm = LabelMap.from_class_names(["a", "b", "c"]) path = tmp_path / "class_names.json" # type: ignore[operator] lm.save(path) data = json.loads(path.read_text()) # type: ignore[attr-defined] @@ -368,3 +368,74 @@ def test_weights_encode_class_NAMES_through_the_map() -> None: weights = inverse_frequency_weights([Label("cat")] * 3 + [Label("dog")], 2, lm) assert weights is not None assert np.allclose(weights, np.array([2 / 3, 2.0])) + + +# --------------------------------------------------------------------------- # +# The vocabulary rides the encoded data +# --------------------------------------------------------------------------- # +# A consumer that needs to name a predicted class id, or persist the mapping beside a +# checkpoint, should not have to be handed a separate LabelMap and keep it in sync with +# the dataset. Before this, one consumer monkey-patched the attribute on and read it +# back with a getattr — an undeclared convention nothing could see. + + +def test_encode_carries_the_vocabulary_onto_the_stream() -> None: + from recordstream import iter_key + + records = [{"class": Label(n)} for n in ["dog", "cat", "bird"]] + label_map = LabelMap.fit(iter_key(records, "class")) + + assert label_map.encode(records).class_names == ["bird", "cat", "dog"] + + +def test_a_plain_stream_carries_no_vocabulary() -> None: + from recordstream import Stream + + assert Stream(source=[{"class": Label(0)}]).class_names is None + + +def test_class_names_reads_the_first_source_that_has_one() -> None: + """A vocabulary is a property of the RUN, so the caller passes every split.""" + from recordstream import Stream, class_names, iter_key + + records = [{"class": Label(n)} for n in ["dog", "cat"]] + encoded = LabelMap.fit(iter_key(records, "class")).encode(records) + plain = Stream(source=records) + + assert class_names(plain, encoded) == ["cat", "dog"] + + +def test_class_names_skips_none_so_call_sites_need_no_guards() -> None: + from recordstream import class_names, iter_key + + records = [{"class": Label("cat")}] + encoded = LabelMap.fit(iter_key(records, "class")).encode(records) + + assert class_names(None, None, encoded) == ["cat"] + + +def test_class_names_is_none_when_nothing_carries_one() -> None: + """An unencoded run is not an error — it has integer labels and no vocabulary.""" + from recordstream import class_names + + assert class_names(None, [{"class": Label(0)}]) is None + + +def test_class_names_coerces_a_foreign_sources_names_to_str() -> None: + """A Stream validates its own `List[str]`; a foreign dataset attribute is not so lucky.""" + from recordstream import class_names + + class _ForeignDataset: + class_names = [1, 2] # e.g. integer category ids from another library + + assert class_names(_ForeignDataset()) == ["1", "2"] + + +def test_a_stream_rejects_non_string_names_at_construction() -> None: + """The declared `List[str]` is enforced — the slot is config, not a free-for-all.""" + import pytest + + from recordstream import Stream + + with pytest.raises(Exception, match="valid string"): + Stream(source=[], class_names=[1, 2]) diff --git a/tests/test_predictions.py b/tests/test_predictions.py index 74af387..261c638 100644 --- a/tests/test_predictions.py +++ b/tests/test_predictions.py @@ -48,7 +48,7 @@ def test_top1_path_writes_predicted_columns() -> None: op = _CapturingOp() sink = ClassificationPredictionsSink( ops=[op], - label_names={0: "DJI MINI3", 1: "DJI AVATA2", 2: "Other"}, + class_names={0: "DJI MINI3", 1: "DJI AVATA2", 2: "Other"}, top_k=1, ) sink.write(_make_prediction([0.1, 0.7, 0.2], class_idx=1), _make_metadata()) @@ -66,7 +66,7 @@ def test_top_k_path_returns_descending_probabilities() -> None: op = _CapturingOp() sink = ClassificationPredictionsSink( ops=[op], - label_names={0: "a", 1: "b", 2: "c", 3: "d"}, + class_names={0: "a", 1: "b", 2: "c", 3: "d"}, top_k=3, ) sink.write(_make_prediction([0.1, 0.4, 0.45, 0.05], class_idx=2), _make_metadata()) @@ -94,7 +94,7 @@ def test_label_names_string_keys_work() -> None: op = _CapturingOp() sink = ClassificationPredictionsSink( ops=[op], - label_names={"0": "a", "1": "b"}, # str keys + class_names={"0": "a", "1": "b"}, # str keys top_k=1, ) sink.write(_make_prediction([0.2, 0.8], class_idx=1), _make_metadata()) @@ -105,7 +105,7 @@ def test_label_names_missing_key_falls_back_to_str_class_id() -> None: op = _CapturingOp() sink = ClassificationPredictionsSink( ops=[op], - label_names={0: "zero"}, # only class 0 mapped + class_names={0: "zero"}, # only class 0 mapped top_k=1, ) sink.write(_make_prediction([0.1, 0.9], class_idx=1), _make_metadata()) diff --git a/tests/test_record_source.py b/tests/test_record_source.py index baaa41f..373f758 100644 --- a/tests/test_record_source.py +++ b/tests/test_record_source.py @@ -39,15 +39,15 @@ def _records(n: int = 2) -> List[Record]: def test_a_stream_is_returned_as_is() -> None: - """Identity matters: a subclass's own wrap (e.g. its ``label_names``) must survive.""" + """Identity matters: a subclass's own wrap (e.g. its ``class_names``) must survive.""" stream = Stream(source=_records()) assert ensure_record_dataset(stream) is stream def test_a_stream_subclass_keeps_its_attributes() -> None: stream = Stream(source=_records()) - stream.label_names = ["a", "b"] # type: ignore[attr-defined] - assert ensure_record_dataset(stream).label_names == ["a", "b"] # type: ignore[attr-defined] + stream.class_names = ["a", "b"] # type: ignore[attr-defined] + assert ensure_record_dataset(stream).class_names == ["a", "b"] # type: ignore[attr-defined] def test_a_plain_list_becomes_a_map_style_record_dataset() -> None: From d74aa072515a4f125124a055934c6460635fde82 Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 20:25:45 +0200 Subject: [PATCH 056/102] feat(projection): materialize a deferred source in project() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `LabelMap.encode` flowed its source; `project` / `iter_key` / `num_classes` did not. So a consumer had to know which entry point needed `flow(source)` at the call site — and one wrote it defensively before every walk to compensate. `project` now flows first. Flowing a live object is a no-op, so the common path is unchanged; what disappears is having to remember. The projection itself was already the efficient part: `iter_key` goes through `project(source, (key,))`, so a projection-aware source never builds the values the walk did not ask for. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- GEMINI.md | 2 +- docs/projection.md | 2 +- recordstream/projection.py | 8 +++++++ tests/test_labels.py | 43 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 55 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 57fc31d..80b2f02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. -- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. diff --git a/CLAUDE.md b/CLAUDE.md index 57fc31d..80b2f02 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. -- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. diff --git a/GEMINI.md b/GEMINI.md index 57fc31d..80b2f02 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -37,7 +37,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. -- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. diff --git a/docs/projection.md b/docs/projection.md index 0fc96e0..8c78c6f 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -19,7 +19,7 @@ labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, n = num_classes(my_source, key="class") # max(class_id) + 1 — always walks ``` -Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Stream.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Stream` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. +A **deferred** source — a `!class:` marker straight out of a config — is materialized first, so a caller never has to know which entry point flows and which doesn't (flowing a live object is a no-op). Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Stream.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Stream` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. ## `LabelMap` — fittable name↔id encoding diff --git a/recordstream/projection.py b/recordstream/projection.py index 0af17f7..a861c12 100644 --- a/recordstream/projection.py +++ b/recordstream/projection.py @@ -44,7 +44,15 @@ def project(source: Any, keys: Collection[str]) -> Iterator[Record]: Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the efficient path that skips building unrequested values); otherwise falls back to a full iteration that keeps only the requested keys. Lazy: a generator. + + A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized first, + so a caller never has to remember which entry point flows and which does not — + :meth:`~recordstream.LabelMap.encode` already did, and every consumer of this one was + writing ``flow(source)`` at the call site to compensate. Flowing a live object is a no-op. """ + from confluid import flow + + source = flow(source) want = frozenset(keys) if isinstance(source, SupportsProjection): yield from source.project(want) diff --git a/tests/test_labels.py b/tests/test_labels.py index 14e3633..613afdf 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -1,6 +1,7 @@ """Tests for :class:`recordstream.labels.LabelMap` — the fittable name↔id label map.""" import json +from typing import Any import numpy as np import pytest @@ -439,3 +440,45 @@ def test_a_stream_rejects_non_string_names_at_construction() -> None: with pytest.raises(Exception, match="valid string"): Stream(source=[], class_names=[1, 2]) + + +# --------------------------------------------------------------------------- # +# A deferred source materializes itself +# --------------------------------------------------------------------------- # +# `LabelMap.encode` already flowed its source; `project` / `iter_key` did not — so every +# consumer wrote `flow(source)` at the call site to compensate, and had to know which +# entry point needed it. Flowing a live object is a no-op, so this costs nothing. + + +def _deferred(records: list) -> Any: + """A `!class:` marker as a config hands one over — unbuilt.""" + from confluid import Class + + from recordstream import Stream + + return Class(Stream, source=records) + + +def test_project_materializes_a_deferred_source() -> None: + from recordstream import project + + assert list(project(_deferred([{"class": Label(0), "extra": 1}]), ("class",))) == [{"class": Label(0)}] + + +def test_iter_key_materializes_a_deferred_source() -> None: + from recordstream import iter_key + + assert list(iter_key(_deferred([{"class": Label(i)} for i in range(3)]), "class")) == [0, 1, 2] + + +def test_num_classes_materializes_a_deferred_source() -> None: + from recordstream import num_classes + + assert num_classes(_deferred([{"class": Label(i)} for i in range(4)])) == 4 + + +def test_a_live_source_is_unaffected() -> None: + """Flowing a built object is a no-op — the common path must not change.""" + from recordstream import iter_key + + assert list(iter_key([{"class": Label(i)} for i in range(3)], "class")) == [0, 1, 2] From 41bcdbdda7f307d1d9857ade1bc375e205ac7caa Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 21:00:16 +0200 Subject: [PATCH 057/102] chore: link CLAUDE.md and GEMINI.md to AGENTS.md They were regular file copies, which drift: this project's copies were replaced by symlinks so the three agent-facing filenames can never disagree again. --- CLAUDE.md | 58 +------------------------------------------------------ GEMINI.md | 58 +------------------------------------------------------ 2 files changed, 2 insertions(+), 114 deletions(-) mode change 100644 => 120000 CLAUDE.md mode change 100644 => 120000 GEMINI.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 80b2f02..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,57 +0,0 @@ -# RecordStream Mandates - -## Current state - -> **Renamed 2026-07-26 — `sampleflux` → `recordstream`, `Flux` → `Stream`.** The package was named -> for a data model it no longer has: the 2026-07-25 migration made the carrier a **record**, so the -> vocabulary is now one word per concept — a **`Stream`** of **`Record`**s. Import name, distribution -> name, GitHub repo, console script (`recordstream run`), every `recordstream-*` entry point, the -> `RECORDSTREAM_*` StreamStudio socket types, and the on-disk root attr (`recordstream_format`, -> value still `typedrecord-v1`) all moved together; `JointFlux.fluxes` is `JointStream.streams`. -> **No back-compat aliases** — a pre-rename config, saved canvas, or store must be re-pointed -> (a store missing `recordstream_format` raises the usual re-generate error). The word *sample* is -> now reserved for its OTHER meanings and was deliberately NOT renamed: a discrete-time signal -> sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still -> *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). - -Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. - -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. -- **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). -- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. -- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). -- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. -- **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `recordstream.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`recordstream.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(record, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with recordstream.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `recordstream-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. -- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`recordstream.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Stream`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Stream's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `recordstream-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Stream.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. -- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. -- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). -- **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. -- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. -- **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. -- **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. -- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. -- **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. -- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. -- **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. -- **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. -- **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. -- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. - Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). -- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) - -## Testing & Validation -- **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. -- **Multiprocess Safety:** Parallel pipelines MUST use the `spawn` context. Verify pickle-safety of all operations. -- **Line Length:** 120 characters (Black, isort, flake8). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index 80b2f02..0000000 --- a/GEMINI.md +++ /dev/null @@ -1,57 +0,0 @@ -# RecordStream Mandates - -## Current state - -> **Renamed 2026-07-26 — `sampleflux` → `recordstream`, `Flux` → `Stream`.** The package was named -> for a data model it no longer has: the 2026-07-25 migration made the carrier a **record**, so the -> vocabulary is now one word per concept — a **`Stream`** of **`Record`**s. Import name, distribution -> name, GitHub repo, console script (`recordstream run`), every `recordstream-*` entry point, the -> `RECORDSTREAM_*` StreamStudio socket types, and the on-disk root attr (`recordstream_format`, -> value still `typedrecord-v1`) all moved together; `JointFlux.fluxes` is `JointStream.streams`. -> **No back-compat aliases** — a pre-rename config, saved canvas, or store must be re-pointed -> (a store missing `recordstream_format` raises the usual re-generate error). The word *sample* is -> now reserved for its OTHER meanings and was deliberately NOT renamed: a discrete-time signal -> sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still -> *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). - -Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. - -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. -- **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). -- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. -- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). -- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. -- **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `recordstream.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`recordstream.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(record, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with recordstream.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `recordstream-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. -- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`recordstream.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Stream`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Stream's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `recordstream-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Stream.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. -- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. -- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). -- **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. -- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. -- **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. -- **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. -- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. -- **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. -- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. -- **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. -- **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. -- **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. -- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. -- **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. - Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). -- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) - -## Testing & Validation -- **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. -- **Multiprocess Safety:** Parallel pipelines MUST use the `spawn` context. Verify pickle-safety of all operations. -- **Line Length:** 120 characters (Black, isort, flake8). diff --git a/GEMINI.md b/GEMINI.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file From 57d49ca441648b5abf97781d9b32bb163b7d1b5d Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 29 Jul 2026 21:08:03 +0200 Subject: [PATCH 058/102] feat(core): ensure_record_dataset materializes a deferred source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last of the three entry points that disagreed about this. `LabelMap.encode` flowed, `project` now flows, and `ensure_record_dataset` did not — so wrapping a `!class:` marker produced a Stream whose source was still a Fluid, which failed later at first iteration with an error about the Stream rather than about the config that caused it. Its parameter type widens to admit a Fluid and None, because both genuinely occur at the call sites: a config hands over a marker, and an optional split may be unwired (yielding an empty stream, so a caller needs no guard). Three consumers dropped their compensating `flow()` calls. --- recordstream/core.py | 15 +++++- recordstream/flow.py | 115 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/recordstream/core.py b/recordstream/core.py index ce57cd4..e816088 100644 --- a/recordstream/core.py +++ b/recordstream/core.py @@ -707,7 +707,7 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: RecordSource = Union[torch.utils.data.Dataset[Any], Iterable[Record]] -def ensure_record_dataset(source: RecordSource) -> torch.utils.data.Dataset[Any]: +def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> torch.utils.data.Dataset[Any]: """Normalize any wired source into a map-style ``Dataset`` that yields record dicts. A wired ``train_set`` / ``val_set`` / ``test_set`` may be a :class:`Stream`, another torch @@ -723,7 +723,20 @@ def ensure_record_dataset(source: RecordSource) -> torch.utils.data.Dataset[Any] fitting/encoding, collate, metrics) assume record items — no per-call "is this a record?" checks. It lives beside :class:`Stream` because that is the only type it knows: the whole body is "already a Stream? else wrap in one". + + The parameter admits a ``Fluid`` and ``None`` because both genuinely occur at the call + sites: a config hands over a deferred marker, and an optional split may be unwired (which + yields an empty stream, so a caller needs no guard). + + A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized first, + matching :func:`~recordstream.project` and :meth:`~recordstream.LabelMap.encode`. Without + it, wrapping a marker produced a ``Stream`` whose source was still a Fluid — which fails + later, at first iteration, with an error about the Stream rather than about the config that + caused it. Flowing a live object is a no-op. """ + from confluid import flow + + source = flow(source) if isinstance(source, Stream): return source # `cast`: a map-style `Dataset` iterates through Python's legacy `__getitem__` protocol, diff --git a/recordstream/flow.py b/recordstream/flow.py index 4461790..b6e6e89 100644 --- a/recordstream/flow.py +++ b/recordstream/flow.py @@ -39,7 +39,10 @@ ``drop`` flags on the emitted context ops. """ +import concurrent.futures import inspect +import multiprocessing +from copy import deepcopy from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Sequence, Tuple, Union, cast import torch.utils.data @@ -48,7 +51,7 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from recordstream.core import _apply_op +from recordstream.core import OpInvoker, OpMatcher, _apply_op, _extra_op_families, _sync_op_families from recordstream.items import Record from recordstream.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Save, Use, _read_output @@ -245,6 +248,116 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T # --------------------------------------------------------------------------- +def run_steps( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, +) -> Optional[Record]: + """Run ONE record through the parsed steps; ``None`` = filtered (an op returned None). + + The engine's per-record kernel, module-level so a spawn worker can pickle a reference to + it. ``readers`` is the slot-granular reader accounting from :func:`_result_readers`; it + depends only on ``(steps, outputs)``, so a caller running many records MUST compute it + once and pass it in — recomputing per record is an O(steps²) tax on every record (it was + measured at 3.4 µs/record on a 23-step pipeline, roughly half the graph engine's total + overhead over a flat op list). + """ + if readers is None: + readers = _result_readers(steps, outputs) + env: Dict[str, Any] = {} + remaining = {name: len(idx) for name, idx in readers.items()} + + def read_result(name: str, *, copy: bool) -> Any: + value = env[name] + remaining[name] -= 1 + if remaining[name] <= 0: + del env[name] + elif copy: + value = deepcopy(value) + return value + + prev: Optional[str] = None + for step in steps: + # 1. the input record (implicit stream reads move; explicit fan-out reads copy) + if step.from_ is not None: + record = read_result(step.from_, copy=True) + elif prev is not None: + record = read_result(prev, copy=False) + else: + record = seed + + # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) + if step.merge_from: + if not isinstance(record, dict): + raise TypeError( + f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " + f"{type(record).__name__} — expected a record dict." + ) + merged = dict(record) + for ref in step.merge_from: + value = read_result(ref, copy=True) + if not isinstance(value, dict): + raise TypeError( + f"flow step {step.name!r}: merge_from step {ref!r} holds " + f"{type(value).__name__}, expected a record" + ) + merged.update(value) + record = merged + + # 3. per-record parameter binds + if step.op is not None: + op = step.op + if getattr(op, "EXPANDS", False): + raise NotImplementedError( + f"flow step {step.name!r}: {type(op).__name__!r} is a 1→N expanding op — " + "FlowGraph steps are strictly 1→1 (a named-step env has one result per step). " + "Run expanding pipelines through the Stream engine (iterable-only)." + ) + for param, ref in step.bind.items(): + parsed = _split_bind_ref(ref) + if parsed.attr is not None: + producer = next(s for s in steps if s.name == parsed.step) + value = _read_output(producer.op, parsed.attr) + if value is _MISSING: + raise AttributeError( + f"flow step {step.name!r}: bind {param}={ref!r} — " + f"step {parsed.step!r} op has no @output attribute {parsed.attr!r}" + ) + else: + value = read_result(parsed.step, copy=False) + if isinstance(value, dict) and parsed.key: + # "step[key]" = the named entry; bare "step" = the whole record. + value = value[parsed.key] + setattr(op, param, value) + result = _apply_op(record, op) + if result is None: + return None + record = result + + env[step.name] = record + prev = step.name + + return cast(Optional[Record], env.get(outputs)) if outputs in env else None + + +def _graph_worker_task( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, +) -> Optional[Record]: + """Spawn-worker entry point: re-register third-party op families, then run one record. + + Module-level for pickling (the same constraint :func:`recordstream.core._worker_task_multi` + obeys). ``readers`` is deliberately NOT passed across the boundary — it is cheap to derive + once per worker call relative to the process hop, and shipping it would add a second + pickled structure that must stay in sync with ``steps``. + """ + _sync_op_families(families) + return run_steps(seed, steps, outputs) + + @configurable(category="engine") class FlowGraph(torch.utils.data.Dataset[Record]): """Named-step graph engine — executes a ``flow:`` document natively. From a42c0b5032af33c7625681181fcfc1a1f0fc8058 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 30 Jul 2026 11:42:01 +0200 Subject: [PATCH 059/102] =?UTF-8?q?refactor!:=20one=20execution=20model=20?= =?UTF-8?q?=E2=80=94=20the=20step=20graph;=20delete=20the=20lowering=20pas?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ops:` and `flow:` are now two spellings of ONE step graph running on ONE per-record kernel. An `ops:` list compiles to positional steps (`linear_steps` -> s0, s1, …) that never surface; a `flow:` document parses to the same `FlowStep` list with author-chosen names. `Stream` and `FlowGraph` are two facades over that kernel. BREAKING (no back-compat shims): - DELETE `to_ops` / `from_ops` / `flow_yaml_to_stream` / `Stream.from_flow_yaml` - DELETE `recordstream.context` and `recordstream.ops.context` (Save/Use/Drop/ Apply/Capture/MergeFields) + the `recordstream-ops-context` entry point - DELETE the flow<->ops execution-parity suite The lowering pass encoded dataflow as imperative mutation of a per-record cell store, destroying the dependency structure every consumer wants back (reverse- dependency analysis walks `inputs`; a lowered list has none). A branchy graph therefore has NO flat spelling by design: `FlowGraph.to_stream()` raises. Gained: - native spawn parallelism on the graph (no delegation through a lowered list) - 1->N expanding steps in the graph — the remaining subgraph runs once per child, depth-first, over its own shallow env copy (was NotImplementedError) - `_result_readers` hoisted out of the per-record loop, plus an env-free fast path for straight chains (`is_linear`): 23-step chain went 1.41x -> 1.02x vs the old flat loop; unmeasurable with real ops Docs: AGENTS "ONE Execution Model" mandate (supersedes the Context-plane and flow<->ops mandates), architecture.md §3 rewritten as the decision record, graph.md rewritten, README/record-model.md updated. --- AGENTS.md | 7 +- README.md | 4 +- TASKS.md | 2 + docs/architecture.md | 148 ++++---- docs/graph.md | 133 +++---- docs/record-model.md | 5 +- pyproject.toml | 2 - recordstream/__init__.py | 6 +- recordstream/context.py | 119 ------- recordstream/core.py | 195 ++++------ recordstream/flow.py | 674 +++++++++++++---------------------- recordstream/ops/__init__.py | 9 - recordstream/ops/context.py | 314 ---------------- tests/test_categories.py | 12 - tests/test_node_docs.py | 7 - tests/test_typed_flow.py | 254 +++++++++---- 16 files changed, 673 insertions(+), 1218 deletions(-) delete mode 100644 recordstream/context.py delete mode 100644 recordstream/ops/context.py diff --git a/AGENTS.md b/AGENTS.md index 80b2f02..7873917 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ > sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still > *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). -Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → the `Stream`/`JointStream`/`FlowGraph` engines → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config. Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. +Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → ONE step-graph engine behind two facades (`Stream`/`JointStream` for the dataset surface, `FlowGraph` for a `flow:` document) → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config; the context ops + the flow⇄ops lowering pass were DELETED 2026-07-30 (one step-graph engine, see the mandate below). Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. - **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). @@ -24,14 +24,13 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. -- **The Context Is the Graph Data Plane (2026-07-17):** Graph-shaped pipelines (fan-out / fan-in / cross-branch values) execute on the PLAIN sequential engine via the six context ops in `recordstream.ops.context` — `Save` (fork snapshot → cell), `Use` (stream := cell; deep-copies unless `drop`, which frees the cell = move), `Drop` (explicit cell hygiene; deleting a missing cell RAISES — a liveness bug must fail loudly), `Apply` (setattr a wrapped op's `param` from a cell — the ConfigureOp paradigm with the value coming from a cell; a record cell contributes its `key`-named entry when `key` is set, else the WHOLE record; a raw cell value is used verbatim — `_cell_field`), `Capture` (record a wrapped op's live `@output` into a cell — stochastic-correct, reads through `.target`/`.op` wrapper chains), and `MergeFields` (fan-in; UNION the named source cells' entries into the incoming record via dict update, in slot order with last-write-wins on a key collision — `keys` restricts the union, `drop` frees merged cells; avoid a deliberate collision by `RenameField` on the producing branch). They move data through a per-record **`Context`** (`recordstream.context` — a named-cell store, NOT `@configurable`, never in YAML) that the engine creates fresh per source item and activates via a `contextvars.ContextVar` (`_worker_task_multi`, `__getitem__`, and the streamed route's `_Carried(record, ctx)` carrier), so ops reach it inside `__call__` (`context.require(op_name)`) with no signature change and the executor stays `for op in ops`. HARD INVARIANTS: (1) context wiring NEVER touches the record's own entries — a linear run's record is byte-identical whether or not Context threading exists (pinned in the record-model suite under `tests/`); (2) a straight sequence stays a bare `ops:` list with zero extras; (3) cells are stored BY REFERENCE and copied on read (`Use` without `drop`) — the context ops are THE graph-wiring plane (what `flow:` documents and graph exporters lower to); (4) context cells may NOT cross a stream-level op boundary (`Parallel`) — the streamed route raises `RuntimeError` on live cells at the boundary (v1 limit; `Parallel`'s inner chain gets its own contexts via `_worker_task`); (5) outside an engine, a manual loop opts in with `with recordstream.context.activate(Context()):`. All six are `@configurable(category="op", group="structure")`, zero-arg constructible, entry-pointed as `recordstream-ops-context`. Context ops apply their wrapped op through `_apply_op` (the op-family dispatch), so a bare library transform can be `Apply`/`Capture`-wrapped too. -- **`flow:` Documents ⇄ Flat Op Lists — Two Engines, One Parity Contract (2026-07-17):** The READABLE authoring form of a graph pipeline is the `flow:` named-step document (`recordstream.flow`): a mapping `step-name → op` where the name is the reference handle; reserved step keys `from` (input step; omitted = previous; MUST name an EARLIER step — document order is the schedule, forward refs raise, cycles are inexpressible), `merge_from` (fan-in slot — UNION the named steps' record ENTRIES into this step's incoming record, in slot order with last-write-wins), and `bind` (`{param: step}` = the step's WHOLE result record; `{param: step[key]}` = the named ENTRY of the step's record result, lowered to `Apply(key=...)`; `{param: step.attr}` = the step op's live `@output`, lowered through `Capture` — stochastic-correct). A plain-mapping step with no op is a pure fan-in; `{}` is the identity step naming the source. `outputs:` picks the yielded step (default last). Steps apply their ops via `core._apply_op`, so bare library transforms sit in flow steps too. Executed natively by **`FlowGraph`** (`category="engine"`, a torch Dataset sibling of `Stream`; per-record env with copy-on-read/move-on-last-read and AUTOMATIC cell lifetimes; `.parallel()` deliberately delegates to the LOWERED form on Stream's spawn pool — one worker implementation) AND convertible BOTH ways: `to_ops(flow)` lowers to the flat context-ops list (cell names = step names, liveness compiled into `drop` flags, a linear flow lowers to the BARE op list) and `from_ops(ops)` lifts a flat list back (context ops absorbed into step grammar; `Drop`s vanish — liveness is recomputed; unreferenced steps get auto names). **Execution parity both ways is a pinned hard contract** (the flow parity suite under `tests/`); any change to a context op's semantics, the step grammar, or either engine MUST keep the parity suite green. Reader accounting is SLOT-granular (`_result_readers` returns `(consumer, slot)` pairs) because one consumer may read the same producer through its input slot AND a bind slot — only the input slot of the immediately-next step can ride the linear stream (the earlier index-only version double-counted and under-saved; do not regress it). An op whose ctor has a param named like a reserved step key is REJECTED in flow documents (`_check_reserved_collision`). Marker flow: `FlowGraph.from_yaml` uses `confluid.resolve()` (markers stay unbuilt) and `parse_flow` pops reserved keys from marker kwargs BEFORE flowing each op per step (the two-levels-deep constraint). Entry point `recordstream-flow`; `FlowGraph`/`to_ops`/`from_ops`/`Context` are package-root exports; `Stream.from_flow_yaml` is the serial-engine loader twin of `FlowGraph.from_yaml`/`FlowGraph.from_ops_yaml`. +- **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). +- **1→N Expanding Steps Fork the REMAINING Subgraph (2026-07-30, supersedes the flat-engine pending-queue rule):** An op carrying `EXPANDS = True` yields N children from one record; the remaining steps then run ONCE PER CHILD over that child's own shallow copy of the step environment (independent name→result maps, shared values), DEPTH-FIRST so sibling order matches the nested-loop intuition. An empty expansion or a `None` child drops that branch. This works in EVERY route — serial, spawn-parallel (the worker returns a LIST), and inside a `flow:` graph (the old `FlowGraph` raised `NotImplementedError` on an expanding step; that limit is gone). CONSEQUENCES: (1) `__len__`/`__getitem__` RAISE on `Stream` AND `FlowGraph` when any step op expands — the expanded index map is unknowable up front, so the pipeline is ITERABLE-ONLY (iterate, wrap in a torch IterableDataset, window at the SOURCE for random access, or `list(...)`); (2) `run_steps` (the strict 1→1 twin used for indexing) raises rather than silently dropping siblings. Pins: `tests/test_typed_flow.py::TestExpandingSteps`. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). - **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. -- **1→N Expanding Ops Make a Pipeline ITERABLE-ONLY (2026-07-17):** An op that carries `EXPANDS = True` is an EXPANDING op — one carrier in, several out. The engine flattens expansions in ALL routes via `core._worker_task_multi` (a pending-queue executor: the first child continues inline, siblings go to the FRONT of the queue reversed — DEPTH-FIRST, so chained expansions keep nested-loop order) and the streamed route's `per_record` (`yield from` children); each child continues through the REMAINING ops with `ctx.copy()` (shallow — independent cell sets, shared values). An empty expansion / a `None` child just drops. CONSEQUENCES: (1) `Stream.__len__`/`__getitem__` RAISE an actionable `TypeError` when `Stream._expands` (any materialized op expands) — the expanded length/index map is unknowable; iterate, wrap in a torch IterableDataset, window at the SOURCE for random access (the `RFUAVSource` pattern — see the TASKS.md windowing-refactor flag), or `list(stream)`; (2) `_worker_task` (the strict 1→1 helper `Parallel` uses) REJECTS expanding ops with a clear error; (3) `FlowGraph` steps are strictly 1→1 (a named step env has one result per step) — `_run` raises `NotImplementedError` on an expanding step op. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. diff --git a/README.md b/README.md index 9ca548a..765088a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `RecordS - **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. - **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. - **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. -- **Graph pipelines, serial engine:** readable [`flow:` documents](docs/graph.md) with named steps, fan-out/fan-in and per-record `bind:` parameters — executed natively by `FlowGraph` or lowered (bidirectionally, with pinned execution parity) to a flat context-ops list on the plain sequential `Stream` engine. +- **Graph pipelines:** readable [`flow:` documents](docs/graph.md) of named steps — `from:` forks, `merge_from:` merges, `bind:` feeds one step's value into another's parameter. An `ops:` list is the same engine's linear spelling; both parse to one step graph. - **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. - **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-recordstreamstoragequery) — filter stored datasets without loading a single array. - **Passive Introspection:** ops declare the value types they [handle / consume / produce](docs/record-model.md) and are discoverable by category for visual editors and schema generators. @@ -98,7 +98,7 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit |---|---| | [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | | [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), 1→N expanding ops | -| [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, the six Context ops on the serial engine, bidirectional flow⇄ops conversion, `Stream.from_ops_yaml` | +| [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | | [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap`, class-balance weights | diff --git a/TASKS.md b/TASKS.md index 127ea61..d0fd364 100644 --- a/TASKS.md +++ b/TASKS.md @@ -22,3 +22,5 @@ workspace root `TASKS.md`. Completed items are not archived here — git history - [ ] **Grouped splits** — honour a `group_by` metadata key (patient id, source file) so records from one group never leak across train/val. @feature - [ ] **Pre-computed split manifests** — export the train/val index lists + seed as Confluid artifacts for reproducibility and dataset cards. @low @feature - [ ] **Auto-split when `val_set` is missing** — implicit fraction split if only `train_set` is wired; deliberately deferred in favour of explicit YAML, revisit for ergonomics. @low +- [ ] **Give `Stream`'s streamed route a graph shape** @low — a stream-level op (`Parallel`, anything exposing `.stream`) sees the WHOLE stream, so it cannot be a step in the per-record graph; `Stream._iter_streamed` therefore splits the chain at each one and runs the per-record segments through the kernel. That is correct but it is a second iteration strategy. Modelling a stream-level stage as a step KIND would make it one shape again. Not urgent: the split is small and the semantics are clear. +- [ ] **`docs/configure.md` still describes `ConfigureOp` beside the deleted context ops** @low — the op itself is unchanged and stays (its compute-chain side-branch is one node where `bind:` needs a producer step), but the surrounding prose should be re-read now that `Apply`/`Capture` are gone. diff --git a/docs/architecture.md b/docs/architecture.md index 135b401..c28ca43 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -272,91 +272,105 @@ mask = batch_tensor(batch, "target").long() # a segmenter's [N, H, --- -## 3. The per-record Context is an ambient wiring plane (`recordstream.context`, 2026-07-17) +## 3. The graph IS the execution model — the lowering pass was deleted (2026-07-30) + +*(Supersedes "The per-record Context is an ambient wiring plane", 2026-07-17.)* ### Context -Graph-shaped pipelines — fan-out, fan-in, cross-branch values — need somewhere to hold a value -between the op that produces it and the op that consumes it. The obvious candidate, extra keys on -the record itself, was rejected: the record is the carrier that **persists** — it flows into -sinks, crosses process boundaries, and is the record's serialized identity — while wiring data is -transient scaffolding that should be gone by the end of a well-formed graph. Three constraints -shaped the mechanism: ops keep the plain `__call__(record)` signature (no threading a context -parameter through every op), the executor stays a bare `for op in ops` loop (graphs run on the -*plain sequential engine*), and a linear pipeline's records must stay byte-for-byte untouched. +Between 2026-07-17 and 2026-07-30 this package had two ways to run a pipeline. A `flow:` +document (named steps, explicit `from:`/`merge_from:`/`bind:` edges) was the readable authoring +form; a flat `ops:` list was the execution form. A **lowering pass** (`to_ops`) compiled the +first into the second by inserting six *context ops* — `Save`/`Use`/`Drop`/`Apply`/`Capture`/ +`MergeFields` — that moved records through an ambient per-record cell store, and a **lifting +pass** (`from_ops`) reconstructed a flow document from such a list. Execution parity in both +directions was a pinned contract with its own suite. + +The arrangement was coherent but it cost a second executor (`FlowGraph` duplicating `Stream`'s +iteration, length, indexing and batching while being strictly less capable — no `to_sink`, no +`project`, and a `NotImplementedError` on 1→N expanding steps), a permanent parity tax on every +change to an op's semantics, and an ambient `contextvars` plane that nothing in the workspace's +15 real configs ever used. Adoption told the story plainly: every config on disk was a linear +`ops:` list; zero were `flow:` documents; the only producer of branchy pipelines — the visual +editor — compiled its canvas graph *down* to context ops and then lifted it *back* to a flow +document purely for readability. + +The decisive argument was about the consumer nobody had built yet. A lowered list re-encodes +dataflow as imperative mutation of named cells, which is exactly the information a compiler +needs and cannot recover: reverse-dependency analysis walks `node.inputs` backwards from the +outputs, and a flat list has no inputs. Handing a compiler the lowered form means asking it to +run the lifting pass first to rebuild what was just destroyed. ### Decision -`recordstream/context.py` is a **per-record named-cell store activated ambiently**: the engine -creates one fresh `Context` per source item and activates it around the op loop via a -`contextvars.ContextVar`; the six wiring ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields` -in `recordstream.ops.context`) reach it inside `__call__` through `require(op_name)` — no signature -change anywhere. Deliberate semantics: cells are stored **by reference** and copy-on-read is the -*reading* op's decision (`Use` deep-copies unless `drop` frees the cell = move); a missing cell -on read or delete **raises loudly** with the live-cell list (a liveness bug must never pass -silently); 1→N expansion children get `Context.copy()` (shallow — independent cell *sets*, shared -values); cells may NOT cross a stream-level op boundary (`Parallel` raises on live cells — each -inner chain gets its own contexts). A `Context` is never `@configurable` and never appears in -YAML — it is pure runtime plumbing. The public surface is two-tier by design: the `Context` class -is a package-root export, while `activate`/`current`/`require` stay module-qualified — reachable, -but visibly plumbing. `FlowGraph` deliberately does NOT use this module: its named-step documents -give the compiler full knowledge of cell lifetimes, so it manages its own per-record env -directly, held to the context-op semantics by the pinned flow⇄ops execution-parity contract. +**One execution model: the step graph.** Both spellings parse to the same `FlowStep` list and run +through the same per-record kernel (`recordstream.flow.run_steps_multi`). + +- An `ops:` list compiles to positional steps (`core.linear_steps` — `s0`, `s1`, …) whose names + never surface. A sequence IS a graph; no lifting is involved. +- A `flow:` document parses to the same steps with author-chosen names and explicit edges. +- The kernel takes an **env-free fast path** for a straight chain (`is_linear`), so the linear + case carries none of the graph bookkeeping. +- `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and + `recordstream.ops.context` are **deleted**, with no back-compat shims. + +Fan-out, fan-in and cross-step values are expressed as step GRAMMAR rather than as ops: `from:` +is the fork, `merge_from:` the union, `bind:` the cross-step value (including a producer's live +`@output` via `step.attr`). Branch isolation, which the cell store provided by deep-copying on +read, is now a property of the environment: each expansion branch gets its own shallow copy of +the step env, and a fan-out read copies. ### Consequences -- A plain sequential `ops:` list executes a real fan-out/fan-in graph — which is exactly what - graph exporters (a visual canvas, the `flow:` compiler) lower to, so ONE executor serves both - linear and graph pipelines. -- Linear pipelines are provably untouched: no context op ⇒ the Context is created and never - used; the record-byte-identical invariant is pinned in the record-model suite under `tests/`. -- Spawn-parallelism is safe by construction: contexts are created *inside* the worker and never - pickled or shared across processes. -- Ambient state cuts both ways: running an op list containing context ops *outside* an engine - needs an explicit `with activate(Context()):` — forgetting it is a loud, actionable - `RuntimeError`, not silent misbehavior. -- Custom ops can join the wiring plane through the same `require()` seam the built-in six use — - the module being public is what keeps the wiring plane open rather than a closed set of six. +- **One executor.** `Stream` and `FlowGraph` are two facades over one kernel; the parity suite is + gone because there is nothing left to keep in parity. +- **Expanding ops work everywhere.** The graph gained 1→N support (the remaining subgraph runs per + child, depth-first) that the old `FlowGraph` refused outright. +- **A branchy pipeline has no flat spelling — deliberately.** `FlowGraph.to_stream()` raises for + one, and a visual editor's ops-export raises pointing at its flow export. This is the honest + consequence of deleting the pass that manufactured such a spelling. +- **Compilation becomes possible.** A backend reads `FlowGraph.steps` and maps each step to an IR + node with real `inputs`; reverse-dependency pruning runs on the result. +- **Measured cost:** on a 23-step pipeline of trivial ops the graph engine was 1.41x the old flat + loop; hoisting a per-record analysis pass and adding the linear fast path brought it to 1.02x, + and with real ops in the chain the difference is not measurable. +- **Lost with the cell store:** a hand-written wiring op that stashed a value under its own cell + name. Anything that must persist belongs in the record; anything that wires belongs in the + grammar. ### Example -```python -from recordstream import Stream -from recordstream.ops.context import MergeFields, Save - -# Fan-out/fan-in on the PLAIN sequential engine: snapshot → mutate the stream → merge back. -stream = Stream( - source=my_source, - ops=[ - Save(name="clean"), # snapshot into a cell - my_augment_op, # the stream mutates freely - MergeFields(sources=["clean"], keys=["mask"], drop=["clean"]), # fan-in, cell freed - ], -) - -# The same op list outside an engine needs the Context an engine would have created: -from recordstream.context import Context, activate, require +```yaml +# Fan-out -> two branches -> fan-in, entirely in step grammar. No cells, no snapshots. +flow: + spec: !class:mypkg.MakeSpectrogram {} + masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} + boost: !class:mypkg.Boost {from: spec} # second reader of `spec` = the fork + out: {from: boost, merge_from: [masked]} # union, last-write-wins +outputs: out +``` -with activate(Context()): - for op in ops: - record = op(record) +```python +# The same graph, and what a compiler front end reads off it. +from recordstream.flow import parse_flow -# A custom op joins the wiring plane through the same seam the built-in six use: -# require("MyOp").get("clean") / require("MyOp").put("my_cell", value) +steps, outputs = parse_flow(doc["flow"], doc["outputs"]) +for step in steps: + print(step.name, "<-", step.from_, step.merge_from) # every edge, explicit +# out <- boost ('masked',) ``` ### What you may change (and where it's documented) -- **Writing a custom wiring op** is the supported extension point: call `require("YourOpName")` - inside `__call__`, follow the by-reference/copy-on-read discipline, and free cells you consume. - Usage of the six built-in ops lives in [graph.md](graph.md). -- **Keep the surface narrow.** Don't root-export `activate`/`current`/`require`, and don't grow - `Context` into a general blackboard — anything that should *persist with the record* belongs in - the record itself, not in a cell. -- **Changing cell semantics** (by-reference storage, loud missing-cell errors, the `Parallel` - boundary rule, `copy()` shallowness) is an architectural change: the flow⇄ops parity suite and - the pinned context invariants define the contract. Update this record and the recordstream - `AGENTS.md` context mandate together. +- **Adding a step-grammar key** is an architectural change: it widens the contract every consumer + (the engine, a compiler front end, a visual editor's compiler) reads. Update this record, the + `AGENTS.md` flow mandate, and [graph.md](graph.md) together. +- **The linear fast path** (`is_linear`) is an optimization, not a semantic: it must produce + results identical to the general path, and the suite pins that both spellings agree. +- **Do NOT reintroduce a lowering pass.** A flat list that encodes branches as cell mutations is + a second execution model wearing the first one's clothes; the reason it was removed is written + above. If a future runtime genuinely needs a flattened schedule, it owns that pass — over its + own IR, downstream of the graph. --- diff --git a/docs/graph.md b/docs/graph.md index fa27050..2f37063 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -1,15 +1,35 @@ -# Graph pipelines — flow documents, the FlowGraph engine and Context ops +# Graph pipelines — `flow:` documents and the `FlowGraph` engine -## Flow documents & the FlowGraph engine (`recordstream.flow`) +A pipeline is a **graph of named steps**. There is ONE engine and ONE execution model; `ops:` +and `flow:` are two spellings of it, and which one you write is purely about whether the +pipeline branches. -The **readable authoring form** of a graph pipeline is a `flow:` document — named steps where a step's name is how later steps reference its result: +## `ops:` — the linear spelling + +A straight chain is a graph where every step reads the one before it, so it needs no names: + +```yaml +ops: + - !class:recordstream.ops.image.ConvertToImage {width: 224, height: 224} + - !class:albumentations.Normalize {mean: [0.485, 0.456, 0.406], std: [0.229, 0.224, 0.225]} + - !class:recordstream.ops.torch.ToTensor {normalize: false} +``` + +The engine compiles that list into positional steps (`s0`, `s1`, `s2`) and runs it on the same +kernel a `flow:` document uses. The names never surface — nothing in an `ops:` document can +reference a step. Repeats stay distinct: the same op twice in a row is two steps. + +## `flow:` — the named spelling, for branchy pipelines + +When a pipeline forks, merges, or feeds one step's value into another's parameter, the steps +need names — and the name is how a later step refers to an earlier one: ```yaml flow: - spec: !class:mypkg.MakeSpectrogram {} # input: the source record (writes key `image`) + spec: !class:mypkg.MakeSpectrogram {} # input: the source record masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} # 2nd reader of `spec` = fan-out thresh: !class:recordstream.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} - gated: # a step with bind: uses the plain-mapping form (op: + reserved keys) + gated: # a step with bind: uses the plain-mapping form op: !class:recordstream.ops.numpy.Threshold {output: gated_mask} from: spec bind: @@ -18,79 +38,64 @@ flow: outputs: out ``` -Two YAML spelling rules (both verified): SCALAR/list reserved keys (`from:`, `merge_from:`) may ride -inside a `!class:` marker's mapping alongside its kwargs — but **`bind:` (a nested mapping) MUST use -the plain-mapping step form** (`op:` + reserved keys, the `gated` step above): a nested mapping under -a `!class:` marker is consumed by Confluid as addressed configuration and never reaches the step -grammar. Write bind refs in block style or quoted — `{low_level: thresh[image]}` inline is a YAML -parse error (`[` opens a flow sequence). +Two YAML spelling rules (both verified): SCALAR/list reserved keys (`from:`, `merge_from:`) may +ride inside a `!class:` marker's mapping alongside its kwargs — but **`bind:` (a nested mapping) +MUST use the plain-mapping step form** (`op:` + reserved keys, the `gated` step above): a nested +mapping under a `!class:` marker is consumed by Confluid as addressed configuration and never +reaches the step grammar. Write bind refs in block style or quoted — `{low_level: thresh[image]}` +inline is a YAML parse error (`[` opens a flow sequence). Step grammar (three reserved keys, stripped before the op is built): -- **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document order is the schedule and cycles are inexpressible). -- **`merge_from:`** — fan-in: UNION the named steps' record ENTRIES into this step's incoming record, in listed order, last-write-wins on a key collision (the `MergeFields` slot semantics). -- **`bind:`** — `{param: ref}` per-record parameters: a bare `step` binds the step's WHOLE result record, `step[key]` the named ENTRY of its record, and `step.attr` the step op's live `@output` (lowered through `Capture` — stochastic-correct). +- **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document + order is the schedule and cycles are inexpressible). +- **`merge_from:`** — fan-in: UNION the named steps' record ENTRIES into this step's incoming + record, in listed order, last-write-wins on a key collision. +- **`bind:`** — `{param: ref}` per-record parameters: a bare `step` binds the step's WHOLE result + record, `step[key]` the named ENTRY of its record, and `step.attr` the step op's live + `@output` (read after it ran — stochastic-correct). -A plain-mapping step with no op (`out: {from: a, merge_from: [b]}`) is a pure fan-in; `{}` is the identity (names the source). Cell lifetimes are **automatic** in both forms. Steps apply their ops through the engine's op-family dispatch, so bare library transforms sit in flow steps too. +A plain-mapping step with no op (`out: {from: a, merge_from: [b]}`) is a pure fan-in; `{}` is the +identity (names the source). `outputs:` picks the yielded step (default: the last). Steps apply +their ops through the engine's op-family dispatch, so bare library transforms sit in flow steps +too. -Two engines, one contract — **bidirectional conversion with execution parity**: +## Running one ```python -from recordstream import Stream, FlowGraph, to_ops, from_ops - -graph = FlowGraph.from_yaml("graph.yaml", source=src) # native named-step engine -stream = Stream.from_flow_yaml("graph.yaml", source=src) # same graph, LOWERED to the - # flat context-ops list (serial) -ops = to_ops(graph.steps, graph.output_step) # flow -> flat ops -flow2 = from_ops(ops) # flat ops -> flow (lifting) -``` - -`FlowGraph` is a `torch.utils.data.Dataset` like `Stream` (`__len__`/`__getitem__`/`.batch`/`.parallel` — parallel runs the lowered form on Stream's spawn pool, one worker implementation). A purely linear flow lowers to the bare op list — zero context ops. - -## Graph pipelines on a flat op list (Context ops) - -A branchy pipeline — fan-out, fan-in, a value computed on one branch feeding a parameter on another — runs on the **plain sequential `Stream` engine** via six *context ops* (`recordstream.ops.context`). The engine creates one per-record **`Context`** (a named-cell store, `recordstream.context`) around each record's trip through the op list; the context ops move data between the linear stream and those cells. Graph wiring never mutates the record's entries — a linear run's record stays byte-identical whether or not context threading exists. +from recordstream import FlowGraph, Stream +from recordstream.sources import HuggingFaceSource -| Op | Semantics | -|---|---| -| `Save(name)` | snapshot the stream record into a cell (pass-through) — the fork point | -| `Use(name, drop=False)` | stream := the cell's value; deep-copies unless `drop` frees the cell (move) | -| `Drop(names)` | free cells explicitly | -| `Apply(op, param, source, key="", drop=False)` | set `op.` from a cell (a record cell contributes its `key`-named entry, or the whole record when `key` is blank; a raw cell value verbatim), then apply `op` | -| `Capture(op, output, name)` | apply `op`, record its live `@output` into a cell (stochastic-correct) | -| `MergeFields(sources, keys, drop)` | fan-in: UNION the named cells' entries into the incoming record (listed order, last-write-wins; `keys` restricts the union) | +graph = FlowGraph.from_yaml("graph.yaml", source=HuggingFaceSource(path="mnist")) +for record in graph: + ... -```yaml -ops: - - !class:recordstream.ops.context.Save(name=fork) # fork the stream - - !class:albumentations.GaussNoise {p: 1.0} # branch A rides the stream - - !class:recordstream.ops.context.Save(name=branch_a) - - !class:recordstream.ops.context.Use(name=fork,drop=true) # branch B restarts from the fork - - !class:recordstream.ops.numpy.Threshold - low_level: 0.5 - - !class:recordstream.ops.context.MergeFields # fan-in - sources: [branch_a] - keys: [image] - drop: [branch_a] +graph.parallel(4) # spawn workers, one future per source record +len(graph); graph[3] # map-style access (unavailable if a step op is 1→N expanding) ``` -A straight sequence needs none of this — a bare `ops:` list stays exactly as before. Outside an engine (a hand-rolled loop), activate a Context explicitly: - -```python -from recordstream.context import Context, activate +`FlowGraph` is a map-style dataset like `Stream` (`__len__`/`__getitem__`/`.batch`/`.parallel`). +Both classes hold a step graph and call the same per-record kernel; a straight chain takes an +env-free fast path in that kernel, so `ops:` costs no more to run than it ever did. -with activate(Context()): - for op in ops: - record = op(record) -``` +A LINEAR graph converts to a `Stream` (`graph.to_stream()`) because an op list can express a +straight chain. A branchy one does not — and there is no lowering pass that would manufacture a +flat spelling for it. That pass existed until 2026-07-30 (`to_ops`/`from_ops` plus six context +ops that re-encoded dataflow as imperative mutations of a per-record cell store); it was deleted +because it destroyed the very structure every consumer — a compiler, a visual editor, a reader — +wants back. Rationale: [architecture.md](architecture.md). -Cells hold whole records (from `Save`) or raw values (from `Capture`); `Apply` reads a record cell's `key`-named entry (whole record when `key` is blank), `MergeFields` unions each cell's entries. Copy discipline: cells are stored by reference, deep-copied on read (`Use` without `drop`), moved on last read (`drop=True`). On a deliberate key collision at the fan-in, rename on the producing branch first (`RenameField`, `recordstream.ops.structure`). These ops are what a `flow:` graph document lowers to. Why the wiring plane is an ambient per-record store instead of extra record keys (and why `FlowGraph` doesn't use it) is recorded in [architecture.md](architecture.md#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17). +## Expanding (1→N) steps -> **Carrying a snapshot the context ops cannot?** Context cells are the wiring plane, but they deliberately raise across a `Parallel` boundary and never persist into a sink. For the two jobs cells cannot do — carrying a snapshot **across a `Parallel` boundary** and deliberately **persisting a snapshot into a sink** — copy the value under its own key with `CopyField` (`recordstream.ops.structure`); the snapshot then rides the record as a real entry. Everything else — fan-out, fan-in, cross-branch values — uses the context ops above. +A step whose op carries `EXPANDS = True` yields several records from one. The remaining subgraph +runs once per child over its own shallow copy of the step environment, depth-first, so sibling +order matches the nested-loop intuition. Such a pipeline is ITERABLE-ONLY: `__len__`/`__getitem__` +raise, because the expanded index map is unknowable up front. ## Reattach an ops-only YAML (`Stream.from_ops_yaml`) -A `{ops: [!class:…()]}` document — e.g. one exported by an external pipeline-authoring tool — can be attached to any source: +A `{ops: [!class:…()]}` document — e.g. one exported by a visual editor — can be attached to any +source: ```python from recordstream import Stream @@ -99,4 +104,8 @@ from recordstream.sources import HuggingFaceSource stream = Stream.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) ``` -The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: `Stream` also flows any still-deferred marker in place at engine-route entry (the same lazy-flow convention the composing ops use), which is what lets a bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. The manual equivalent is `Stream(source=src, ops=confluid.materialize(confluid.load("ops.yaml")["ops"]))`. +The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so +a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: +`Stream` also flows any still-deferred marker in place at engine-route entry, which is what lets a +bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. +`FlowGraph.from_ops_yaml` loads the same document as a linear step graph. diff --git a/docs/record-model.md b/docs/record-model.md index 0e1fd59..090f777 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -401,8 +401,9 @@ flow: `bind:` references have three shapes: a bare `step` binds the step's WHOLE result record, `step[key]` binds the named ENTRY of that step's record, and `step.attr` binds the step op's live -`@output`. Lowering (`to_ops`) compiles `merge_from` to the `MergeFields` context op and entry-binds -to `Apply(key=...)`; lifting (`from_ops`) round-trips both. See [graph.md](graph.md). +`@output`. All three are read by the engine directly from the step grammar — there is no lowering +to a flat op list (the pass that did that, and the six context ops it emitted, were deleted +2026-07-30). See [graph.md](graph.md). ## Storage — the record key-group layout diff --git a/pyproject.toml b/pyproject.toml index 3e796a4..328ce93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,6 @@ recordstream-ops-random-apply = "recordstream.ops.random_apply" # changes need an editable reinstall before StreamStudio/navigaitor discovery sees the module. recordstream-ops-configure = "recordstream.ops.configure" recordstream-ops-formula = "recordstream.ops.formula" -# Context ops (Save/Use/Drop/Apply/Capture/MergeFields) — the graph-plane building blocks lowered from flow: docs -recordstream-ops-context = "recordstream.ops.context" # The FlowGraph engine (flow: named-step documents + the flow<->ops converters) recordstream-flow = "recordstream.flow" # The queryable-metadata scan protocol + MetadataFilterSource view source diff --git a/recordstream/__init__.py b/recordstream/__init__.py index ba5dcc1..7df91ab 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -12,7 +12,6 @@ # --- shared infrastructure ----------------------------------------------------------------- from recordstream.batch import batch_metadata, batch_tensor, batch_values, multi_hot from recordstream.collate import collate, collate_records, get_collate, register_collate, registered_collates -from recordstream.context import Context from recordstream.core import ( FilterOp, JointStream, @@ -26,7 +25,7 @@ # --- the record data model + transforms + item codec ---------------------------------------- from recordstream.dispatch import dispatch, register_kernel, registered_kernels -from recordstream.flow import FlowGraph, from_ops, to_ops +from recordstream.flow import FlowGraph from recordstream.io import ( EncodedField, EncodedItem, @@ -110,7 +109,6 @@ "encode_record", "decode_record", # ---- shared infrastructure ---- - "Context", "Stream", "JointStream", "RecordSource", @@ -120,8 +118,6 @@ "register_op_family", "registered_op_families", "FlowGraph", - "from_ops", - "to_ops", "collate", "batch_metadata", "batch_tensor", diff --git a/recordstream/context.py b/recordstream/context.py deleted file mode 100644 index fa53838..0000000 --- a/recordstream/context.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Per-record named-cell store — the graph data plane for graph-shaped pipelines. - -A :class:`Context` holds named **cells** for exactly one record's trip through the op -list: branch snapshots (a cell holding a record dict), captured -``@output`` values, and per-record parameters. The context ops in -:mod:`recordstream.ops.context` (``Save`` / ``Use`` / ``Drop`` / ``Apply`` / ``Capture`` / -``Mix``) move data between the linear record stream and these cells, which is what lets -a plain sequential op list execute a fan-out/fan-in graph. - -The context ops route graph data through these per-record Context CELLS and never touch -the record's own fields — each typed item still owns its own metadata inside the record. -The Context is the *wiring* plane — engine-created, per record, empty again by the end of -a well-formed graph (every cell freed after its last read). Nothing here is -``@configurable``; a Context never appears in YAML. - -The engine (``Stream`` — and ``FlowGraph``, which manages its env directly) creates one -Context per source item and activates it around the op loop via a -:class:`contextvars.ContextVar`, so ops reach it inside ``__call__`` with no signature -change (:func:`current` / :func:`require`). A hand-rolled loop outside an engine opts in -explicitly:: - - with activate(Context()): - for op in ops: - record = op(record) -""" - -import contextvars -from contextlib import contextmanager -from typing import Any, Dict, Iterator, Optional, Tuple - -__all__ = ["Context", "activate", "current", "require"] - - -class Context: - """Named-cell store for one record's trip through a graph-shaped pipeline. - - Cells are stored and returned **by reference** — copy semantics are the reading - op's decision (``Use`` deep-copies unless it drops the cell). - """ - - __slots__ = ("_cells",) - - def __init__(self) -> None: - self._cells: Dict[str, Any] = {} - - def put(self, name: str, value: Any) -> None: - """Store ``value`` under ``name`` (overwrites an existing cell).""" - self._cells[name] = value - - def get(self, name: str) -> Any: - """Return the cell's value by reference; a missing cell is an actionable error.""" - if name not in self._cells: - live = ", ".join(sorted(self._cells)) or "" - raise KeyError( - f"Context has no cell {name!r} (live cells: {live}). " - f"A cell must be written (Save / Capture) before it is read, and is gone after " - f"a drop — check the op order and drop flags." - ) - return self._cells[name] - - def delete(self, name: str) -> None: - """Free the cell; deleting a missing cell is an error (it flags a liveness bug).""" - if name not in self._cells: - live = ", ".join(sorted(self._cells)) or "" - raise KeyError(f"Context cannot drop missing cell {name!r} (live cells: {live}).") - del self._cells[name] - - def live(self) -> Tuple[str, ...]: - """Names of all currently-held cells (sorted, for stable error messages/tests).""" - return tuple(sorted(self._cells)) - - def copy(self) -> "Context": - """Shallow copy — same cell values, independent cell *set* (for 1→N expansion children).""" - clone = Context() - clone._cells = dict(self._cells) - return clone - - def clear(self) -> None: - """Drop every cell.""" - self._cells.clear() - - def __contains__(self, name: object) -> bool: - return name in self._cells - - def __len__(self) -> int: - return len(self._cells) - - def __repr__(self) -> str: # pragma: no cover - debug aid - return f"Context(cells={sorted(self._cells)})" - - -_CURRENT: contextvars.ContextVar[Optional[Context]] = contextvars.ContextVar("recordstream_context", default=None) - - -def current() -> Optional[Context]: - """The active per-record :class:`Context`, or ``None`` outside an engine/`activate` block.""" - return _CURRENT.get() - - -def require(op_name: str = "context op") -> Context: - """The active Context, or an actionable error naming the op that needed it.""" - ctx = _CURRENT.get() - if ctx is None: - raise RuntimeError( - f"{op_name}: no active Context. Context ops need the per-record Context the engine " - f"creates — run the pipeline through Stream/FlowGraph, or wrap a manual loop in " - f"`with recordstream.context.activate(Context()):`." - ) - return ctx - - -@contextmanager -def activate(ctx: Context) -> Iterator[Context]: - """Activate ``ctx`` as the current per-record Context for the enclosed block.""" - token = _CURRENT.set(ctx) - try: - yield ctx - finally: - _CURRENT.reset(token) diff --git a/recordstream/core.py b/recordstream/core.py index e816088..cb03ec9 100644 --- a/recordstream/core.py +++ b/recordstream/core.py @@ -1,7 +1,7 @@ import concurrent.futures import multiprocessing from contextlib import nullcontext -from typing import Any, Callable, Collection, Dict, Iterable, Iterator, List, NamedTuple, Optional, Tuple, Union, cast +from typing import Any, Callable, Collection, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union, cast import torch.utils.data from confluid import configurable @@ -10,7 +10,6 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from recordstream.context import Context, activate from recordstream.items import NDArrayItem, Record, item_data, with_data logger = get_logger(__name__) @@ -146,7 +145,7 @@ def _apply_op(record: Record, op: Any) -> Optional[Record]: The single op-application chokepoint shared by the sequential, parallel (via :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths; composing - ops (``Pipeline`` / ``Parallel`` / ``Enable`` / ``RandomApply`` / the context ops) + ops (``Pipeline`` / ``Parallel`` / ``Enable`` / ``RandomApply`` / ``ConfigureOp``) route their inner ops through here so every op is applied identically. Each op family is invoked the way its library expects — no wrapper/adapter classes: the registered families (:func:`register_op_family`; built-ins ``albumentations`` / @@ -289,13 +288,6 @@ def __call__(self, record: Record) -> Optional[Record]: return {**record, self.key: new_value} -class _Carried(NamedTuple): - """A record travelling the streamed route together with its per-record Context.""" - - record: Any - ctx: Context - - def _expand(op: Any, record: Any) -> List[Any]: """Run a 1→N EXPANDING op and return its flattened children.""" raw = op(record) @@ -304,76 +296,36 @@ def _expand(op: Any, record: Any) -> List[Any]: return [child for child in raw if child is not None] +def linear_steps(ops: Sequence[Any]) -> Tuple[List[Any], str]: + """Compile a flat op list into the linear step graph the engine executes. + + A sequence IS a graph — every step reads the previous one — so an ``ops:`` list needs no + lifting to run on the graph kernel, just names. The names are positional (``s0``, ``s1``, + …) and never surface: nothing in an ``ops:`` document can reference a step, so they exist + only to key the step environment. Positional (not op-class) naming is deliberate — the + same op twice in a row is two distinct steps, which a name-keyed mapping would collapse. + + Returns ``(steps, output_step)``; an empty list yields ``([], "")``, the identity graph. + """ + from recordstream.flow import FlowStep + + steps = [FlowStep(name=f"s{i}", op=op, from_=None, bind={}, merge_from=()) for i, op in enumerate(ops)] + return cast(List[Any], steps), (steps[-1].name if steps else "") + + def _worker_task( record: Any, ops: List[Any], families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None ) -> Optional[Any]: """Single-result worker for STRICTLY 1→1 op lists (the ``Parallel`` op's contract). Kept for callers that need exactly one carrier back; expanding ops raise here — - route expanding pipelines through :func:`_worker_task_multi`. + route expanding pipelines through the iterating engine. """ - results = _worker_task_multi(record, ops, allow_expansion=False, families=families) - return results[0] if results else None - - -def _worker_task_multi( - record: Any, - ops: List[Any], - allow_expansion: bool = True, - families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, -) -> List[Any]: - """Top-level helper for multiprocess workers. Must be at top level for pickling. - - Runs one source :class:`Record` through the op list and returns EVERY resulting record — - usually one, zero when filtered, several when a 1→N EXPANDING op fired; each expansion - child continues through the REMAINING ops with a shallow copy of the per-record Context, - depth-first so sibling order matches the nested-loop intuition. - - Activates ONE fresh per-record :class:`~recordstream.context.Context` around the op loop so - context ops (``Save``/``Use``/``Apply``/``Capture``/``MergeFields``) can move data between - the linear stream and named cells — the executor itself stays a plain ``for op in ops`` - loop. Contexts are created inside the worker (spawn-safe: ops pickle, a Context never - crosses a process boundary). ``families`` carries the parent process's non-builtin op - families into a spawn worker (:func:`_sync_op_families` — matchers/invokers pickle by - reference); in-process callers omit it. - """ - from collections import deque + from recordstream.flow import run_steps _sync_op_families(families) - pending: "deque[Tuple[Any, Context, int]]" = deque([(record, Context(), 0)]) - out: List[Any] = [] - while pending: - current, ctx, start = pending.popleft() - alive = True - with activate(ctx): - i = start - while i < len(ops): - op = ops[i] - i += 1 - if _op_expands(op): - if not allow_expansion: - raise TypeError( - f"op {type(op).__name__!r} is a 1→N expanding op, which this strictly " - "1→1 route cannot carry — run it through the Stream iteration paths." - ) - children = _expand(op, current) - if not children: - alive = False - break - # Depth-first: the first child continues inline; its siblings go to the - # FRONT of the queue (reversed, so sibling order is preserved). - for child in reversed(children[1:]): - pending.appendleft((child, ctx.copy(), i)) - current = children[0] - continue - result = _apply_op(current, op) - if result is None: - alive = False - break - current = result - if alive and current is not None: - out.append(current) - return out + steps, outputs = linear_steps(ops) + return run_steps(record, steps, outputs) @configurable(category="engine") @@ -475,13 +427,6 @@ def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "St ops = list(_confluid_materialize(raw_ops)) return cls(source=source, ops=ops) - @classmethod - def from_flow_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Stream": - """Attach a ``{flow: {...}}`` graph document to ``source``, LOWERED to the serial form.""" - from recordstream.flow import flow_yaml_to_stream - - return cast("Stream", flow_yaml_to_stream(path, source=source)) - @property def _expands(self) -> bool: """True when any (materialized) op is a 1→N expanding op — the pipeline is then iterable-only.""" @@ -530,14 +475,13 @@ def __getitem__(self, index: int) -> Any: "iterator; give the source a __len__ (then Stream caches on first access) or wrap " "it in ``list(...)`` before handing it to Stream." ) + from recordstream.flow import run_steps + _check_ops_materialized(self.ops) - record: Any = raw - with activate(Context()): - for op in self.ops: - result = _apply_op(record, op) - if result is None: - raise IndexError(f"Record {index} filtered out by {op}") - record = result + steps, outputs = linear_steps(self.ops) + record = run_steps(raw, steps, outputs) + if record is None: + raise IndexError(f"Record {index} filtered out by the pipeline") return cast(Record, record) def to_sink(self, sink: Any) -> None: @@ -601,68 +545,52 @@ def __iter__(self) -> Iterator[Any]: yield from it def _iter_streamed(self) -> Iterator[Record]: - """Mixed per-record / stream-level op chain (a stream-level op exposes ``.stream``).""" + """Mixed per-record / stream-level op chain (a stream-level op exposes ``.stream``). + + A stream-level op (``Parallel``) sees the WHOLE stream rather than one record, so it + cannot be a step in the per-record graph — the chain is split at each such op and the + per-record runs between them go through the ordinary kernel. Records travel as plain + records: the per-record Context they used to be paired with is gone, and with it the + "cells cannot cross a stream-op boundary" restriction that pairing imposed. + """ + from recordstream.flow import run_steps_multi + source = self._guard_live_source() if source is None: return _check_ops_materialized(self.ops) - def to_carried() -> Iterator[Optional[_Carried]]: - for item in source: - yield _Carried(item, Context()) - - def per_record(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[_Carried]]: - expands = _op_expands(op) - for c in stream: - if c is None: - continue - with activate(c.ctx): - if expands: - children = _expand(op, c.record) - else: - s = _apply_op(c.record, op) - if expands: - for j, child in enumerate(children): - yield _Carried(child, c.ctx if j == 0 else c.ctx.copy()) - else: - yield None if s is None else _Carried(s, c.ctx) - - def strip(stream: Iterator[Optional[_Carried]], op: Any) -> Iterator[Optional[Record]]: - for c in stream: - if c is None: - yield None + def per_record(stream: Iterator[Optional[Record]], op: Any) -> Iterator[Optional[Record]]: + steps, outputs = linear_steps([op]) + for record in stream: + if record is None: continue - if c.ctx.live(): - raise RuntimeError( - f"Stream: context cells {c.ctx.live()!r} are still live at the stream-level op " - f"{type(op).__name__!r}. Context cells cannot cross a stream-op boundary " - f"(e.g. Parallel) — drop them before it, or move the whole graph inside it." - ) - yield c.record - - def wrap(stream: Iterator[Optional[Record]]) -> Iterator[Optional[_Carried]]: - for s in stream: - yield None if s is None else _Carried(s, Context()) - - carried: Iterator[Optional[_Carried]] = to_carried() + yield from run_steps_multi(record, steps, outputs) + + carried: Iterator[Optional[Record]] = iter(source) for op in self.ops: if hasattr(op, "stream") and callable(op.stream): - carried = wrap(op.stream(strip(carried, op))) + carried = op.stream(carried) else: carried = per_record(carried, op) - for c in carried: - if c is not None: - yield c.record + for record in carried: + if record is not None: + yield record def _iter_sequential(self) -> Iterator[Record]: - """Standard single-threaded execution.""" + """Standard single-threaded execution — the flat op list run as a linear step graph.""" + from recordstream.flow import _result_readers, run_steps_multi + source = self._guard_live_source() if source is None: return _check_ops_materialized(self.ops) + # Compile + analyse ONCE per iteration, never per record (see run_steps_multi). + steps, outputs = linear_steps(self.ops) + readers = _result_readers(steps, outputs) for item in source: - yield from _worker_task_multi(item, self.ops) + yield from run_steps_multi(item, steps, outputs, readers) def _iter_parallel(self) -> Iterator[Record]: """Multiprocess execution engine.""" @@ -674,11 +602,14 @@ def _iter_parallel(self) -> Iterator[Record]: # We use 'spawn' to be consistent with Loggair and prevent CI deadlocks ctx = multiprocessing.get_context("spawn") + from recordstream.flow import _graph_worker_task + + steps, outputs = linear_steps(self.ops) with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: futures = [] extra_families = _extra_op_families() # ship third-party op families to the workers for item in source: - futures.append(executor.submit(_worker_task_multi, item, self.ops, True, extra_families)) + futures.append(executor.submit(_graph_worker_task, item, steps, outputs, extra_families)) for future in futures: yield from future.result() @@ -704,6 +635,10 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: #: :class:`Stream` is), or any iterable of records (a recordstream source, a plain list of #: record dicts). Consumers annotate their slots ``Optional[Lazy[RecordSource]]`` — ``Lazy`` #: because they flow the slot themselves at run time. +#: What a wired dataset slot may hold: anything MAP-STYLE (``__len__`` + ``__getitem__`` — +#: which a :class:`Stream` is) or any iterable of records. Expressed structurally rather than +#: as ``torch.utils.data.Dataset`` so the engine stays framework-free; torch's ``DataLoader`` +#: is itself duck-typed and consumes either. RecordSource = Union[torch.utils.data.Dataset[Any], Iterable[Record]] diff --git a/recordstream/flow.py b/recordstream/flow.py index b6e6e89..fd74ebd 100644 --- a/recordstream/flow.py +++ b/recordstream/flow.py @@ -1,21 +1,20 @@ -"""The ``flow:`` document, the :class:`FlowGraph` engine, and the flow⇄ops converters. +"""The ``flow:`` document and the :class:`FlowGraph` engine. -A **flow document** is the readable, named-step form of a graph-shaped pipeline: a -mapping of ``step-name → op``, where a step's name is also the name later steps use to -reference its result. It is the authoring format (humans and graph exporters -write it); the flat context-ops form (:mod:`recordstream.ops.context`) is the serial -execution format the plain :class:`~recordstream.core.Stream` engine runs. The two convert -**bidirectionally**: :func:`to_ops` lowers a flow into a flat op list, :func:`from_ops` -lifts a flat op list back — with execution parity in both directions. +A **flow document** is the named-step form of a pipeline: a mapping of ``step-name → op``, +where a step's name is how later steps reference its result. It is the spelling to reach for +when a pipeline BRANCHES; a straight chain is written as a plain ``ops:`` list, which the +engine compiles to positional steps (``recordstream.core.linear_steps``). Both parse to the +same :class:`FlowStep` list and run through the same per-record kernel — there is ONE +execution model, and no lowering pass between the two forms (the flow⇄ops converters and the +per-record context ops they emitted were deleted 2026-07-30; see ``docs/architecture.md`` §3). .. code-block:: yaml flow: - spec: !class:waivefront.SpectrogramOp() # input: the source record - rescaled: !class:recordstream.ops.numpy.RescaleOp() # input: previous step - masked: !class:waivefront.SegmentOp() {from: spec} # 2nd reader of spec = fan-out - thresh: !class:recordstream.ops.formula.FormulaOp(formula="a*0.5") {from: masked} - out: {from: masked, merge_from: [rescaled]} # typed fan-in (no op) + spec: !class:mypkg.MakeSpectrogram() # input: the source record + rescaled: !class:recordstream.ops.numpy.Threshold() # input: previous step + masked: !class:mypkg.Segment() {from: spec} # 2nd reader of spec = fan-out + out: {from: masked, merge_from: [rescaled]} # fan-in (no op) outputs: out Step grammar (the three RESERVED step keys, stripped before the op is built): @@ -23,20 +22,19 @@ - ``from:`` — the step supplying this step's input record. Omitted = the previous step (the first step reads the source record). Must name an EARLIER step: document order is the schedule, so forward references are errors and cycles are inexpressible. -- ``merge_from:`` — fan-in: UNION another step's record entries into this step's incoming - record before the op runs (the ``MergeFields`` slot semantics — last-write-wins on a - key collision, in listed order). +- ``merge_from:`` — fan-in: UNION the named steps' record entries into this step's incoming + record before the op runs (listed order, last-write-wins on a key collision). - ``bind:`` — ``{param: ref}`` per-record parameters: ``ref`` is a step name (the step's - whole result record, ``step[key]`` for a named entry, or the raw value) or - ``step.attr`` (the step op's live ``@output`` after it ran — lowered through ``Capture``). + whole result record), ``step[key]`` (one entry of it), or ``step.attr`` (the step op's + live ``@output`` after it ran — read through wrapper chains by :func:`_read_output`). A step may be a plain mapping with no op (``out: {from: a, merge_from: [b]}``) — a pure fan-in/identity step; ``{}`` is the identity (used to give the source a referable name). ``outputs:`` names the step whose result the pipeline yields (default: the last step). -Cell-lifetime management is AUTOMATIC in both forms: :class:`FlowGraph` frees each step -result after its last reader, and :func:`to_ops` computes the same liveness into -``drop`` flags on the emitted context ops. +Step results are freed automatically: :func:`_result_readers` counts each step's readers +slot-granularly and the kernel drops a result after its last one. A straight chain needs no +environment at all — :func:`is_linear` routes it to :func:`_run_linear`. """ import concurrent.futures @@ -51,16 +49,42 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from recordstream.core import OpInvoker, OpMatcher, _apply_op, _extra_op_families, _sync_op_families +from recordstream.core import ( + OpInvoker, + OpMatcher, + _apply_op, + _expand, + _extra_op_families, + _op_expands, + _sync_op_families, +) from recordstream.items import Record -from recordstream.ops.context import _MISSING, Apply, Capture, Drop, MergeFields, Save, Use, _read_output logger = get_logger(__name__) RESERVED_STEP_KEYS = ("from", "merge_from", "bind") """Step-grammar keys stripped from a step mapping before the op is constructed.""" -__all__ = ["FlowGraph", "FlowStep", "from_ops", "parse_flow", "to_ops", "RESERVED_STEP_KEYS"] +__all__ = ["FlowGraph", "FlowStep", "parse_flow", "run_steps", "run_steps_multi", "RESERVED_STEP_KEYS"] + +_MISSING = object() + + +def _read_output(op: Any, name: str) -> Any: + """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. + + Backs the ``bind: {param: "step.attr"}`` grammar — the step op's live ``@output`` after it + ran. The wrapper walk matters because a step op may be a composing op (``ConfigureOp`` + wrapping the real op in ``target``). Returns ``_MISSING`` when absent. + """ + cur, seen = op, set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + value = getattr(cur, name, _MISSING) + if value is not _MISSING: + return value + cur = getattr(cur, "target", None) or getattr(cur, "op", None) + return _MISSING class FlowStep(NamedTuple): @@ -226,7 +250,12 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T following step can ride the linear stream. Slots: ``"in"`` (input), ``"merge"``, ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A ``bind`` step-result reference counts; an ``@output`` (``step.attr``) reference does NOT. + + NO steps is the identity graph (a bare ``ops: []``): nothing is produced, so nothing is + read — and there is no output step to account for. """ + if not steps: + return {} readers: Dict[str, List[Tuple[int, str]]] = {s.name: [] for s in steps} for i, step in enumerate(steps): implicit = steps[i - 1].name if i > 0 else None @@ -248,25 +277,106 @@ def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[T # --------------------------------------------------------------------------- -def run_steps( +def run_steps_multi( seed: Any, steps: Sequence[FlowStep], outputs: str, readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, -) -> Optional[Record]: - """Run ONE record through the parsed steps; ``None`` = filtered (an op returned None). +) -> List[Record]: + """Run ONE source record through the parsed steps, returning EVERY resulting record. The engine's per-record kernel, module-level so a spawn worker can pickle a reference to - it. ``readers`` is the slot-granular reader accounting from :func:`_result_readers`; it + it. Usually one record back, zero when a step filtered (an op returned ``None``), several + when a 1→N EXPANDING step fired. + + ``readers`` is the slot-granular reader accounting from :func:`_result_readers`; it depends only on ``(steps, outputs)``, so a caller running many records MUST compute it once and pass it in — recomputing per record is an O(steps²) tax on every record (it was measured at 3.4 µs/record on a 23-step pipeline, roughly half the graph engine's total overhead over a flat op list). + + EXPANSION semantics: a step whose op carries ``EXPANDS`` yields N children, and the + REMAINING subgraph runs once per child over its own shallow copy of the step environment + (independent name→result maps, shared values — the graph twin of ``Context.copy()``). + Traversal is DEPTH-FIRST, so sibling order matches the nested-loop intuition and the flat + engine's documented order. An empty expansion or a ``None`` child just drops that branch. + + NO steps is the IDENTITY graph — the seed comes straight back. That is what makes a bare + ``Stream(source=..., ops=[])`` yield its source unchanged once the flat engine routes + through this kernel. """ + if not steps: + return [] if seed is None else [cast(Record, seed)] + out: List[Record] = [] + if is_linear(steps, outputs): + _run_linear(seed, steps, 0, out) + return out if readers is None: readers = _result_readers(steps, outputs) - env: Dict[str, Any] = {} - remaining = {name: len(idx) for name, idx in readers.items()} + base_remaining = {name: len(idx) for name, idx in readers.items()} + _run_from(0, seed, steps, outputs, {}, base_remaining, None, out) + return out + + +def is_linear(steps: Sequence[FlowStep], outputs: str) -> bool: + """True when the graph is a straight chain — no named reference reaches back. + + Every step reads the one before it, nothing binds, nothing merges, and the yielded step + is the last one. Such a graph needs no step ENVIRONMENT at all: the record can ride a + local variable exactly as it did in the flat op loop, which is what keeps an ``ops:`` + list as cheap to run as before it became a graph (the env bookkeeping measured ~33% + of engine overhead on a 23-step chain). + """ + if not steps or outputs != steps[-1].name: + return False + return all(s.from_ is None and not s.bind and not s.merge_from for s in steps) + + +def _run_linear( + record: Any, + steps: Sequence[FlowStep], + index: int, + out: List[Record], +) -> None: + """Run a straight chain from ``steps[index:]`` — the env-free path (see :func:`is_linear`). + + Same expansion contract as :func:`_run_from`: a 1→N step forks the remaining chain, + depth-first, so sibling order matches the nested-loop intuition. + """ + for i in range(index, len(steps)): + op = steps[i].op + if op is None: + continue + if _op_expands(op): + for child in _expand(op, record): + _run_linear(child, steps, i + 1, out) + return + result = _apply_op(record, op) + if result is None: + return + record = result + out.append(cast(Record, record)) + + +def _run_from( + index: int, + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + env: Dict[str, Any], + remaining: Dict[str, int], + prev: Optional[str], + out: List[Record], +) -> None: + """Run ``steps[index:]`` over ``env``, appending every surviving result to ``out``. + + Recurses ONCE PER CHILD at an expanding step (recursion depth = the number of expanding + steps on the path, not the record count), which is what gives depth-first sibling order + for free. + + Each expansion branch gets its OWN shallow copy of the step environment (independent + name→result maps, shared values), so siblings cannot see each other's results. + """ def read_result(name: str, *, copy: bool) -> Any: value = env[name] @@ -277,8 +387,8 @@ def read_result(name: str, *, copy: bool) -> Any: value = deepcopy(value) return value - prev: Optional[str] = None - for step in steps: + for i in range(index, len(steps)): + step = steps[i] # 1. the input record (implicit stream reads move; explicit fan-out reads copy) if step.from_ is not None: record = read_result(step.from_, copy=True) @@ -308,12 +418,6 @@ def read_result(name: str, *, copy: bool) -> Any: # 3. per-record parameter binds if step.op is not None: op = step.op - if getattr(op, "EXPANDS", False): - raise NotImplementedError( - f"flow step {step.name!r}: {type(op).__name__!r} is a 1→N expanding op — " - "FlowGraph steps are strictly 1→1 (a named-step env has one result per step). " - "Run expanding pipelines through the Stream engine (iterable-only)." - ) for param, ref in step.bind.items(): parsed = _split_bind_ref(ref) if parsed.attr is not None: @@ -330,15 +434,47 @@ def read_result(name: str, *, copy: bool) -> Any: # "step[key]" = the named entry; bare "step" = the whole record. value = value[parsed.key] setattr(op, param, value) + + # 4. a 1→N step forks the REMAINING subgraph, one branch per child + if _op_expands(op): + for child in _expand(op, record): + child_env = dict(env) + child_env[step.name] = child + _run_from(i + 1, seed, steps, outputs, child_env, dict(remaining), step.name, out) + return + result = _apply_op(record, op) if result is None: - return None + return record = result env[step.name] = record prev = step.name - return cast(Optional[Record], env.get(outputs)) if outputs in env else None + if outputs in env: + out.append(cast(Record, env[outputs])) + + +def run_steps( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, +) -> Optional[Record]: + """Strictly 1→1 twin of :func:`run_steps_multi` — one result back, or ``None``. + + For callers that need exactly one carrier (indexing, a single-record probe). An expanding + step RAISES here rather than silently dropping its siblings; route those through + :func:`run_steps_multi`. + """ + for step in steps: + if step.op is not None and _op_expands(step.op): + raise TypeError( + f"flow step {step.name!r}: {type(step.op).__name__!r} is a 1→N expanding op, which " + "this strictly 1→1 route cannot carry — iterate the graph instead." + ) + results = run_steps_multi(seed, steps, outputs, readers) + return results[0] if results else None def _graph_worker_task( @@ -346,27 +482,27 @@ def _graph_worker_task( steps: Sequence[FlowStep], outputs: str, families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, -) -> Optional[Record]: +) -> List[Record]: """Spawn-worker entry point: re-register third-party op families, then run one record. Module-level for pickling (the same constraint :func:`recordstream.core._worker_task_multi` - obeys). ``readers`` is deliberately NOT passed across the boundary — it is cheap to derive - once per worker call relative to the process hop, and shipping it would add a second - pickled structure that must stay in sync with ``steps``. + obeys). Returns a LIST because an expanding step makes one seed yield several records. + ``readers`` is deliberately NOT passed across the boundary — it is cheap to derive once per + worker call relative to the process hop, and shipping it would add a second pickled + structure that must stay in sync with ``steps``. """ _sync_op_families(families) - return run_steps(seed, steps, outputs) + return run_steps_multi(seed, steps, outputs) @configurable(category="engine") class FlowGraph(torch.utils.data.Dataset[Record]): """Named-step graph engine — executes a ``flow:`` document natively. - The readable twin of :class:`~recordstream.core.Stream`: steps run in document order over - a per-record environment of named results, with fan-out isolation (copy-on-read, move - on last read) and automatic cell lifetimes. Any FlowGraph converts to a flat op list - for the serial engine (:func:`to_ops`) and back (:func:`from_ops`) — execution parity - between the two is a pinned contract. + The named-step twin of :class:`~recordstream.core.Stream`, over the SAME kernel: steps run + in document order against a per-record environment of named results, with fan-out isolation + (copy-on-read, move on last read) and automatic result lifetimes. A LINEAR graph converts + to a Stream (:meth:`to_stream`); a branchy one has no flat spelling by design. Args: source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. @@ -389,6 +525,7 @@ def __init__( self._chunk_size = int(chunk_size) self._workers = 1 self._parsed: Optional[Tuple[List[FlowStep], str]] = None + self._readers: Optional[Dict[str, List[Tuple[int, str]]]] = None # -- parsing ----------------------------------------------------------- @@ -416,6 +553,13 @@ def _ensure_parsed(self) -> Tuple[List[FlowStep], str]: self._parsed = parse_flow(cast(Dict[str, Any], self.flow), self.outputs) return self._parsed + def _ensure_readers(self) -> Dict[str, List[Tuple[int, str]]]: + """The reader accounting, computed ONCE per graph (see :func:`run_steps`).""" + if self._readers is None: + steps, outputs = self._ensure_parsed() + self._readers = _result_readers(steps, outputs) + return self._readers + @classmethod def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": """Build a FlowGraph from a ``{flow: {...}, outputs: ...}`` YAML document (or inline string). @@ -430,95 +574,24 @@ def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": @classmethod def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": - """Lift a flat ``{ops: [...]}`` YAML document into a FlowGraph (via :func:`from_ops`).""" - from recordstream.core import Stream + """Load a flat ``{ops: [...]}`` YAML document as a LINEAR step graph. + + No lifting is involved: a sequence IS a graph, so the op list becomes positional + steps (``recordstream.core.linear_steps``) — the same compilation a ``Stream``'s + ``ops`` list goes through, because they are the same thing spelled two ways. + """ + from recordstream.core import Stream, linear_steps stream = Stream.from_ops_yaml(path, source=source) - flow_doc, outputs = from_ops(stream.ops) - return cls(source=source, flow=flow_doc, outputs=outputs) + steps, outputs = linear_steps(stream.ops) + return cls(source=source, flow=steps, outputs=outputs) # -- execution --------------------------------------------------------- def _run(self, seed: Any) -> Optional[Any]: """Run one record through the steps; ``None`` = filtered (an op returned None).""" steps, outputs = self._ensure_parsed() - readers = _result_readers(steps, outputs) - env: Dict[str, Any] = {} - remaining = {name: len(idx) for name, idx in readers.items()} - - def read_result(name: str, *, copy: bool) -> Any: - value = env[name] - remaining[name] -= 1 - if remaining[name] <= 0: - del env[name] - elif copy: - from copy import deepcopy - - value = deepcopy(value) - return value - - prev: Optional[str] = None - for step in steps: - # 1. the input record (implicit stream reads move; explicit fan-out reads copy) - if step.from_ is not None: - record = read_result(step.from_, copy=True) - elif prev is not None: - record = read_result(prev, copy=False) - else: - record = seed - - # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) - if step.merge_from: - if not isinstance(record, dict): - raise TypeError( - f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " - f"{type(record).__name__} — expected a record dict." - ) - merged = dict(record) - for ref in step.merge_from: - value = read_result(ref, copy=True) - if not isinstance(value, dict): - raise TypeError( - f"flow step {step.name!r}: merge_from step {ref!r} holds " - f"{type(value).__name__}, expected a record" - ) - merged.update(value) - record = merged - - # 3. per-record parameter binds - if step.op is not None: - op = step.op - if getattr(op, "EXPANDS", False): - raise NotImplementedError( - f"flow step {step.name!r}: {type(op).__name__!r} is a 1→N expanding op — " - "FlowGraph steps are strictly 1→1 (a named-step env has one result per step). " - "Run expanding pipelines through the Stream engine (iterable-only)." - ) - for param, ref in step.bind.items(): - parsed = _split_bind_ref(ref) - if parsed.attr is not None: - producer = next(s for s in steps if s.name == parsed.step) - value = _read_output(producer.op, parsed.attr) - if value is _MISSING: - raise AttributeError( - f"flow step {step.name!r}: bind {param}={ref!r} — " - f"step {parsed.step!r} op has no @output attribute {parsed.attr!r}" - ) - else: - value = read_result(parsed.step, copy=False) - if isinstance(value, dict) and parsed.key: - # "step[key]" = the named entry; bare "step" = the whole record. - value = value[parsed.key] - setattr(op, param, value) - result = _apply_op(record, op) - if result is None: - return None - record = result - - env[step.name] = record - prev = step.name - - return cast(Optional[Record], env.get(outputs)) if outputs in env else None + return run_steps(seed, steps, outputs, self._ensure_readers()) def __iter__(self) -> Iterator[Any]: if self.source is None: @@ -541,22 +614,48 @@ def _iter_records(self) -> Iterator[Record]: yield from self._iter_parallel() return assert self.source is not None + steps, outputs = self._ensure_parsed() + readers = self._ensure_readers() for item in self.source: - result = self._run(item) - if result is not None: - yield result + yield from run_steps_multi(item, steps, outputs, readers) def _iter_parallel(self) -> Iterator[Record]: - """Multiprocess execution — delegates to the serial engine over the LOWERED op list.""" - from recordstream.core import Stream + """Multiprocess execution — the graph's OWN spawn pool, one future per source record. + Mirrors :meth:`recordstream.core.Stream._iter_parallel`: ``spawn`` (consistent with + Loggair, no CI deadlocks), third-party op families shipped to the workers by + reference. The steps pickle because their ops already must; the source never crosses + the boundary (only the seed record does). + """ assert self.source is not None - stream = Stream(source=self.source, ops=to_ops(self.steps, self.output_step)).parallel(self._workers) - yield from stream + steps, outputs = self._ensure_parsed() + ctx = multiprocessing.get_context("spawn") + + with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: + extra_families = _extra_op_families() + futures = [ + executor.submit(_graph_worker_task, item, steps, outputs, extra_families) for item in self.source + ] + for future in futures: + yield from future.result() + + @property + def _expands(self) -> bool: + """True when any step op is 1→N — the length/index map is then unknowable.""" + return any(step.op is not None and _op_expands(step.op) for step in self._ensure_parsed()[0]) + + def _guard_not_expanding(self, operation: str) -> None: + if self._expands: + raise TypeError( + f"FlowGraph.{operation} is unavailable: a step op is 1→N EXPANDING, so the " + "expanded length/index map is unknowable. Iterate the graph, wrap it in a torch " + "IterableDataset, window at the SOURCE for random access, or call .collect()." + ) def __len__(self) -> int: from collections.abc import Sized + self._guard_not_expanding("__len__") if isinstance(self.source, Sized): return len(self.source) return 0 @@ -564,6 +663,7 @@ def __len__(self) -> int: def __getitem__(self, index: int) -> Any: if self.source is None: raise TypeError("FlowGraph source is None — cannot index.") + self._guard_not_expanding("__getitem__") if hasattr(self.source, "__getitem__"): raw = self.source[index] else: @@ -577,7 +677,7 @@ def __getitem__(self, index: int) -> Any: return result def parallel(self, workers: int = 4) -> "FlowGraph": - """Enable multiprocess execution (spawn, via the lowered serial form).""" + """Enable multiprocess execution on the graph's own spawn pool.""" self._workers = workers return self @@ -591,287 +691,19 @@ def collect(self) -> List[Any]: return list(self) def to_stream(self) -> Any: - """The serial-engine twin: a Stream running the LOWERED flat op list (same results).""" - from recordstream.core import Stream - - return Stream(source=self.source, ops=to_ops(self.steps, self.output_step)) - - -# --------------------------------------------------------------------------- -# Lowering: flow -> flat context-ops list -# --------------------------------------------------------------------------- - - -def to_ops(steps: Union[Sequence[FlowStep], Dict[str, Any]], outputs: str = "") -> List[Any]: - """Lower a flow (parsed steps or a raw flow mapping) into a flat context-ops list. - - The result runs on the plain serial :class:`~recordstream.core.Stream` engine and is the - serialization form a graph exporter's serial mode emits. Cell names are the step - names (deterministic, diffable); liveness is compiled into ``drop`` flags so a - well-formed graph leaves the Context empty. A purely linear flow lowers to the bare - op list — zero context ops. - """ - if isinstance(steps, dict): - parsed, outputs = parse_flow(steps, outputs) - else: - parsed = list(steps) - outputs = outputs or (parsed[-1].name if parsed else "") - - readers = _result_readers(parsed, outputs) - needs_cell: Dict[str, bool] = {} - cell_reads_left: Dict[str, int] = {} - for i, step in enumerate(parsed): - consumers = list(readers[step.name]) - stream_read: Optional[Tuple[int, str]] = None - if i + 1 < len(parsed) and (parsed[i + 1].from_ or step.name) == step.name: - stream_read = (i + 1, "in") - elif i == len(parsed) - 1: - stream_read = (len(parsed), "out") - cell_reads = [c for c in consumers if c != stream_read] - needs_cell[step.name] = bool(cell_reads) - cell_reads_left[step.name] = len(cell_reads) - - ops: List[Any] = [] - attr_cells: Dict[str, str] = {} # "step.attr" -> cell name - - # Pre-scan @output refs: the producer op must be wrapped in Capture at ITS step. - attr_refs: Dict[str, List[str]] = {} - for step in parsed: - for ref in step.bind.values(): - parsed_ref = _split_bind_ref(ref) - if parsed_ref.attr is not None: - attr_refs.setdefault(parsed_ref.step, []) - if parsed_ref.attr not in attr_refs[parsed_ref.step]: - attr_refs[parsed_ref.step].append(parsed_ref.attr) - - def take_cell(name: str) -> Tuple[str, bool]: - """(cell, is_last_read) — decrement the read counter.""" - cell_reads_left[name] -= 1 - return name, cell_reads_left[name] <= 0 - - prev_name: Optional[str] = None - for i, step in enumerate(parsed): - # 1. input slot (explicit from == previous step consumes the stream — no Use) - if step.from_ is not None and step.from_ != prev_name: - cell, last = take_cell(step.from_) - ops.append(Use(name=cell, drop=last)) - - # 2. fan-in slot — typed union (MergeFields) - if step.merge_from: - merge_drops: List[str] = [] - merge_cells: List[str] = [] - for ref in step.merge_from: - cell, last = take_cell(ref) - merge_cells.append(cell) - if last: - merge_drops.append(cell) - ops.append(MergeFields(sources=merge_cells, drop=merge_drops)) - - # 3. the op, wrapped for binds (Apply) and @output captures (Capture) - emitted: Optional[Any] = step.op - if emitted is not None: - for param, ref in step.bind.items(): - parsed_ref = _split_bind_ref(ref) - if parsed_ref.attr is not None: - cell = attr_cells[ref] - cell_reads_left.setdefault(cell, 1) - cell_reads_left[cell] -= 1 - emitted = Apply(op=emitted, param=param, source=cell, drop=cell_reads_left[cell] <= 0) - else: - cell, last = take_cell(parsed_ref.step) - emitted = Apply(op=emitted, param=param, source=cell, key=parsed_ref.key or "", drop=last) - captures = attr_refs.get(step.name, []) - if captures: - for attr in captures: - cell = f"{step.name}.{attr}" - attr_cells[cell] = cell - cell_reads_left[cell] = sum( - 1 for s in parsed for r in s.bind.values() if r == f"{step.name}.{attr}" - ) - if len(captures) == 1: - emitted = Capture(op=emitted, output=captures[0], name=f"{step.name}.{captures[0]}") - else: - emitted = Capture(op=emitted, captures={a: f"{step.name}.{a}" for a in captures}) - ops.append(emitted) - elif step.merge_from is None and step.from_ is None and i == 0: - # identity first step ({}: names the source) — nothing to run - pass - - # 4. persist the result for non-stream readers - if needs_cell[step.name]: - ops.append(Save(name=step.name)) - - prev_name = step.name - - # 5. the output: if it is not the final stream, fetch it. - if parsed and outputs != parsed[-1].name: - cell, last = take_cell(outputs) - ops.append(Use(name=cell, drop=last)) - - # 6. safety net: any cells the liveness pass left alive get an explicit Drop. - leftovers = [name for name, left in cell_reads_left.items() if left > 0 and needs_cell.get(name, True)] - if leftovers: - ops.append(Drop(names=sorted(leftovers))) + """The ``Stream`` twin of a LINEAR graph — same source, same ops, same engine. - return ops - - -# --------------------------------------------------------------------------- -# Lifting: flat context-ops list -> flow -# --------------------------------------------------------------------------- - - -_CONTEXT_OP_CLASSES = (Save, Use, Drop, Apply, Capture, MergeFields) - - -def _ctx_view(raw: Any) -> Optional[type]: - """The context-op class ``raw`` represents, live instance OR confluid marker; else None.""" - if isinstance(raw, _ConfluidFluid): - target = getattr(raw, "target", None) - return target if isinstance(target, type) and target in _CONTEXT_OP_CLASSES else None - return type(raw) if isinstance(raw, _CONTEXT_OP_CLASSES) else None - - -def _ctx_field(raw: Any, name: str, default: Any = None) -> Any: - """Read a context-op field off a live instance OR a marker's kwargs.""" - if isinstance(raw, _ConfluidFluid): - return raw.kwargs.get(name, default) - return getattr(raw, name, default) - - -def _capture_items(raw: Any) -> Dict[str, str]: - """A Capture's ``{output_attr: cell}`` map, live instance or marker.""" - if not isinstance(raw, _ConfluidFluid): - return cast(Capture, raw)._items() - items = dict(raw.kwargs.get("captures") or {}) - output = str(raw.kwargs.get("output", "") or "") - if output: - items.setdefault(output, str(raw.kwargs.get("name", "") or "") or output) - return items - - -def _auto_name(op: Any, index: int, taken: Dict[str, int]) -> str: - if op is None: - base = "step" - elif isinstance(op, _ConfluidFluid): - target = getattr(op, "target", None) - base = getattr(target, "__name__", str(target)).lower() - else: - base = type(op).__name__.lower() - taken[base] = taken.get(base, 0) + 1 - return base if taken[base] == 1 else f"{base}_{taken[base]}" - - -def from_ops(ops: Sequence[Any], outputs: str = "") -> Tuple[Dict[str, Any], str]: - """Lift a flat op list into a ``(flow_mapping, outputs)`` pair. - - Context ops are absorbed into step grammar: ``Save`` names the preceding step (or an - identity first step for a source fork), ``Use`` starts a branch (``from:``), - ``MergeFields`` becomes ``merge_from`` on the following step (or a pure fan-in step), - ``Apply``/``Capture`` unwrap into ``bind:`` references, and ``Drop`` vanishes (liveness - is recomputed on lowering). A plain linear list lifts to a linear flow with - auto-generated step names. The result round-trips. - - Accepts LIVE ops or confluid ``Instance``/``Class`` MARKERS interchangeably. - """ - flow_map: Dict[str, Dict[str, Any]] = {} - taken: Dict[str, int] = {} - prev_name: Optional[str] = None - capture_cells: Dict[str, str] = {} # cell -> "step.attr" bind ref - pending: Dict[str, Any] = {} # accumulating step grammar (from/merge_from/...) - - def cell_ref(cell: str) -> str: - """Map a cell name to its bind reference (an @output capture or a step result).""" - return capture_cells.get(cell, cell) - - def flush_step(op: Optional[Any], explicit_name: Optional[str] = None) -> str: - nonlocal prev_name, pending - name = explicit_name or _auto_name(op, len(flow_map), taken) - entry: Any - if op is not None and not pending: - entry = op # a grammar-less step is just its op (the compact document form) - else: - entry = dict(pending) - if op is not None: - entry["op"] = op - flow_map[name] = entry - pending = {} - prev_name = name - return name - - for raw in ops: - view = _ctx_view(raw) - if view is Save: - save_name = str(_ctx_field(raw, "name", "")) - if prev_name is not None: - # rename the just-flushed step to the cell name - entry = flow_map.pop(prev_name) - for e in flow_map.values(): - b = e.get("bind") if isinstance(e, dict) else None - if b: - for p, r in list(b.items()): - head, dot, attr = r.partition(".") - if head == prev_name: - b[p] = save_name + (dot + attr if dot else "") - flow_map[save_name] = entry - for cell, ref in list(capture_cells.items()): - head, dot, attr = ref.partition(".") - if head == prev_name: - capture_cells[cell] = save_name + (dot + attr if dot else "") - prev_name = save_name - else: - # Save before any op: an identity step naming the source - flow_map[save_name] = {} - prev_name = save_name - continue - if view is Use: - pending["from"] = cell_ref(str(_ctx_field(raw, "name", ""))) - continue - if view is MergeFields: - sources = [cell_ref(str(c)) for c in (_ctx_field(raw, "sources", None) or [])] - pending["merge_from"] = sources - pending["__mix_pending__"] = True - continue - if view is Drop: - continue # liveness is recomputed on lowering + Only a straight chain converts: a `Stream` carries an op LIST, which cannot express + fan-out. A branchy graph has no flat spelling (that is what the deleted lowering pass + manufactured, at the cost of destroying the structure), so it raises. + """ + from recordstream.core import Stream - # A real op (possibly Apply/Capture-wrapped): unwrap into bind grammar. - bind: Dict[str, str] = {} - captures: Dict[str, str] = {} - op: Any = raw - while _ctx_view(op) in (Apply, Capture): - if _ctx_view(op) is Capture: - for attr, cell in _capture_items(op).items(): - captures[cell] = attr - else: - ref = cell_ref(str(_ctx_field(op, "source", ""))) - apply_key = str(_ctx_field(op, "key", "") or "") - bind[str(_ctx_field(op, "param", ""))] = f"{ref}[{apply_key}]" if apply_key else ref - op = _ctx_field(op, "op") - pending.pop("__mix_pending__", None) - if bind: - pending["bind"] = bind - name = flush_step(op) - for cell, attr in captures.items(): - capture_cells[cell] = f"{name}.{attr}" - - # A trailing MergeFields (or Use) with no following op = a pure fan-in step. - if pending: - pending.pop("__mix_pending__", None) - flush_step(None) - - if not flow_map: - raise ValueError("from_ops: no steps could be lifted (empty op list?)") - out = outputs or prev_name or next(reversed(flow_map)) - return flow_map, out - - -def flow_yaml_to_stream(path: str, source: Optional[Any] = None) -> Any: - """Convenience: load a ``flow:`` YAML and return the SERIAL engine (lowered Stream).""" - from recordstream.core import Stream - - doc = _confluid_resolve(path) - if not isinstance(doc, dict) or "flow" not in doc: - raise ValueError(f"flow_yaml_to_stream: {path!r} has no 'flow:' mapping") - parsed, outputs = parse_flow(doc["flow"], str(doc.get("outputs", "") or "")) - return Stream(source=source, ops=to_ops(parsed, outputs)) + steps, outputs = self._ensure_parsed() + if not is_linear(steps, outputs): + raise TypeError( + "FlowGraph.to_stream: this graph is not a straight chain (it forks or merges), " + "and a Stream's ops list cannot express that. Iterate the FlowGraph directly — " + "it is the same engine." + ) + return Stream(source=self.source, ops=[s.op for s in steps if s.op is not None]) diff --git a/recordstream/ops/__init__.py b/recordstream/ops/__init__.py index 3ba1d9f..62bda26 100644 --- a/recordstream/ops/__init__.py +++ b/recordstream/ops/__init__.py @@ -15,8 +15,6 @@ - recordstream.ops.configure: ConfigureOp (per-record parameter injection) - recordstream.ops.formula: FormulaOp (math formula over one record entry) - recordstream.ops.sink: RecordSinkOp (adapt a DataSink as a pass-through op) - - recordstream.ops.context: Save, Use, Drop, Apply, Capture, MergeFields (the per-record - Context graph plane — the flat-list building blocks a branchy flow: document lowers to) - recordstream.ops.debug: PrintRecordOp (per-record summary probe) The sequential composer ``Pipeline`` lives in :mod:`recordstream.transform` (package-root @@ -24,7 +22,6 @@ """ from recordstream.ops.configure import ConfigureOp -from recordstream.ops.context import Apply, Capture, Drop, MergeFields, Save, Use from recordstream.ops.debug import PrintRecordOp from recordstream.ops.enable import Enable from recordstream.ops.formula import FormulaOp @@ -38,29 +35,23 @@ from recordstream.ops.torch import ToTensor __all__ = [ - "Apply", - "Capture", "CocoToTorchVisionDetection", "ConfigureOp", "ConnectedComponents", "ConvertToImage", "CopyField", "DecodeTarget", - "Drop", "DropField", "Enable", "EncodeTarget", "FormulaOp", "MasksToDetectionBoxes", - "MergeFields", "Parallel", "PrintRecordOp", "RandomApply", "RenameField", - "Save", "RecordSinkOp", "SelectFields", "Threshold", "ToTensor", - "Use", ] diff --git a/recordstream/ops/context.py b/recordstream/ops/context.py deleted file mode 100644 index 88e8308..0000000 --- a/recordstream/ops/context.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Context ops — move data between the per-record :class:`~recordstream.context.Context` and the stream. - -The six flat-list building blocks of graph-shaped pipelines: ``Save`` (fork snapshot), -``Use`` (branch start), ``Drop`` (cell hygiene), ``Apply`` (per-record parameter from a -cell), ``Capture`` (an op's ``@output`` into a cell), and ``MergeFields`` (fan-in). A -branchy canvas graph or ``flow:`` document lowers to a plain sequential op list containing -these (``recordstream.flow.to_ops``), executable by the ordinary ``Stream`` engine — and lifts -back (``from_ops``). - -Graph wiring lives on the engine-created Context data plane, so the record itself stays -byte-identical to a linear run. Cells are stored by reference (ops are copy-on-write by -convention); ``Use`` copies on read unless it drops the cell — the same copy-on-read / -move-on-drop idiom as ``Apply(source=cell)``. -""" - -from copy import deepcopy -from typing import Any, Dict, List, Optional, cast - -from confluid import configurable, flow -from confluid.fluid import Fluid - -from recordstream.context import require -from recordstream.items import Record - -_MISSING = object() - - -def _flow_if_fluid(value: Any) -> Any: - """Materialize a still-deferred confluid marker (nested op values need per-item flow).""" - return flow(value) if isinstance(value, Fluid) else value - - -def _read_output(op: Any, name: str) -> Any: - """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. - - Reads a live ``@output`` attribute through wrapper chains (incl. our own ``Apply.op`` slot) so - ``Capture(op=Apply(op=X, …))`` reaches X's ``@output``. Returns ``_MISSING`` when absent. - """ - cur, seen = op, set() - while cur is not None and id(cur) not in seen: - seen.add(id(cur)) - value = getattr(cur, name, _MISSING) - if value is not _MISSING: - return value - cur = getattr(cur, "target", None) or getattr(cur, "op", None) - return _MISSING - - -def _cell_field(value: Any, key: str = "") -> Any: - """A cell's contribution to a value slot. - - A record (dict) cell contributes its ``key``-named entry when ``key`` is given, else - the whole record verbatim; a raw cell value is used verbatim. - """ - if isinstance(value, dict) and key: - return value[key] - return value - - -@configurable(category="op", group="structure") -class Save: - """Snapshot the stream record into a Context cell (pass-through). - - The record continues down the linear stream unchanged AND becomes readable by later - ``Use`` / ``Apply`` / ``MergeFields`` steps — the fork point of a fan-out. Stored by - reference (readers copy); ops are copy-on-write by convention, so the snapshot stays - intact as the stream continues. - - Args: - name: Context cell to store the record under; required at call time, validated lazily. - """ - - def __init__(self, name: str = "") -> None: - # Lazy / zero-arg: store config only; the cell name is validated at first call. - self.name = str(name) - - def __call__(self, record: Record) -> Record: - if not self.name: - raise ValueError("Save: 'name' (the context cell to write) is required") - require("Save").put(self.name, record) - return record - - -@configurable(category="op", group="structure") -class Use: - """Replace the stream record with a Context cell's value (a branch start). - - The incoming record is discarded; the cell's value becomes the stream record - (a raw cell value is used verbatim). Reads a DEEP COPY so two branches - reading one fork stay independent — unless ``drop`` frees the cell, which skips the - copy (move semantics, the right choice for a cell's LAST reader). - - Args: - name: Context cell to read; required at call time, validated lazily. - drop: When True, free the cell after reading and skip the defensive copy (move semantics). - """ - - def __init__(self, name: str = "", drop: bool = False) -> None: - # Lazy / zero-arg: store config only; the cell name is validated at first call. - self.name = str(name) - self.drop = bool(drop) - - def __call__(self, record: Any) -> Any: - if not self.name: - raise ValueError("Use: 'name' (the context cell to read) is required") - ctx = require("Use") - value = ctx.get(self.name) - if self.drop: - ctx.delete(self.name) - else: - value = deepcopy(value) - return value - - -@configurable(category="op", group="structure") -class Drop: - """Free Context cells (pass-through) — the explicit liveness hygiene step. - - Deleting a missing cell raises: in a compiled graph that means the liveness pass and - the op order disagree, which should fail loudly rather than leak. - - Args: - names: Context cells to delete after this point; an empty list (default) is a no-op. - """ - - def __init__(self, names: Optional[List[str]] = None) -> None: - # Lazy / zero-arg: store config only. - self.names = list(names) if names else [] - - def __call__(self, record: Record) -> Record: - if self.names: - ctx = require("Drop") - for name in self.names: - ctx.delete(name) - return record - - -@configurable(category="op", group="structure") -class Apply: - """Set a wrapped op's parameter from a Context cell, then apply the op. - - The declarative per-record-parameter step (``ConfigureOp`` with the value coming from - a cell instead of an inline compute chain): the cell holds a prior branch's result — - a record cell contributes its ``key``-named entry (or the whole record when ``key`` is - blank), a raw cell value (e.g. a ``Capture``\\ d ``@output``) is used as-is. The value - is ``setattr``'d as ``param`` on ``op`` post-construction (the confluid paradigm), - then ``op`` runs on the incoming record. - - Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call - (like ``ConfigureOp``), so an ``Apply()`` built from YAML costs nothing. - - Args: - op: The op to configure and apply; required at call time, validated lazily. - param: Attribute name on ``op`` to set with the cell value; required at call time. - source: Context cell holding the value; required at call time, validated lazily. - key: For a record cell — the named entry to contribute. Blank (default) = the whole cell value. - drop: When True, free the source cell after reading it. - """ - - def __init__( - self, - op: Optional[object] = None, - param: str = "", - source: str = "", - key: str = "", - drop: bool = False, - ) -> None: - # Lazy / zero-arg: store config only; op/param/source are validated at first call. - self.op = op - self.param = str(param) - self.source = str(source) - self.key = str(key) - self.drop = bool(drop) - - def __call__(self, record: Any) -> Optional[Any]: - if self.op is None: - raise ValueError("Apply: an 'op' to configure and apply is required") - if not self.param: - raise ValueError("Apply: 'param' (the op attribute to set) is required") - if not self.source: - raise ValueError("Apply: 'source' (the context cell holding the value) is required") - self.op = _flow_if_fluid(self.op) - ctx = require("Apply") - value = ctx.get(self.source) - if self.drop: - ctx.delete(self.source) - value = _cell_field(value, key=self.key) - op = cast(Any, self.op) - setattr(op, self.param, value) - # _apply_op = the engine's op-family dispatch, so a bare library transform wired as - # the wrapped op applies exactly as in a bare ops list. - from recordstream.core import _apply_op - - return _apply_op(record, op) - - def close(self) -> None: - """Propagate close() to the wrapped op if it owns resources.""" - close_fn = getattr(self.op, "close", None) - if callable(close_fn): - close_fn() - - -@configurable(category="op", group="structure") -class Capture: - """Apply an op, then record its ``@output`` attribute(s) into Context cells. - - Records a wrapped op's live ``@output``: the wrapped op runs once (stochastic-correct - — the value is read from the actual run, never recomputed) and each requested - ``@output`` is stored as a raw cell value for a later ``Apply``/``Mix`` to read. The - returned record is ``op(record)`` — transformations are kept. - - Confluid ``!class:`` / ``!lazy:`` markers in ``op`` are flowed lazily at first call, - so a ``Capture()`` built from YAML costs nothing. - - Args: - op: The op to apply; its ``@output`` attributes are read after it runs. Required at call time. - output: A single ``@output`` attribute name to capture. Blank = capture only the ``captures`` entries. - name: Context cell for the ``output`` value. Blank (default) = the ``output`` name itself. - captures: Mapping of ``@output`` attribute name -> context cell, for capturing several outputs in one apply. - """ - - def __init__( - self, - op: Optional[object] = None, - output: str = "", - name: str = "", - captures: Optional[Dict[str, str]] = None, - ) -> None: - # Lazy / zero-arg: store config only; op/outputs are validated at first call. - self.op = op - self.output = str(output) - self.name = str(name) - self.captures = dict(captures) if captures else {} - - def _items(self) -> Dict[str, str]: - """The full ``{output_name: cell_name}`` map — ``captures`` plus the single-output form.""" - items = dict(self.captures) - if self.output: - items.setdefault(self.output, self.name or self.output) - return items - - def __call__(self, record: Record) -> Optional[Record]: - if self.op is None: - raise ValueError("Capture: an 'op' to apply is required") - items = self._items() - if not items: - raise ValueError("Capture: nothing to capture — set 'output' (and 'name') or 'captures'") - self.op = _flow_if_fluid(self.op) - ctx = require("Capture") - op = cast(Any, self.op) - # _apply_op = the engine's op-family dispatch (bare library transforms capture too). - from recordstream.core import _apply_op - - result = _apply_op(record, op) - if result is None: - return None # the wrapped op filtered the record (FilterOp semantics) - for attr, cell in items.items(): - value = _read_output(op, attr) - if value is _MISSING: - raise AttributeError(f"Capture: {type(op).__name__!r} has no @output attribute {attr!r} to capture") - ctx.put(cell, value) - return cast(Optional[Record], result) - - def close(self) -> None: - """Propagate close() to the wrapped op if it owns resources.""" - close_fn = getattr(self.op, "close", None) - if callable(close_fn): - close_fn() - - -@configurable(category="op", group="structure") -class MergeFields: - """Fan-in: UNION the named cells' entries into the incoming record. - - Each source cell (a record saved by an earlier branch) contributes its ENTRIES, - united in listed order with last-write-wins on a key collision (the deterministic - slot-order rule; avoid a deliberate collision by renaming on the producing branch — - ``recordstream.ops.structure.RenameField``). ``keys`` selects a subset of a source's - entries before the union. - - Args: - sources: Context cells (earlier branch results) to union into the incoming record, in order. - keys: Restrict the union to these keys across all sources. Empty (default) = every entry. - drop: Context cells to free after merging (defaults to none). - """ - - def __init__( - self, - sources: Optional[List[str]] = None, - keys: Optional[List[str]] = None, - drop: Optional[List[str]] = None, - ) -> None: - # Lazy / zero-arg: store config only; cells are resolved at first call. - self.sources = list(sources) if sources else [] - self.keys = list(keys) if keys else [] - self.drop = list(drop) if drop else [] - - def __call__(self, record: Record) -> Record: - if not self.sources: - raise ValueError("MergeFields: 'sources' (the context cells to union) is required") - if not isinstance(record, dict): - raise TypeError(f"MergeFields: the incoming carrier is {type(record).__name__}, expected a record dict.") - ctx = require("MergeFields") - merged = dict(record) - for cell_name in self.sources: - value = ctx.get(cell_name) - if not isinstance(value, dict): - raise TypeError(f"MergeFields: cell {cell_name!r} holds {type(value).__name__}, expected a record") - if self.keys: - value = {k: value[k] for k in self.keys if k in value} - merged.update(value) - for cell_name in self.drop: - ctx.delete(cell_name) - return merged diff --git a/tests/test_categories.py b/tests/test_categories.py index 678e9bc..8b8efc6 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -12,7 +12,6 @@ from recordstream import Pipeline from recordstream.core import FilterOp, JointStream, Stream, WrappedOp from recordstream.ops.configure import ConfigureOp -from recordstream.ops.context import Apply, Capture, Drop, MergeFields, Save, Use from recordstream.ops.debug import PrintRecordOp from recordstream.ops.enable import Enable from recordstream.ops.formula import FormulaOp @@ -70,12 +69,6 @@ def test_op_classes_tagged() -> None: DropField, CopyField, SelectFields, - Save, - Use, - Drop, - Apply, - Capture, - MergeFields, PrintRecordOp, ): assert cls.__confluid_category__ == "op", cls.__name__ @@ -104,8 +97,6 @@ def test_op_group_tags() -> None: assert DecodeTarget.__confluid_group__ == "structure" assert CocoToTorchVisionDetection.__confluid_group__ == "structure" assert MasksToDetectionBoxes.__confluid_group__ == "structure" - for ctx_op in (Save, Use, Drop, Apply, Capture, MergeFields): - assert ctx_op.__confluid_group__ == "structure", ctx_op.__name__ assert Parallel.__confluid_group__ == "compose" assert Enable.__confluid_group__ == "compose" assert Pipeline.__confluid_group__ == "compose" @@ -155,8 +146,5 @@ def test_groups_enumerable_via_registry() -> None: "CocoToTorchVisionDetection", "MasksToDetectionBoxes", "SelectFields", - "Save", - "Use", - "MergeFields", } <= registry.list_classes(group="structure") assert "Pipeline" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py index bf7f381..61e952f 100644 --- a/tests/test_node_docs.py +++ b/tests/test_node_docs.py @@ -15,7 +15,6 @@ from recordstream import Pipeline, Transform from recordstream.core import FilterOp, JointStream, Stream, WrappedOp from recordstream.ops.configure import ConfigureOp -from recordstream.ops.context import Apply, Capture, Drop, MergeFields, Save, Use from recordstream.ops.debug import PrintRecordOp from recordstream.ops.enable import Enable from recordstream.ops.formula import FormulaOp @@ -48,12 +47,6 @@ DropField, CopyField, SelectFields, - Save, - Use, - Drop, - Apply, - Capture, - MergeFields, ConfigureOp, FormulaOp, Enable, diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index d0d7c8e..48a3e25 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -1,14 +1,13 @@ """FlowGraph over dict records — merge_from fan-in, step[key]/bare-step bind, lowering parity.""" from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Optional import numpy as np import pytest -from recordstream import FlowGraph, Image, Label, Mask, Pipeline, Record, Stream, Transform, to_ops -from recordstream.flow import from_ops, parse_flow -from recordstream.ops.context import MergeFields +from recordstream import FlowGraph, Image, Label, Mask, Pipeline, Record, Stream, Transform +from recordstream.flow import parse_flow from recordstream.ops.structure import RenameField, SelectFields @@ -150,59 +149,6 @@ def test_merge_from_forward_ref_raises(self) -> None: parse_flow({"a": {"merge_from": ["b"]}, "b": {}}) -class TestLoweringParity: - def _flow(self) -> Dict[str, Any]: - return { - "start": {}, - "masked": {"op": _MakeMask(), "from": "start"}, - "mask_only": {"op": SelectFields(keys=["mask"]), "from": "masked"}, - "boosted": {"op": _AddOffset(offset=1.0), "from": "start"}, - "out": {"from": "boosted", "merge_from": ["mask_only"]}, - } - - def test_to_ops_runs_on_stream(self) -> None: - # The lowered flat op list (MergeFields wiring) matches the native FlowGraph result. - steps, outputs = parse_flow(self._flow()) - native = list(FlowGraph(source=[_seed(0.25)], flow=self._flow(), outputs="out")) - lowered = list(Stream(source=[_seed(0.25)], ops=to_ops(steps, outputs))) - assert len(native) == len(lowered) == 1 - assert list(native[0].keys()) == list(lowered[0].keys()) - assert np.array_equal(np.asarray(native[0]["image"]), np.asarray(lowered[0]["image"])) - assert np.array_equal(np.asarray(native[0]["mask"]), np.asarray(lowered[0]["mask"])) - - def test_round_trip_from_ops(self) -> None: - steps, outputs = parse_flow(self._flow()) - ops = to_ops(steps, outputs) - assert any(isinstance(op, MergeFields) for op in ops) - lifted, lifted_out = from_ops(ops) - relowered = to_ops(*parse_flow(lifted, lifted_out)) - native = list(Stream(source=[_seed(0.5)], ops=relowered)) - assert len(native) == 1 and "mask" in native[0] - - def test_key_bind_round_trip(self) -> None: - class _Reader(Transform): - def __init__(self, item: Any = None) -> None: - super().__init__() - self.item = item - - def __call__(self, record: Record) -> Record: - return {**record, "echo": self.item} - - flow = { - "start": {}, - "probe": {"op": _AddOffset(offset=3.0), "from": "start"}, - "final": {"op": _Reader(), "from": "start", "bind": {"item": "probe[image]"}}, - } - steps, outputs = parse_flow(flow) - ops = to_ops(steps, outputs) - lifted, _ = from_ops(ops) - # the key-bind grammar survives the round trip - final_step = lifted["final"] if "final" in lifted else list(lifted.values())[-1] - assert isinstance(final_step, dict) and final_step["bind"]["item"].endswith("[image]") - (out,) = list(Stream(source=[_seed(0.0)], ops=ops)) - assert np.allclose(np.asarray(out["echo"]), 3.0) - - class TestRecordsThroughStream: def test_stream_carries_record_dicts_verbatim(self) -> None: stream = Stream(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) @@ -257,13 +203,19 @@ def test_yaml_bind_via_plain_mapping_step(self, tmp_path: Path) -> None: assert int(np.asarray(out["mask"]).sum()) == 8 # fixed 0.5 threshold assert int(np.asarray(out["gated_mask"]).sum()) == 6 # per-record amax(a)*0.6 bind - def test_yaml_bind_parity_with_lowered_stream(self, tmp_path: Path) -> None: + def test_yaml_bind_runs_the_same_from_either_loader(self, tmp_path: Path) -> None: + # FlowGraph.from_yaml and a hand-parsed flow are the same graph on the same engine. path = tmp_path / "graph.yaml" path.write_text(self._doc()) - a = list(FlowGraph.from_yaml(str(path), source=[self._record()]))[0] - b = list(Stream.from_flow_yaml(str(path), source=[self._record()]))[0] - assert np.array_equal(np.asarray(a["gated_mask"]), np.asarray(b["gated_mask"])) - assert np.array_equal(np.asarray(a["mask"]), np.asarray(b["mask"])) + record = self._record() + + from_yaml = list(FlowGraph.from_yaml(str(path), source=[dict(record)])) + import confluid + + doc = confluid.resolve(str(path)) + parsed = list(FlowGraph(source=[dict(record)], flow=doc["flow"], outputs=str(doc.get("outputs", "")))) + assert len(from_yaml) == len(parsed) == 1 + assert set(from_yaml[0]) == set(parsed[0]) def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path: Path) -> None: # Pin the confluid behavior that makes the op:-form MANDATORY for bind — if this @@ -283,3 +235,181 @@ def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path: Path) - marker = confluid.resolve(str(path))["flow"]["gated"] assert "bind" not in marker.kwargs # consumed as addressed configuration + + +class TestNativeExecution: + """The graph engine runs on its OWN executor — it never lowers to run (2026-07-29).""" + + def test_reader_accounting_is_computed_once_per_graph(self, monkeypatch: Any) -> None: + # _result_readers depends only on (steps, outputs); recomputing it per record was an + # O(steps^2) tax measured at ~half the graph engine's overhead over a flat op list. + import recordstream.flow as flow_mod + + calls = {"n": 0} + real = flow_mod._result_readers + + def counting(steps: Any, outputs: str) -> Any: + calls["n"] += 1 + return real(steps, outputs) + + monkeypatch.setattr(flow_mod, "_result_readers", counting) + graph = FlowGraph(source=[_seed(1.0) for _ in range(25)], flow={"plus": _AddOffset(offset=2.0)}) + assert len(list(graph)) == 25 + assert calls["n"] == 1 + + def test_there_is_no_lowering_pass_left_to_call(self) -> None: + # The delegation this replaced built Stream(ops=to_ops(...)). Both converters are gone + # with the context ops they targeted; a reintroduced one would be a second executor. + import recordstream + import recordstream.flow as flow_mod + + for gone in ("to_ops", "from_ops", "flow_yaml_to_stream"): + assert not hasattr(flow_mod, gone), f"{gone} is back — the lowering pass has returned" + assert not hasattr(recordstream, gone) + assert not hasattr(Stream, "from_flow_yaml") + + def test_parallel_runs_the_graph_natively(self) -> None: + source = [_seed(float(i)) for i in range(6)] + serial = list(FlowGraph(source=source, flow={"plus": _AddOffset(offset=2.0)})) + parallel = list(FlowGraph(source=source, flow={"plus": _AddOffset(offset=2.0)}).parallel(2)) + + assert len(parallel) == len(serial) == 6 + for got, want in zip(parallel, serial): + assert np.allclose(np.asarray(got["image"]), np.asarray(want["image"])) + + def test_parallel_preserves_source_order(self) -> None: + source = [_seed(float(i)) for i in range(8)] + out = list(FlowGraph(source=source, flow={"plus": _AddOffset(offset=1.0)}).parallel(3)) + assert [float(np.asarray(r["image"]).flat[0]) for r in out] == [float(i) + 1.0 for i in range(8)] + + +class _SplitChannels(Transform): + """A 1→N EXPANDING step: one record per channel of the image.""" + + EXPANDS = True + + def __call__(self, record: Record) -> Any: # type: ignore[override] + image = record["image"] + return [{**record, "image": Image(np.asarray(image)[..., c : c + 1]), "channel": c} for c in range(3)] + + +class _Tag(Transform): + """Marks the record so a post-expansion step is observable.""" + + def __init__(self, tag: str = "") -> None: + super().__init__() + self.tag = tag + + def __call__(self, record: Record) -> Record: + return {**record, "tag": self.tag} + + +class TestExpandingSteps: + """A 1→N step forks the REMAINING subgraph, one branch per child (2026-07-29).""" + + def test_expansion_yields_one_record_per_child(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"split": _SplitChannels()}) + out = list(graph) + assert [r["channel"] for r in out] == [0, 1, 2] + + def test_downstream_steps_run_once_per_child(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"split": _SplitChannels(), "tagged": _Tag(tag="t")}) + out = list(graph) + assert [r["channel"] for r in out] == [0, 1, 2] + assert all(r["tag"] == "t" for r in out) + + def test_depth_first_sibling_order_across_chained_expansions(self) -> None: + # Nested-loop order (the flat engine's documented contract): the INNER expansion + # varies fastest. Two 3-way splits => 9 branches, the second split's channel cycling + # 0,1,2 within each child of the first. + graph = FlowGraph(source=[_seed(1.0)], flow={"a": _SplitChannels(), "b": _SplitChannels()}) + out = list(graph) + assert len(out) == 9 + assert [r["channel"] for r in out] == [0, 1, 2] * 3 + + def test_branches_do_not_share_env_state(self) -> None: + # Each child gets its OWN shallow copy of the step environment (the graph twin of + # Context.copy()): a later fan-in must not see a sibling's result. + flow = { + "src": {}, + "split": _SplitChannels(), + "out": {"from": "split", "merge_from": ["src"]}, + } + out = list(FlowGraph(source=[_seed(1.0)], flow=flow, outputs="out")) + assert [r["channel"] for r in out] == [0, 1, 2] + + def test_len_and_getitem_raise_for_an_expanding_graph(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"split": _SplitChannels()}) + with pytest.raises(TypeError, match="EXPANDING"): + len(graph) + with pytest.raises(TypeError, match="EXPANDING"): + graph[0] + + def test_empty_expansion_drops_the_branch(self) -> None: + class _Drop(Transform): + EXPANDS = True + + def __call__(self, record: Record) -> Any: # type: ignore[override] + return [] + + assert list(FlowGraph(source=[_seed(1.0)], flow={"gone": _Drop()})) == [] + + def test_expansion_survives_the_spawn_boundary(self) -> None: + # The worker returns a LIST precisely so one seed can yield several records. + source = [_seed(1.0), _seed(2.0)] + out = list(FlowGraph(source=source, flow={"split": _SplitChannels()}).parallel(2)) + assert len(out) == 6 + assert [r["channel"] for r in out] == [0, 1, 2, 0, 1, 2] + + +class TestOneExecutor: + """`ops:` is the LINEAR SPELLING of a graph — both forms run the same kernel (2026-07-29).""" + + def test_an_ops_list_compiles_to_a_linear_step_graph(self) -> None: + from recordstream.core import linear_steps + + ops = [_AddOffset(offset=1.0), _AddOffset(offset=2.0)] + steps, outputs = linear_steps(ops) + assert [s.name for s in steps] == ["s0", "s1"] + assert outputs == "s1" + # Positional names, so the SAME op twice is two steps (a name-keyed mapping would collapse them). + assert all(s.from_ is None and not s.bind and not s.merge_from for s in steps) + + def test_a_repeated_op_stays_two_distinct_steps(self) -> None: + from recordstream.core import linear_steps + + op = _AddOffset(offset=1.0) + steps, _ = linear_steps([op, op]) + assert len(steps) == 2 + (out,) = list(Stream(source=[_seed(0.0)], ops=[op, op])) + assert np.allclose(np.asarray(out["image"]), 2.0) # applied twice, not once + + def test_stream_and_flowgraph_agree_on_the_same_linear_chain(self) -> None: + ops = [_AddOffset(offset=1.0), _MakeMask()] + flat = list(Stream(source=[_seed(0.75)], ops=ops)) + graph = list(FlowGraph(source=[_seed(0.75)], flow={f"s{i}": op for i, op in enumerate(ops)})) + assert len(flat) == len(graph) == 1 + assert np.allclose(np.asarray(flat[0]["image"]), np.asarray(graph[0]["image"])) + assert np.array_equal(np.asarray(flat[0]["mask"]), np.asarray(graph[0]["mask"])) + + def test_an_empty_ops_list_is_the_identity(self) -> None: + # The kernel treats "no steps" as the identity graph; a bare Stream must still yield. + source = [_seed(1.0), _seed(2.0)] + assert len(list(Stream(source=source, ops=[]))) == 2 + assert len(list(Stream(source=source))) == 2 + + def test_a_linear_chain_takes_the_env_free_path(self) -> None: + from recordstream.core import linear_steps + from recordstream.flow import is_linear + + steps, outputs = linear_steps([_AddOffset(offset=1.0), _AddOffset(offset=2.0)]) + assert is_linear(steps, outputs) + # A fan-out graph must NOT qualify — it needs the step environment. + branchy, out = parse_flow({"start": {}, "a": {"op": _AddOffset(offset=1.0), "from": "start"}}, "a") + assert not is_linear(branchy, out) + + def test_stream_expansion_still_yields_every_child(self) -> None: + # The flat engine's 1→N contract, now served by the shared kernel's linear path. + out = list(Stream(source=[_seed(1.0)], ops=[_SplitChannels(), _Tag(tag="t")])) + assert [r["channel"] for r in out] == [0, 1, 2] + assert all(r["tag"] == "t" for r in out) From 617fb9b67a9cb02db62c16e3156ae96bffbcdbb5 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 30 Jul 2026 11:55:32 +0200 Subject: [PATCH 060/102] =?UTF-8?q?chore:=20regenerate=20Jenkinsfile.local?= =?UTF-8?q?=20=E2=80=94=20guard=20on=20pyproject.toml,=20not=20the=20direc?= =?UTF-8?q?tory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AUTO-GENERATED artifact, regenerated from aisland's template after the guard fix: `git submodule deinit` leaves an empty DIRECTORY behind, so the old `[ -d ]` predicate was true for every detached dependency and the editable-install branch failed the build against an empty directory. Now `[ -f /pyproject.toml ]`. Regenerate with: aisland jenkins scaffold --project --force --- Jenkinsfile.local | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 1515ae4..9e1b046 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,9 +42,9 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "if [ -d '${env.WORKSPACE_ROOT}/loggair' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" - sh "if [ -d '${env.WORKSPACE_ROOT}/confluid' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" - sh "if [ -d '${env.WORKSPACE_ROOT}/liquifai' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/confluid/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/liquifai/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships From d9d6270ced3ffc820dc99467b690bd2f67393411 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 30 Jul 2026 13:54:09 +0200 Subject: [PATCH 061/102] feat(deps): make torch an optional extra; the core engine is numpy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `import recordstream` no longer imports any ML framework, so a Keras-only, TensorFlow-only or plain-numpy consumer stops installing ~2GB it never calls (marainer inherited it transitively, declaring no torch of its own). The coupling was almost entirely nominal. Four mechanisms replace it: * `Stream`/`FlowGraph` drop the `torch.utils.data.Dataset` base for a `MapStyle` Protocol (`__len__`/`__getitem__`) — all a `DataLoader` ever needed, since it duck-types its argument. Nothing did `isinstance(x, Dataset)` and nothing subclassed `Stream`, so the base bought only the dependency. * `_compat.is_torch_tensor` answers "is this a torch tensor?" via `sys.modules` instead of importing: a tensor cannot exist unless torch is already loaded, so the check is exact rather than a heuristic. This is what storage/base.py and ops/image.py used a module-level import for. * `ToTensor` becomes a lazy export (PEP 562 `__getattr__` over `_OPTIONAL_OPS`), raising an ImportError that names the extra. * outputs.py splits by what needs a runtime: the TypedDicts stay module-level (typing-only, generic in the array type) while the softmax/argmax builders import torch in the function body. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a type-checked `DataLoader(stream)` call now needs `cast(Any, stream)`. That bridge belongs at the call site — re-adding the base to satisfy a stub would restore the dependency. Verified in a subprocess with torch made unimportable: the package imports and iterates a Stream with zero torch modules loaded. --- AGENTS.md | 1 + README.md | 13 +++ docs/architecture.md | 105 +++++++++++++++++++ pyproject.toml | 15 ++- recordstream/_compat.py | 35 +++++++ recordstream/core.py | 61 ++++++++--- recordstream/flow.py | 3 +- recordstream/ops/__init__.py | 31 +++++- recordstream/ops/image.py | 6 +- recordstream/outputs.py | 17 +-- recordstream/storage/base.py | 6 +- tests/test_optional_torch.py | 194 +++++++++++++++++++++++++++++++++++ 12 files changed, 456 insertions(+), 31 deletions(-) create mode 100644 recordstream/_compat.py create mode 100644 tests/test_optional_torch.py diff --git a/AGENTS.md b/AGENTS.md index 7873917..87eaad7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). +- **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and marainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch"]`, so `aisland framework set torch` installs `recordstream[dev,torch]` and a selection without it installs `[dev]`; the same committed selection is what generated CI installs. Pins: `tests/test_optional_torch.py`. - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. - **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. diff --git a/README.md b/README.md index 765088a..e46a159 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,19 @@ RecordStream is designed to sit between your data catalog and your training loop pip install git+https://github.com/Gearlux/recordstream.git@main ``` +The core engine is **numpy**, and installs no ML framework. PyTorch is an extra, needed only for +the pieces that genuinely produce tensors — the `ToTensor` op and the `classification_output` / +`segmentation_output` builders: + +```bash +pip install "recordstream[torch] @ git+https://github.com/Gearlux/recordstream.git@main" +``` + +Everything else works without it. A `Stream` is map-style (`__len__`/`__getitem__`), so a +`DataLoader` still accepts one directly on a torch install; `batch_values`, `multi_hot` and the +class-balance statistics return numpy, so a non-torch backend converts in one line. Reaching for +`recordstream.ops.ToTensor` without the extra raises an `ImportError` naming it. + ## 📄 License MIT diff --git a/docs/architecture.md b/docs/architecture.md index c28ca43..a6f7781 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -827,3 +827,108 @@ def predict_step(self, batch, batch_idx): - **A different balancing policy** (effective-number, sqrt-inverse): add it beside `inverse_frequency_weights` in `recordstream.labels` as another statistic returning numpy. Do NOT add the injection here — that stays a per-backend method on the runnable. + +--- + +## 9. torch is an extra; the engine is numpy (2026-07-30) + +### Context + +`recordstream` declared `torch` as a hard dependency, so `import recordstream` imported ~2GB of +PyTorch — and marainer inherited it transitively, declaring no torch of its own. That was fine +while every consumer was a Lightning trainer. It stopped being fine when a second training engine +landed: a Keras-on-TensorFlow install, or a plain-numpy dataset-conversion job, paid for a +framework it never called. + +Auditing what actually needed torch found the coupling was almost entirely nominal: + +- **`Stream` and `FlowGraph` subclassed `torch.utils.data.Dataset`.** This was the expensive + line, and it bought nothing. `Dataset` is an empty base — `DataLoader` duck-types its argument, + needing only `__len__` and `__getitem__` (verified against a plain class with those two + methods and no base). Nothing in the workspace does `isinstance(x, Dataset)`, and nothing + subclasses `Stream`. +- **`storage/base.py` and `ops/image.py` imported torch for `isinstance(x, torch.Tensor)` alone** — + to decide whether a payload needed `.detach().cpu().numpy()` before being written or rendered. +- Only `ops/torch.py` (`ToTensor`) and `outputs.py`'s `softmax`/`argmax` builders genuinely + compute with it. + +The two isinstance sites are the interesting case, because the naive fix — a lazy in-function +`import torch` — still *imports torch* the first time a record is written. + +### Decision + +**`torch` moved from `dependencies` to `[project.optional-dependencies] torch`, and the core +imports no framework.** Four mechanisms, one per coupling: + +1. **The `Dataset` base is dropped** in favour of a `MapStyle` Protocol (`__len__` + + `__getitem__`) — the engine still *says* "map-style dataset" in its own vocabulary, and + `RecordSource = Union[MapStyle, Iterable[Record]]` stays the contract `ensure_record_dataset` + enforces. +2. **Type identity without an import**: `recordstream._compat.is_torch_tensor` consults + `sys.modules` rather than importing. This is exact, not a heuristic — *a torch tensor cannot + exist in a process that has not imported torch*, so the absence of the module proves the + negative. It is the same instinct as the op-family matchers, which identify an albumentations + or torchvision transform by its MRO module name. +3. **`ToTensor` is a lazy export** — `recordstream.ops` maps it in `_OPTIONAL_OPS` and resolves it + in a PEP 562 module `__getattr__`, raising an `ImportError` that names the extra instead of a + traceback from three libraries down. `__dir__` still advertises it so completion works. +4. **`outputs.py` splits by what needs a runtime**: the `TypedDict` contracts stay module-level + (they are typing-only, and generic in the array type), while `classification_output` / + `segmentation_output` import torch in the function body — they are the only part that computes. + +### Consequences + +- **`DataLoader(stream)` now needs `cast(Any, stream)` in type-checked code.** torch's *stub* + declares `Dataset[T]`; the runtime accepts any map-style object. This is a stub's stricter view + of a contract that works, and the bridge belongs at the four call sites (all in tests) rather + than in the engine — re-adding the base to satisfy a stub would restore the 2GB dependency to + silence a type checker. +- **`MapStyle` must be referenced as the real class in any annotation a consumer introspects, never + a string forward-ref.** confluid evaluates annotations in the *consumer's* namespace, so + `RecordSource = Union["MapStyle", ...]` raised `NameError: name 'MapStyle' is not defined` from + a consumer's `__init__` scan, three packages away. +- **The numpy-return rule elsewhere is now load-bearing, not stylistic.** `batch_values`, + `multi_hot`, `batch_metadata` and the class-balance statistics return numpy precisely so this + boundary holds; only `batch_tensor` is torch. +- **`recordstream.ops.torch` cannot be eagerly imported by anything in the package** — a new + convenience re-export there would silently undo all of the above. + +### Example + +```python +# storage/base.py — recognise a tensor without importing torch +from recordstream._compat import is_torch_tensor + +def to_numpy(data): + return data.detach().cpu().numpy() if is_torch_tensor(data) else np.asarray(data) +``` + +```python +# recordstream/ops/__init__.py — the op is reachable, the import is not eager +_OPTIONAL_OPS = {"ToTensor": ("recordstream.ops.torch", "torch")} + +def __getattr__(name): + entry = _OPTIONAL_OPS.get(name) + if entry is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_path, extra = entry + try: + return getattr(importlib.import_module(module_path), name) + except ImportError as exc: + raise ImportError(f"recordstream.ops.{name} needs the {extra!r} extra: " + f"pip install 'recordstream[{extra}]'") from exc +``` + +### What you may change (and where it's documented) + +- **Add another optional-framework op**: put it in its own module, add one `_OPTIONAL_OPS` entry + and one `__all__` entry, and declare the extra in `pyproject.toml`. No other edit — the error + message and `dir()` follow from the mapping. +- **Recognise another framework's tensor type** (a TF tensor, a jax array): add a sibling to + `_compat.py` using the same `sys.modules` rule. Do not add a module-level import of that + framework anywhere in the core. +- **Need a torch-typed return from an existing helper**: add a `dtype=`/`device=` parameter and + keep the numpy default, as `batch_tensor` does — do not change an existing numpy return, or the + next backend has to reimplement it. +- **Installation** is documented in the README's Installation section; what each extra provides + is the `pyproject.toml` comment beside it. diff --git a/pyproject.toml b/pyproject.toml index 328ce93..f71b15e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ dependencies = [ "Pillow", "h5py", "zarr", - "torch", "typing-extensions", "datasets", "albumentations", @@ -25,6 +24,12 @@ dependencies = [ requires-python = ">=3.12" [project.optional-dependencies] +# The torch-shaped surface, NOT the engine: `recordstream.ops.torch` (ToTensor), +# `batch_tensor`, and the `outputs` builders. The core is numpy — `Stream` is a plain +# map-style class, and the two places that recognise a torch tensor do it through +# `_compat.is_torch_tensor`, which consults sys.modules rather than importing. So a +# consumer on Keras/JAX installs recordstream without a 2GB framework it never calls. +torch = ["torch"] dev = [ "black>=24.0.0,<25.0.0", "isort>=5.13.0,<6.0.0", @@ -113,6 +118,14 @@ recordstream = "recordstream.cli:main" [project.entry-points."liquifai.apps"] recordstream = "recordstream.cli:app" +# Which of this project's extras are ML FRAMEWORKS, for `aisland framework`. Selecting a name +# here is what makes `aisland setup` add the matching extra to this project's install. +# +# The core engine is numpy — `torch` powers only `ToTensor` and the output builders, so a +# workspace that selects no framework still gets a fully working record engine. +[tool.aisland] +frameworks = ["torch"] + [tool.setuptools.packages.find] where = ["."] include = ["recordstream*"] diff --git a/recordstream/_compat.py b/recordstream/_compat.py new file mode 100644 index 0000000..586fb8d --- /dev/null +++ b/recordstream/_compat.py @@ -0,0 +1,35 @@ +"""Answering "is this value from an optional framework?" without importing that framework. + +recordstream's core is numpy. A handful of places still need to recognise a torch tensor — +storage converting a payload before writing, image conversion detaching one — and doing that +with a module-level ``import torch`` made a 2GB framework a hard dependency of a package whose +own work is arrays. + +The trick is that the question answers itself: **a torch tensor cannot exist unless torch has +already been imported.** So consulting ``sys.modules`` is exact, not a heuristic — if the module +is absent the value is provably not one of its types, and if it is present we do a real +``isinstance`` with no import of our own. + +This is the same instinct as the op-family matchers in :mod:`recordstream.core`, which identify +an albumentations or torchvision transform by its MRO module name rather than importing either. +""" + +import sys +from typing import Any + +__all__ = ["is_torch_tensor"] + + +def is_torch_tensor(value: Any) -> bool: + """True when ``value`` is a ``torch.Tensor``, without importing torch. + + Exact rather than duck-typed: when ``torch`` is already loaded this is a real + ``isinstance`` check; when it is not, no torch tensor can exist in the process, so the + answer is ``False``. + + Example:: + + payload = value.detach().cpu().numpy() if is_torch_tensor(value) else np.asarray(value) + """ + torch = sys.modules.get("torch") + return torch is not None and isinstance(value, torch.Tensor) diff --git a/recordstream/core.py b/recordstream/core.py index cb03ec9..76b83cf 100644 --- a/recordstream/core.py +++ b/recordstream/core.py @@ -1,9 +1,23 @@ import concurrent.futures import multiprocessing from contextlib import nullcontext -from typing import Any, Callable, Collection, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union, cast +from typing import ( + Any, + Callable, + Collection, + Dict, + Iterable, + Iterator, + List, + Optional, + Protocol, + Sequence, + Tuple, + Union, + cast, + runtime_checkable, +) -import torch.utils.data from confluid import configurable from confluid import load as _confluid_load from confluid import materialize as _confluid_materialize @@ -163,6 +177,23 @@ def _apply_op(record: Record, op: Any) -> Optional[Record]: return cast(Optional[Record], op(record)) +@runtime_checkable +class MapStyle(Protocol): + """A map-style dataset: ``len(ds)`` and ``ds[i]``. + + What recordstream MEANS by "a dataset", said structurally so the engine never imports a + framework to express it. ``Stream`` used to inherit ``torch.utils.data.Dataset``, which made + torch a hard dependency of a package whose own work is numpy — for nothing: that base is not + load-bearing. ``DataLoader`` duck-types its argument (a plain object with these two methods + works), nothing in the workspace does ``isinstance(x, Dataset)``, and the annotation is the + only thing the inheritance ever bought. + """ + + def __len__(self) -> int: ... + + def __getitem__(self, index: int) -> Any: ... + + def _describe_deferred_source(source: Any) -> str: """Return a human-friendly description of a still-deferred Confluid source. @@ -359,7 +390,7 @@ def __len__(self) -> int: @configurable(category="engine") -class Stream(torch.utils.data.Dataset[Record]): +class Stream: """ The primary stream engine for RecordStream. Wraps any iterable or indexed dataset and provides a functional API. @@ -631,18 +662,18 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: #: What a wired dataset slot may hold — the contract :func:`ensure_record_dataset` enforces, -#: named ONCE here rather than restated by every consumer: a map-style ``Dataset`` (which a -#: :class:`Stream` is), or any iterable of records (a recordstream source, a plain list of -#: record dicts). Consumers annotate their slots ``Optional[Lazy[RecordSource]]`` — ``Lazy`` -#: because they flow the slot themselves at run time. -#: What a wired dataset slot may hold: anything MAP-STYLE (``__len__`` + ``__getitem__`` — -#: which a :class:`Stream` is) or any iterable of records. Expressed structurally rather than -#: as ``torch.utils.data.Dataset`` so the engine stays framework-free; torch's ``DataLoader`` -#: is itself duck-typed and consumes either. -RecordSource = Union[torch.utils.data.Dataset[Any], Iterable[Record]] - - -def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> torch.utils.data.Dataset[Any]: +#: What a wired dataset slot may hold, named ONCE here rather than restated by every consumer: +#: anything MAP-STYLE (``__len__`` + ``__getitem__`` — which a :class:`Stream` is), or any +#: iterable of records (a recordstream source, a plain list of record dicts). Consumers annotate +#: their slots ``Optional[Lazy[RecordSource]]`` — ``Lazy`` because they flow the slot at run time. +#: +#: Expressed with the structural :class:`MapStyle` rather than ``torch.utils.data.Dataset`` so the +#: engine can say "a dataset" without importing a framework; torch's ``DataLoader`` is itself +#: duck-typed and consumes either. +RecordSource = Union[MapStyle, Iterable[Record]] + + +def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> "Stream": """Normalize any wired source into a map-style ``Dataset`` that yields record dicts. A wired ``train_set`` / ``val_set`` / ``test_set`` may be a :class:`Stream`, another torch diff --git a/recordstream/flow.py b/recordstream/flow.py index fd74ebd..f0d9fd9 100644 --- a/recordstream/flow.py +++ b/recordstream/flow.py @@ -43,7 +43,6 @@ from copy import deepcopy from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Sequence, Tuple, Union, cast -import torch.utils.data from confluid import configurable, flow from confluid import resolve as _confluid_resolve from confluid.fluid import Fluid as _ConfluidFluid @@ -496,7 +495,7 @@ def _graph_worker_task( @configurable(category="engine") -class FlowGraph(torch.utils.data.Dataset[Record]): +class FlowGraph: """Named-step graph engine — executes a ``flow:`` document natively. The named-step twin of :class:`~recordstream.core.Stream`, over the SAME kernel: steps run diff --git a/recordstream/ops/__init__.py b/recordstream/ops/__init__.py index 62bda26..b005c07 100644 --- a/recordstream/ops/__init__.py +++ b/recordstream/ops/__init__.py @@ -21,6 +21,9 @@ export) — one list mixing native ops with bare albumentations / torchvision-v2 transforms. """ +import importlib +from typing import Any, Dict, List, Tuple + from recordstream.ops.configure import ConfigureOp from recordstream.ops.debug import PrintRecordOp from recordstream.ops.enable import Enable @@ -32,7 +35,6 @@ from recordstream.ops.sink import RecordSinkOp from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes -from recordstream.ops.torch import ToTensor __all__ = [ "CocoToTorchVisionDetection", @@ -55,3 +57,30 @@ "Threshold", "ToTensor", ] + + +#: Ops whose module needs an optional framework — resolved on first attribute access (PEP 562) +#: so ``import recordstream`` never imports one. ``ToTensor`` is the only such op today: its +#: module is torch by definition, and eagerly re-exporting it here was the single line that made +#: torch a hard dependency of the whole engine. +_OPTIONAL_OPS: Dict[str, Tuple[str, str]] = {"ToTensor": ("recordstream.ops.torch", "torch")} + + +def __getattr__(name: str) -> Any: + """Resolve an optional-framework op on first use, or name the extra that provides it.""" + entry = _OPTIONAL_OPS.get(name) + if entry is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_path, extra = entry + try: + return getattr(importlib.import_module(module_path), name) + except ImportError as exc: + raise ImportError( + f"recordstream.ops.{name} needs the {extra!r} extra: pip install 'recordstream[{extra}]'\n" + f"(underlying import error: {exc})" + ) from exc + + +def __dir__() -> List[str]: + """Advertise the lazily-resolved names to ``dir()`` / tab-completion.""" + return sorted({*globals(), *_OPTIONAL_OPS}) diff --git a/recordstream/ops/image.py b/recordstream/ops/image.py index b198200..db74971 100644 --- a/recordstream/ops/image.py +++ b/recordstream/ops/image.py @@ -21,11 +21,11 @@ from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, get_args import numpy as np -import torch from confluid import configurable from loggair import get_logger from PIL import Image, ImageDraw +from recordstream._compat import is_torch_tensor from recordstream.items import Image as ImageItem from recordstream.items import NDArrayItem, Record, item_data from recordstream.transform import Transform @@ -123,7 +123,7 @@ def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: if hasattr(data, "convert"): # PIL.Image.Image data = np.array(data.convert("RGB")) - elif isinstance(data, torch.Tensor): + elif is_torch_tensor(data): data = data.detach().cpu().numpy() if not isinstance(data, np.ndarray): @@ -238,7 +238,7 @@ def _coerce_to_ndarray(value: Any) -> Optional[np.ndarray]: return None if hasattr(data, "convert"): # PIL.Image.Image data = np.array(data.convert("RGB")) - elif isinstance(data, torch.Tensor): + elif is_torch_tensor(data): data = data.detach().cpu().numpy() try: arr = np.asarray(data) diff --git a/recordstream/outputs.py b/recordstream/outputs.py index 7b5e73b..b557363 100644 --- a/recordstream/outputs.py +++ b/recordstream/outputs.py @@ -26,11 +26,10 @@ class ids. That guess is not hypothetical: two independently-written detector wr the contracts. """ -from typing import Generic, List, TypedDict, TypeVar +from typing import TYPE_CHECKING, Generic, List, TypedDict, TypeVar -import torch -import torch.nn.functional as F -from torch import Tensor +if TYPE_CHECKING: # torch is imported inside the BUILDERS — the contracts are typing-only + from torch import Tensor #: The array type a contract carries — ``torch.Tensor``, ``np.ndarray``, a TF/JAX array. ArrayT = TypeVar("ArrayT") @@ -93,7 +92,7 @@ class SegmentationOutput(TypedDict, Generic[ArrayT]): DetectionPredictions = List[DetectionOutput] -def classification_output(logits: Tensor) -> ClassificationOutput[Tensor]: +def classification_output(logits: "Tensor") -> "ClassificationOutput[Tensor]": """Build a full :class:`ClassificationOutput` from raw ``[B, C]`` logits. Example:: @@ -101,13 +100,19 @@ def classification_output(logits: Tensor) -> ClassificationOutput[Tensor]: def predict_step(self, batch, batch_idx): return classification_output(self(x)) # what a predictions sink reads """ + import torch + import torch.nn.functional as F + probs = F.softmax(logits, dim=-1) class_idx = torch.argmax(logits, dim=-1).to(torch.int64) return ClassificationOutput(logits=logits, probs=probs, class_idx=class_idx) -def segmentation_output(logits: Tensor) -> SegmentationOutput[Tensor]: +def segmentation_output(logits: "Tensor") -> "SegmentationOutput[Tensor]": """Build a full :class:`SegmentationOutput` from raw ``[B, C, H, W]`` logits.""" + import torch + import torch.nn.functional as F + probs = F.softmax(logits, dim=1) mask = torch.argmax(logits, dim=1).to(torch.int64) return SegmentationOutput(logits=logits, probs=probs, mask=mask) diff --git a/recordstream/storage/base.py b/recordstream/storage/base.py index 8caf38c..1d8b90b 100644 --- a/recordstream/storage/base.py +++ b/recordstream/storage/base.py @@ -2,8 +2,8 @@ from typing import Any, Dict, Iterator, Protocol, Self, Tuple, runtime_checkable import numpy as np -import torch +from recordstream._compat import is_torch_tensor from recordstream.items import Record #: Root-attribute format tag stamped on stores written in the record key-group layout. @@ -39,7 +39,7 @@ def to_numpy(data: Any) -> Any: Detaches and moves to CPU first so tensors carrying grad or living on a GPU convert cleanly. Non-tensor values pass through unchanged. """ - if isinstance(data, torch.Tensor): + if is_torch_tensor(data): return data.detach().cpu().numpy() return data @@ -85,7 +85,7 @@ def split_attrs(attrs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: plain: Dict[str, Any] = {} arrays: Dict[str, Any] = {} for key, value in attrs.items(): - if isinstance(value, (np.ndarray, torch.Tensor)): + if isinstance(value, np.ndarray) or is_torch_tensor(value): arrays[key] = to_numpy(value) elif isinstance(value, np.generic): plain[key] = value.item() diff --git a/tests/test_optional_torch.py b/tests/test_optional_torch.py new file mode 100644 index 0000000..53ab9ef --- /dev/null +++ b/tests/test_optional_torch.py @@ -0,0 +1,194 @@ +"""torch is an EXTRA — the engine's core is numpy. + +The claim is not "recordstream avoids torch" (`ToTensor` and the output builders need it); it is +that **importing recordstream imports no ML framework**, so a Keras-only, TensorFlow-only or +plain-numpy consumer does not install ~2GB it never calls. + +The load-bearing test here is :func:`test_importing_recordstream_works_with_torch_blocked`, which +proves it in a SUBPROCESS with torch made unimportable. The rest guard the three mechanisms that +make that true, each of which was a real coupling before: the `Dataset` base class, the eager +`ToTensor` export, and the module-level `isinstance(x, torch.Tensor)` checks. +""" + +import subprocess +import sys +import textwrap + +import numpy as np +import pytest + +from recordstream import Stream +from recordstream._compat import is_torch_tensor +from recordstream.core import MapStyle + +# --------------------------------------------------------------------------- # +# The whole claim, executed +# --------------------------------------------------------------------------- # + +_IMPORT_WITH_TORCH_BLOCKED = textwrap.dedent( + """ + import sys + + class _NoTorch: + \"\"\"A meta-path finder that makes torch unimportable, simulating an install without it.\"\"\" + + def find_spec(self, name, path=None, target=None): + if name == "torch" or name.startswith("torch."): + raise ImportError("torch is not installed (blocked by this test)") + return None + + sys.meta_path.insert(0, _NoTorch()) + + import recordstream + from recordstream import Stream, LabelMap, collate_records, multi_hot, batch_values + + # Not just the package: the surfaces a non-torch backend actually uses must work too. + stream = Stream(source=[{"class": 1}, {"class": 0}]) + assert len(stream) == 2 + assert [r["class"] for r in stream] == [1, 0] + + leaked = sorted(m for m in sys.modules if m == "torch" or m.startswith("torch.")) + print("LEAKED:" + ",".join(leaked) if leaked else "CLEAN") + """ +) + + +def test_importing_recordstream_works_with_torch_blocked(tmp_path: object) -> None: + """`import recordstream` must not need torch, and must not import it as a side effect. + + Run in a subprocess because this process has already imported torch — the check is only + meaningful in an interpreter where it was never available. `cwd` is a temp dir so the import + resolves to the INSTALLED package, not a same-named source directory (PEP 420 shadowing). + """ + result = subprocess.run( + [sys.executable, "-c", _IMPORT_WITH_TORCH_BLOCKED], + capture_output=True, + text=True, + cwd=str(tmp_path), + ) + + assert result.returncode == 0, f"importing recordstream without torch failed:\n{result.stderr}" + assert "CLEAN" in result.stdout, f"torch was imported anyway: {result.stdout.strip()}" + + +# --------------------------------------------------------------------------- # +# 1. The `Dataset` base — dropped, because a DataLoader never needed it +# --------------------------------------------------------------------------- # + + +def test_stream_does_not_inherit_torchs_dataset() -> None: + """The regression guard: re-adding the base would silently make torch mandatory again. + + Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base + bought nothing but the import. + """ + import torch.utils.data + + assert not issubclass(Stream, torch.utils.data.Dataset) + + +def test_stream_is_map_style_which_is_all_a_dataloader_needs() -> None: + """`DataLoader` duck-types its argument — `__len__` + `__getitem__` IS the contract.""" + stream = Stream(source=[{"class": i} for i in range(4)]) + + assert isinstance(stream, MapStyle) + assert len(stream) == 4 and stream[2]["class"] == 2 + + +def test_a_dataloader_actually_accepts_a_stream() -> None: + """The duck-typing claim, executed rather than asserted about. + + `cast(Any, ...)` at the call site is the accepted cost: torch's STUB still declares + `Dataset[T]`, so type checkers reject the runtime-valid call. + """ + from typing import Any, cast + + import torch.utils.data + + stream = Stream(source=[{"class": i} for i in range(4)]) + loader = torch.utils.data.DataLoader(cast(Any, stream), batch_size=2) + + assert len(list(loader)) == 2 + + +# --------------------------------------------------------------------------- # +# 2. `ToTensor` — reachable, but never eagerly imported +# --------------------------------------------------------------------------- # + + +def test_to_tensor_is_not_imported_eagerly_but_is_still_reachable() -> None: + import recordstream.ops as ops + + assert "ToTensor" in ops._OPTIONAL_OPS, "the lazy export must stay registered" + assert ops.ToTensor is not None + assert "ToTensor" in dir(ops), "TAB completion / dir() must still advertise it" + + +def test_a_missing_optional_op_names_the_extra_to_install(monkeypatch: pytest.MonkeyPatch) -> None: + """The error an operator without the extra sees — not a traceback from three libraries down.""" + import importlib + + import recordstream.ops as ops + + def _fail(name: str) -> None: + raise ImportError("No module named 'torch'") + + monkeypatch.setattr(importlib, "import_module", _fail) + + with pytest.raises(ImportError, match=r"recordstream\[torch\]"): + ops.__getattr__("ToTensor") + + +def test_an_unknown_attribute_still_raises_attribute_error() -> None: + """The lazy `__getattr__` must not turn every typo into an ImportError.""" + import recordstream.ops as ops + + with pytest.raises(AttributeError): + ops.__getattr__("NoSuchOp") + + +# --------------------------------------------------------------------------- # +# 3. Recognising a tensor without importing torch +# --------------------------------------------------------------------------- # + + +def test_is_torch_tensor_recognizes_a_real_tensor() -> None: + import torch + + assert is_torch_tensor(torch.zeros(3)) + + +@pytest.mark.parametrize("value", [np.zeros(3), [1, 2, 3], "not a tensor", None, 7]) +def test_is_torch_tensor_rejects_everything_else(value: object) -> None: + assert not is_torch_tensor(value) + + +def test_is_torch_tensor_is_false_when_torch_was_never_imported(monkeypatch: pytest.MonkeyPatch) -> None: + """The mechanism: a torch tensor cannot exist unless torch is loaded, so `sys.modules` is EXACT. + + With torch hidden the answer must be False without any attempt to import it — which is what + makes this usable at module level in a package that does not depend on torch. + """ + import torch + + monkeypatch.delitem(sys.modules, "torch") + tensor = torch.zeros(3) # a real tensor, while the module is hidden + + assert not is_torch_tensor(tensor) + + +# --------------------------------------------------------------------------- # +# 4. The packaging half — the code above is only true if the metadata agrees +# --------------------------------------------------------------------------- # + + +def test_pyproject_declares_torch_as_an_extra_not_a_dependency() -> None: + import tomllib + from pathlib import Path + + pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" + project = tomllib.loads(pyproject.read_text())["project"] + + required = [d for d in project["dependencies"] if d.split(">")[0].split("=")[0].strip() == "torch"] + assert not required, f"torch must not be a hard dependency, found: {required}" + assert any("torch" in d for d in project["optional-dependencies"]["torch"]) From 61f7f15fb6a995740c40d83a71093d4e5bdb5214 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 30 Jul 2026 13:54:22 +0200 Subject: [PATCH 062/102] chore(tests): type-ignore the deliberately invalid class_names call The test asserts the RUNTIME check rejects non-string names, so mypy was rejecting the very call under test. Pre-existing; surfaced by running the gate for the torch change. --- tests/test_labels.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_labels.py b/tests/test_labels.py index 613afdf..583830e 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -439,7 +439,9 @@ def test_a_stream_rejects_non_string_names_at_construction() -> None: from recordstream import Stream with pytest.raises(Exception, match="valid string"): - Stream(source=[], class_names=[1, 2]) + # The wrong type is the POINT — this asserts the runtime check, which is what a caller + # building a Stream from YAML gets. mypy would otherwise reject the very call under test. + Stream(source=[], class_names=[1, 2]) # type: ignore[list-item] # --------------------------------------------------------------------------- # From 8d71d3da6edf7abe675e907fad196ec80c7915ae Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 30 Jul 2026 13:54:22 +0200 Subject: [PATCH 063/102] chore(ci): regenerate pipelines with the framework extras MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated by `aisland jenkins scaffold --force`. CI now installs `.[dev,torch]` — the project's declared framework extras intersected with the committed selection — instead of a bare `.[dev]`, which after the extras split would have tested a torch-free install. --- .github/workflows/ci.yml | 8 ++++---- Jenkinsfile | 2 +- Jenkinsfile.local | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c079cdf..75773b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch]" - name: Run Isort run: isort --check-only . - name: Run Black @@ -60,7 +60,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch]" - name: Run Tests run: | if [ -d tests ] && find tests -name '*.py' | grep -q .; then @@ -94,7 +94,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch]" - name: Run Examples run: | for f in examples/*.py; do @@ -132,7 +132,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. uv pip install --system -e ".[notebook]" 2>/dev/null || echo "No [notebook] extras declared; continuing with defaults." diff --git a/Jenkinsfile b/Jenkinsfile index f39d72b..bace0b6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { // (Gearlux distribution names are intentionally unpublished on PyPI). sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" - sh "${VENV_BIN}/uv pip install -e .[dev]" + sh "${VENV_BIN}/uv pip install -e .[dev,torch]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships // notebooks; absence is not an error. diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 9e1b046..b26ad01 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -45,7 +45,7 @@ pipeline { sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/confluid/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/liquifai/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" - sh "${VENV_BIN}/uv pip install -e .[dev]" + sh "${VENV_BIN}/uv pip install -e .[dev,torch]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships // notebooks; absence is not an error. From c4e36387a8dabd1573487f124c9312eb09c3a156 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 30 Jul 2026 15:37:33 +0200 Subject: [PATCH 064/102] =?UTF-8?q?feat:=20recordstream.keras=20=E2=80=94?= =?UTF-8?q?=20the=20KERAS=5FBACKEND=20boundary=20and=20the=20RecordSequenc?= =?UTF-8?q?e=20adapter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keras 3 reads KERAS_BACKEND at import time and defaults to tensorflow, which `recordstream[keras]` does not install. The new module owns that ordering: it setdefaults to the first backend actually present (find_spec, so nothing is imported to look), exposes the one `keras` handle consumers import through, and carries `RecordSequence` — the PyDataset a DataLoader would have been (row order, slicing, per-epoch reshuffle, collate_records). It lives here rather than in a training project because ~60% of it never mentions a task, and its torch twin is a single LazyClass(DataLoader, collate_fn=...) line. What a batch BECOMES stays the caller's, handed in via `transform=` exactly as a collate_fn is handed to a DataLoader. Also: `keras` is an extra naming NO compute engine (Keras 3 is an API, not a runtime) and is declared under [tool.aisland] frameworks, plus the torch-optional coverage in tests/test_optional_torch.py and the architecture record behind both. --- .github/workflows/ci.yml | 12 +- AGENTS.md | 3 +- Jenkinsfile | 6 +- Jenkinsfile.local | 6 +- README.md | 15 ++- docs/architecture.md | 88 ++++++++++++ docs/kinds.md | 20 +++ pyproject.toml | 12 +- recordstream/keras.py | 197 +++++++++++++++++++++++++++ tests/test_keras_sequence.py | 253 +++++++++++++++++++++++++++++++++++ tests/test_optional_torch.py | 53 ++++++++ 11 files changed, 644 insertions(+), 21 deletions(-) create mode 100644 recordstream/keras.py create mode 100644 tests/test_keras_sequence.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75773b6..e63c3bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ # ========================================================================= # AUTO-GENERATED FILE — DO NOT EDIT BY HAND -# Generated by: aisland jenkins scaffold --project recordstream +# Generated by: aisland jenkins scaffold recordstream # Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -# To regenerate: aisland jenkins scaffold --project recordstream --force +# To regenerate: aisland jenkins scaffold recordstream --force # ========================================================================= name: Recordstream CI @@ -31,7 +31,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev,torch]" + uv pip install --system -e ".[dev,torch,keras]" - name: Run Isort run: isort --check-only . - name: Run Black @@ -60,7 +60,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev,torch]" + uv pip install --system -e ".[dev,torch,keras]" - name: Run Tests run: | if [ -d tests ] && find tests -name '*.py' | grep -q .; then @@ -94,7 +94,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev,torch]" + uv pip install --system -e ".[dev,torch,keras]" - name: Run Examples run: | for f in examples/*.py; do @@ -132,7 +132,7 @@ jobs: # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main - uv pip install --system -e ".[dev,torch]" + uv pip install --system -e ".[dev,torch,keras]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. uv pip install --system -e ".[notebook]" 2>/dev/null || echo "No [notebook] extras declared; continuing with defaults." diff --git a/AGENTS.md b/AGENTS.md index 87eaad7..38b1414 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). -- **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and marainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch"]`, so `aisland framework set torch` installs `recordstream[dev,torch]` and a selection without it installs `[dev]`; the same committed selection is what generated CI installs. Pins: `tests/test_optional_torch.py`. +- **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and marainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. +- **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. - **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. diff --git a/Jenkinsfile b/Jenkinsfile index bace0b6..c6c71e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,8 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project recordstream +// Generated by: aisland jenkins scaffold recordstream // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project recordstream --force +// To regenerate: aisland jenkins scaffold recordstream --force // ========================================================================= pipeline { agent any @@ -35,7 +35,7 @@ pipeline { // (Gearlux distribution names are intentionally unpublished on PyPI). sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" - sh "${VENV_BIN}/uv pip install -e .[dev,torch]" + sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships // notebooks; absence is not an error. diff --git a/Jenkinsfile.local b/Jenkinsfile.local index b26ad01..41995f0 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -1,8 +1,8 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project recordstream +// Generated by: aisland jenkins scaffold recordstream // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project recordstream --force +// To regenerate: aisland jenkins scaffold recordstream --force // ========================================================================= pipeline { agent { @@ -45,7 +45,7 @@ pipeline { sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/confluid/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/liquifai/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" - sh "${VENV_BIN}/uv pip install -e .[dev,torch]" + sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships // notebooks; absence is not an error. diff --git a/README.md b/README.md index e46a159..58bfe0c 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit | Page | Covers | |---|---| | [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | -| [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), 1→N expanding ops | +| [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), the Keras `RecordSequence` adapter, 1→N expanding ops | | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | @@ -124,6 +124,7 @@ RecordStream is designed to sit between your data catalog and your training loop - **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability (see [docs/sources.md](docs/sources.md)). - **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node — including bare library transforms — every run reproducible. - **PyTorch**: `Stream` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#batching--collate_records--the-collate-registry-recordstreamcollate) (`collate_records` is the default). +- **Keras 3**: no `DataLoader` exists to do the batching, so [`RecordSequence`](docs/kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you) is the `keras.utils.PyDataset` half — row order, slicing, per-epoch reshuffle, `collate_records` — and a `transform` callable supplies the batch shape, exactly as `collate_fn` does for torch. - **Augmentation libraries**: [albumentations](https://albumentations.ai) and torchvision `transforms.v2` transforms run **as-is** in any ops list — the engine speaks each library's native convention (kwarg vocabulary vs dict walk), so there is nothing to wrap (see [docs/augmentation.md](docs/augmentation.md)). ## 🔧 Installation @@ -132,15 +133,19 @@ RecordStream is designed to sit between your data catalog and your training loop pip install git+https://github.com/Gearlux/recordstream.git@main ``` -The core engine is **numpy**, and installs no ML framework. PyTorch is an extra, needed only for -the pieces that genuinely produce tensors — the `ToTensor` op and the `classification_output` / -`segmentation_output` builders: +The core engine is **numpy**, and installs no ML framework. A framework arrives only with the extra +that needs it: + +| Extra | Provides | +|---|---| +| `torch` | The pieces that genuinely produce tensors — the `ToTensor` op, `batch_tensor`, and the `classification_output` / `segmentation_output` builders | +| `keras` | `recordstream.keras` — the `RecordSequence` `PyDataset` adapter and the `KERAS_BACKEND` ordering. Keras 3 is an API, so this names no compute engine; it runs on whichever of torch / TensorFlow / JAX you have | ```bash pip install "recordstream[torch] @ git+https://github.com/Gearlux/recordstream.git@main" ``` -Everything else works without it. A `Stream` is map-style (`__len__`/`__getitem__`), so a +Everything else works without either. A `Stream` is map-style (`__len__`/`__getitem__`), so a `DataLoader` still accepts one directly on a torch install; `batch_values`, `multi_hot` and the class-balance statistics return numpy, so a non-torch backend converts in one line. Reaching for `recordstream.ops.ToTensor` without the extra raises an `ImportError` naming it. diff --git a/docs/architecture.md b/docs/architecture.md index a6f7781..1318875 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -932,3 +932,91 @@ def __getattr__(name): next backend has to reimplement it. - **Installation** is documented in the README's Installation section; what each extra provides is the `pyproject.toml` comment beside it. + +## 10. The framework's batching half lives beside the collate (`recordstream.keras`, 2026-07-30) + +### Context + +Batching a record source has two halves: **what one batch contains** (recordstream's +`collate_records`) and **which rows go in which batch** (the row order, the slicing, the short +final batch, the per-epoch reshuffle). For torch, the second half is free — a `DataLoader` does it, +duck-typing any `MapStyle` source (§9) and taking `collate_fn=collate_records` for the first half. +So recordstream shipped only half of the pair, and nobody noticed the other half was missing. + +Keras 3 has no `DataLoader`. `keras.utils.PyDataset.__getitem__` must return a whole BATCH, so a +consumer has to write that loop itself. The first one did, in a training project — a +`RecordSequence(keras.utils.PyDataset)` inside an image classifier — and the result read as if the +adapter were part of the task. It was not: sixty percent of it (row order, `np.arange`, the rng, +`on_epoch_end`, `collate_records([source[i] for i in rows])`) mentioned nothing about +classification, while its torch twin in the same project was a single +`LazyClass(DataLoader, shuffle=True, collate_fn=collate_records)` line. A second Keras consumer +would have copied the file. + +### Decision + +**`recordstream.keras.RecordSequence` owns the DataLoader half; the consumer passes the batch +shape in.** The split is drawn exactly where torch draws it: `transform` is the `collate_fn` +equivalent, a callable mapping one collated record to what the model consumes. With no +`transform`, `__getitem__` hands over the batched record — the identity, which is also what +`batches()` yields for pairing per-record predictions with per-record metadata. + +The module also owns the **`KERAS_BACKEND` ordering**, which is why it exists as one module rather +than a class dropped somewhere. Keras 3 reads that variable at import time and defaults to +`tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the compute engine is +the operator's choice), so a bare `import keras` dies inside `keras.src.tree.optree_impl` with +`ModuleNotFoundError: No module named 'tensorflow'`. A `setdefault` to the first backend actually +present — probed with `find_spec`, so nothing is imported just to look — has to run in the LOWEST +layer that imports keras: import sorters put a library import above a first-party one, so a +consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and would lose the +race. + +### Consequences + +- **`RecordSequence` is absent from the package root, deliberately.** `inspect.getmembers` — what + `discovery.scan_module` and the GUI bridges call — getattrs every name a module advertises, so a + PEP 562 lazy export at the root (the `recordstream.ops.ToTensor` pattern) would import keras on + every discovery scan of a torch-only install. The import path is the boundary marker: + `from recordstream.keras import RecordSequence`. +- **It is not `@configurable` and carries no discovery `category`.** It is engine plumbing a + runnable builds in code, like `collate_records`; tagging it would put a keras import in the + registry scan for a class no YAML wires. +- **The row order is a lazy `@property`, not constructor state.** `len(source)` is real work for a + deferred source (a `HuggingFaceSource` LOADS its dataset to answer it), and recordstream + constructors do none — so `RecordSequence()` builds zero-arg and a missing `source` is reported + by `indices` with a clear message. +- **A consumer's keras imports now route through recordstream.** A training project keeps its own + one-line shim for spelling, but the ordering rule has one home; a project that imports keras + ahead of `recordstream.keras` reintroduces the TensorFlow failure. +- **The extra names no compute engine.** `keras = ["keras>=3.0"]` only; torch/TF/jax come from + whichever consumer extra selected one, and `_first_installed_backend` adapts to what is there. + +### Example + +```python +# The consumer supplies the SHAPE; the engine supplies the batching. +from recordstream.keras import RecordSequence + +def to_xy(batch): # the classification decision, 3 lines + x = np.asarray(batch_values(batch, "image"), dtype="float32") + return x, np.asarray(batch_values(batch, "class"), dtype="int64") + +seq = RecordSequence(stream, batch_size=32, shuffle=True, transform=to_xy) +model.fit(seq, epochs=3) + +# ...and the torch twin, for the symmetry this restores: +loader = DataLoader(cast(Any, stream), batch_size=32, shuffle=True, collate_fn=collate_records) +``` + +### What you may change (and where it's documented) + +- **Add another framework's batching adapter** (a JAX/`grain` sampler, a TF `tf.data` generator): + a sibling module behind its own extra, same split — the engine owns row order + collate, the + caller owns the batch shape via a `transform`-shaped parameter. Do not grow `RecordSequence` a + framework switch. +- **`PyDataset`'s prefetch knobs are already declared** (`workers` / `use_multiprocessing` / + `max_queue_size`, forwarded to `super().__init__()` at Keras's own defaults). Any further + passthrough follows the same rule — a named, defaulted, `Args:`-documented parameter, never a + `**kwargs` escape hatch, per the declared-parameter mandate (§6). Pinned by + `test_every_knob_is_a_declared_parameter`. +- **Usage** is [docs/kinds.md](kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you); + what the extra provides is the `pyproject.toml` comment beside it. diff --git a/docs/kinds.md b/docs/kinds.md index bf4d496..198cedc 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -79,6 +79,26 @@ batch_metadata(batch, exclude=("image", "class")) # the remaining columns Only `batch_tensor` is torch; the rest return plain values or numpy, so a non-torch backend reuses them and converts in one line. `dtype` is a parameter, not an opinion — the same knob as `device`. What stays task-side is only WHICH call a trainer makes. +### Keras: `RecordSequence` — the batching half the framework leaves to you + +A `DataLoader` needs one thing from recordstream (`collate_fn=collate_records`) and does the rest itself: row order, batch slicing, the short final batch, the per-epoch reshuffle. Keras 3 has no `DataLoader` — `keras.utils.PyDataset.__getitem__` must return a whole **batch** — so that loop is `RecordSequence`, in `recordstream.keras`: + +```python +from recordstream.keras import RecordSequence + +def to_xy(batch): # your collate_fn equivalent + return batch_values(batch, "image"), batch_values(batch, "class") + +train = RecordSequence(stream, batch_size=32, shuffle=True, transform=to_xy) +model.fit(train, epochs=3) # reshuffles between epochs itself +``` + +`PyDataset`'s prefetch knobs are declared parameters, at Keras's own defaults — pass `workers=4` to overlap a slow record walk (decode, resize, remote read) with the training step, `max_queue_size=` to cap how many prefetched batches may wait, and `use_multiprocessing=True` only when the per-record work is GIL-bound (each process re-pickles the sequence and its source). + +`transform` is the whole task-facing surface: it maps one collated record to what the model consumes, so the shape decision stays in your code exactly as it does with a `DataLoader`. Omit it and `__getitem__` hands over the batched record itself — which is also what `seq.batches()` yields, the pairing half of prediction (a model emits `[N, ...]` while a [`PredictionsSink`](predictions.md) writes per record, so you need the batch its output came from to read that batch's [`batch_metadata`](#reading-a-batch-back-recordstreambatch)). + +Needs the extra — `pip install "recordstream[keras]"`. Importing `recordstream.keras` is also what sets `KERAS_BACKEND` (Keras reads it at import time and would otherwise default to TensorFlow, which this extra does not install), so it must be the first keras-touching import in a process; never `import keras` ahead of it. Why the adapter lives here rather than in a training project is recorded in [architecture.md](architecture.md#10-the-frameworks-batching-half-lives-beside-the-collate-2026-07-30). + ## 1→N expanding ops (iterable-only pipelines) diff --git a/pyproject.toml b/pyproject.toml index f71b15e..3d51ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,11 @@ requires-python = ">=3.12" # `_compat.is_torch_tensor`, which consults sys.modules rather than importing. So a # consumer on Keras/JAX installs recordstream without a 2GB framework it never calls. torch = ["torch"] +# The Keras-shaped surface: `recordstream.keras` (the KERAS_BACKEND ordering + `RecordSequence`, +# the `PyDataset` a `DataLoader` would have been). Keras 3 is an API, not a runtime, so this +# names NO compute engine — `recordstream.keras` defaults KERAS_BACKEND to whichever of +# torch/tensorflow/jax is actually installed, and the consumer's own extra picks one. +keras = ["keras>=3.0"] dev = [ "black>=24.0.0,<25.0.0", "isort>=5.13.0,<6.0.0", @@ -121,10 +126,11 @@ recordstream = "recordstream.cli:app" # Which of this project's extras are ML FRAMEWORKS, for `aisland framework`. Selecting a name # here is what makes `aisland setup` add the matching extra to this project's install. # -# The core engine is numpy — `torch` powers only `ToTensor` and the output builders, so a -# workspace that selects no framework still gets a fully working record engine. +# The core engine is numpy — `torch` powers only `ToTensor` and the output builders, and `keras` +# only `recordstream.keras`, so a workspace that selects no framework still gets a fully working +# record engine. [tool.aisland] -frameworks = ["torch"] +frameworks = ["torch", "keras"] [tool.setuptools.packages.find] where = ["."] diff --git a/recordstream/keras.py b/recordstream/keras.py new file mode 100644 index 0000000..0c6d3de --- /dev/null +++ b/recordstream/keras.py @@ -0,0 +1,197 @@ +"""The Keras boundary: the backend default, the ``keras`` handle, and the batching adapter. + +Three things live here, in the order they have to happen. + +**1. The backend default.** Keras 3 reads ``KERAS_BACKEND`` **at import time** and defaults to +``tensorflow``, which ``recordstream[keras]`` does not install (Keras 3 is an API, not a runtime — +it needs exactly one of torch / TensorFlow / JAX underneath, and which one is the operator's +choice). A bare ``import keras`` on a torch-backed install therefore dies with +``ModuleNotFoundError: No module named 'tensorflow'`` raised from inside +``keras.src.tree.optree_impl`` — a traceback that says nothing about what to do. So this module +sets a default the install can actually honour, the first backend that is PRESENT, and only when +the operator has not chosen: ``setdefault`` never overrides an explicit ``KERAS_BACKEND=jax``. + +**2. The ``keras`` handle.** Consumers import keras THROUGH here (``from recordstream.keras import +keras``) so the ordering above cannot be got wrong by an import that happens to land first — which +is a real hazard, because import sorters group a library import ahead of a first-party one. + +**3. The batching adapter.** For torch, recordstream supplies one function and the framework does +the rest: hand :func:`~recordstream.collate.collate_records` to a ``DataLoader`` as its +``collate_fn`` and torch owns the row order, the batch slicing and the per-epoch reshuffle (a +``Stream`` is map-style, which is all a ``DataLoader`` needs — see +:class:`~recordstream.core.MapStyle`). Keras 3 has no ``DataLoader``: +``keras.utils.PyDataset.__getitem__`` must return a whole BATCH, so somebody has to write that +loop. :class:`RecordSequence` is that loop and nothing else — row order, slicing, reshuffle, +``collate_records`` — the DataLoader half, kept beside the collate half it calls instead of +re-appearing in every training project. + +What a batch BECOMES stays the caller's, exactly as ``collate_fn`` is the caller's on the torch +side: ``transform`` maps one collated record to what the model consumes, so task shapes (a +classifier's ``(x, y)`` tuple, a multi-input model's dict) never enter this module. + +Import it by PATH — ``RecordSequence`` is deliberately absent from the package root, because +``inspect.getmembers`` (what :func:`recordstream.discovery.scan_module` and the GUI bridges use) +getattrs every name a module advertises, so a lazy root export would import keras on every +discovery scan of a torch-only install:: + + from recordstream.keras import RecordSequence + + seq = RecordSequence(stream, batch_size=32, shuffle=True, transform=to_xy) + model.fit(seq, epochs=3) +""" + +import importlib.util +import os +from typing import Any, Callable, Iterator, Optional, cast + +import numpy as np + +from recordstream.collate import collate_records +from recordstream.core import MapStyle +from recordstream.items import Record + +#: Compute backends Keras 3 can run on, in the order this package prefers them. torch first +#: because it is the one every other recordstream extra already implies; the rest are honoured +#: whenever an operator has them installed. +_BACKENDS = ("torch", "tensorflow", "jax") + + +def _first_installed_backend() -> str: + """The first of :data:`_BACKENDS` actually importable, else Keras's own default. + + Picking a backend that is PRESENT rather than one we assume: hard-coding ``torch`` would fail + on a TensorFlow-only install exactly as Keras's own ``tensorflow`` default fails on the + torch-only install this exists to fix. Uses ``find_spec`` so nothing is imported just to look, + which matters because it runs at import time of this module. + + Example:: + + os.environ.setdefault("KERAS_BACKEND", _first_installed_backend()) # "torch" here + """ + for name in _BACKENDS: + if importlib.util.find_spec(name) is not None: + return name + return "tensorflow" # Keras's own default — let IT raise, naming the package to install + + +#: Set BEFORE keras is imported, and only if unset — an explicit choice always wins. +os.environ.setdefault("KERAS_BACKEND", _first_installed_backend()) + +import keras # noqa: E402 - deliberately after the environment default above + +__all__ = ["RecordSequence", "keras", "keras_backend"] + + +def keras_backend() -> str: + """The Keras backend actually in effect (``torch`` / ``tensorflow`` / ``jax``). + + Reads Keras rather than the environment variable, so it reports what is LOADED — Keras + consults ``KERAS_BACKEND`` once at import, and changing it afterwards has no effect. Worth + logging at the start of a run whose engine is selectable. + """ + return str(keras.backend.backend()) + + +class RecordSequence(keras.utils.PyDataset): + """A map-style record source as a ``keras.utils.PyDataset`` of collated batches. + + The DataLoader half of Keras batching: row order, batch slicing, per-epoch reshuffle, and + :func:`~recordstream.collate.collate_records`. It is deliberately task-blind — ``transform`` + is the caller's ``collate_fn``-equivalent and decides what the model actually receives. + + Args: + source: Any map-style record source (a ``Stream`` is one). + batch_size: Rows per batch. A short final batch is yielded as-is, never padded. + shuffle: Reshuffle the row order at the end of every epoch (training). + seed: Shuffle seed, so a shuffled run is reproducible. + transform: Maps one collated record batch to what the model consumes. ``None`` hands + over the batched record itself. + workers: ``PyDataset`` prefetch workers. ``1`` (Keras's own default) loads batches on + the calling thread; higher values overlap the record walk with the training step, + which is what a slow source (decode, resize, remote read) needs. + use_multiprocessing: Run those workers as PROCESSES instead of threads. Each one + re-pickles this object and its source, so it is the wrong default for a source + holding an open handle — reach for it only when the per-record work is + GIL-bound and does not parallelize with threads. + max_queue_size: How many prefetched batches may wait. The memory ceiling of + prefetching: batches are held whole, so a large value on large batches is a + real footprint. + """ + + def __init__( + self, + source: Optional[MapStyle] = None, + batch_size: int = 32, + shuffle: bool = False, + seed: int = 0, + transform: Optional[Callable[[Record], Any]] = None, + workers: int = 1, + use_multiprocessing: bool = False, + max_queue_size: int = 10, + ) -> None: + # Keras's own defaults, restated so they are DECLARED parameters rather than reachable + # only through `**kwargs` — the every-knob-is-a-declared-parameter rule (architecture + # §6): a form/schema generator enumerates the signature, and an undeclared knob is + # invisible to it. + super().__init__(workers=workers, use_multiprocessing=use_multiprocessing, max_queue_size=max_queue_size) + self.source = source + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + self.seed = int(seed) + self.transform = transform + self._rng = np.random.default_rng(seed) + self._indices: Optional[np.ndarray] = None + + @property + def indices(self) -> np.ndarray: + """Row order for the current epoch — built on first use, reshuffled by ``on_epoch_end``. + + Built lazily rather than in the constructor because ``len(source)`` is real work for a + deferred source (a ``HuggingFaceSource`` loads its dataset to answer it), and recordstream + constructors do none. It is also where a missing ``source`` is reported, so zero-arg + construction stays possible. + """ + if self._indices is None: + if self.source is None: + raise RuntimeError("RecordSequence: no 'source' to batch — wire the dataset first.") + order = np.arange(len(self.source)) + if self.shuffle: + self._rng.shuffle(order) + self._indices = order + return self._indices + + def __len__(self) -> int: + """Number of batches — Keras asks once per epoch.""" + return int(np.ceil(len(self.indices) / self.batch_size)) + + def batch(self, index: int) -> Record: + """The collated record batch at ``index``, BEFORE ``transform``.""" + rows = self.indices[index * self.batch_size : (index + 1) * self.batch_size] + source = cast(MapStyle, self.source) # the `indices` read above validated it + # Declared local, not a bare return: `@register_collate` types every registered collate + # as `CollateFn = Callable[[Sequence[Any]], Any]` — deliberately loose, because the + # registry holds task collates with divergent conventions — which erases + # `collate_records`' own `-> Record` at the call site. + collated: Record = collate_records([source[int(i)] for i in rows]) + return collated + + def batches(self) -> Iterator[Record]: + """Every collated batch, in the current epoch's order. + + The pairing half of prediction: a model emits ``[N, ...]`` while a + :class:`~recordstream.predictions.PredictionsSink` writes per record, so the caller needs + the batch its output came from to read that batch's metadata back + (:func:`~recordstream.batch.batch_metadata`). + """ + for index in range(len(self)): + yield self.batch(index) + + def __getitem__(self, index: int) -> Any: + """What Keras feeds the model: the collated batch, mapped by ``transform``.""" + batch = self.batch(index) + return batch if self.transform is None else self.transform(batch) + + def on_epoch_end(self) -> None: + """Reshuffle between epochs when training. Keras calls this itself.""" + if self.shuffle: + self._rng.shuffle(self.indices) diff --git a/tests/test_keras_sequence.py b/tests/test_keras_sequence.py new file mode 100644 index 0000000..2db802c --- /dev/null +++ b/tests/test_keras_sequence.py @@ -0,0 +1,253 @@ +"""`RecordSequence` — the DataLoader half Keras leaves to the caller. + +These pin the plumbing and NOTHING about a task: row order, batch slicing, the short final +batch, the per-epoch reshuffle, and that `transform` is what decides the model's input shape. +A test here that mentioned classes, `(x, y)` tuples or multi-hot targets would mean the task +had leaked back into the engine. + +Skipped as a module when keras is absent — importing `recordstream.keras` is also what sets +`KERAS_BACKEND`, so this must be the first keras-touching import in the process. +""" + +import importlib.util +import os +from typing import Any + +import numpy as np +import pytest + +keras_module = pytest.importorskip("recordstream.keras") +RecordSequence = keras_module.RecordSequence + +from recordstream import Image, Label, Stream # noqa: E402 - after the keras availability gate + + +def _records(n: int = 10) -> list: + return [ + {"image": Image(np.full((4, 4, 3), i, dtype="uint8"), layout="HWC"), "class": Label(i), "idx": i} + for i in range(n) + ] + + +class _CountingSource: + """A map-style source that records how often it is measured and indexed.""" + + def __init__(self, records: list) -> None: + self.records = records + self.len_calls = 0 + self.reads: list = [] + + def __len__(self) -> int: + self.len_calls += 1 + return len(self.records) + + def __getitem__(self, index: int) -> Any: + self.reads.append(index) + return self.records[index] + + +# --------------------------------------------------------------------------- # +# The default: collated record batches, no task shape at all +# --------------------------------------------------------------------------- # + + +def test_without_a_transform_a_batch_is_the_collated_record() -> None: + """`transform=None` is the identity, so the class is usable — and testable — task-free.""" + seq = RecordSequence(Stream(source=_records(8)), batch_size=4) + batch = seq[0] + + assert set(batch) == {"image", "class", "idx"} + assert np.asarray(batch["image"]).shape == (4, 4, 4, 3) + assert batch["idx"] == [0, 1, 2, 3] + + +def test_the_transform_decides_what_keras_receives() -> None: + """The `collate_fn` equivalent: the batch shape is the caller's decision, not this class's.""" + seq = RecordSequence(Stream(source=_records(4)), batch_size=2, transform=lambda b: ("shaped", b["idx"])) + + assert seq[0] == ("shaped", [0, 1]) + + +def test_the_transform_sees_the_same_batch_as_batch() -> None: + seen: list = [] + seq = RecordSequence(Stream(source=_records(4)), batch_size=2, transform=lambda b: seen.append(b)) + seq[1] + + assert seen[0]["idx"] == seq.batch(1)["idx"] + + +# --------------------------------------------------------------------------- # +# Batching — what a DataLoader would have done +# --------------------------------------------------------------------------- # + + +def test_the_last_batch_is_short_not_padded() -> None: + seq = RecordSequence(Stream(source=_records(10)), batch_size=4) + + assert len(seq) == 3 + assert len(seq[2]["idx"]) == 2 + + +def test_batch_size_one_yields_one_batch_per_record() -> None: + seq = RecordSequence(Stream(source=_records(5)), batch_size=1) + + assert len(seq) == 5 + assert [b["idx"] for b in seq.batches()] == [[0], [1], [2], [3], [4]] + + +def test_batches_walks_every_batch_in_epoch_order() -> None: + """The pairing half of prediction: the caller needs the batch its output came from.""" + seq = RecordSequence(Stream(source=_records(7)), batch_size=3) + + assert [b["idx"] for b in seq.batches()] == [[0, 1, 2], [3, 4, 5], [6]] + + +# --------------------------------------------------------------------------- # +# Shuffling — the other thing a DataLoader owns +# --------------------------------------------------------------------------- # + + +def test_unshuffled_order_is_the_sources_order() -> None: + seq = RecordSequence(Stream(source=_records(6)), batch_size=6) + + assert seq[0]["idx"] == [0, 1, 2, 3, 4, 5] + + +def test_shuffling_changes_the_order_but_not_the_content() -> None: + records = _records(12) + shuffled = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=1) + + order = shuffled[0]["idx"] + assert order != sorted(order) + assert sorted(order) == list(range(12)) + + +def test_the_seed_makes_a_shuffled_run_reproducible() -> None: + records = _records(12) + a = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=7) + b = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=7) + c = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=8) + + assert a[0]["idx"] == b[0]["idx"] + assert a[0]["idx"] != c[0]["idx"] + + +def test_on_epoch_end_reshuffles_when_shuffling() -> None: + seq = RecordSequence(Stream(source=_records(12)), batch_size=12, shuffle=True, seed=1) + first = seq[0]["idx"] + seq.on_epoch_end() + + assert seq[0]["idx"] != first + + +def test_on_epoch_end_is_a_no_op_when_not_shuffling() -> None: + """Evaluation and prediction depend on this: their row order must survive every epoch.""" + seq = RecordSequence(Stream(source=_records(6)), batch_size=6) + seq.on_epoch_end() + + assert seq[0]["idx"] == [0, 1, 2, 3, 4, 5] + + +# --------------------------------------------------------------------------- # +# Lazy construction — the recordstream constructor rule +# --------------------------------------------------------------------------- # + + +def test_the_constructor_never_touches_the_source() -> None: + """`len(source)` is real work for a deferred source (a HuggingFaceSource LOADS to answer it), + so the row order is built on first use, not in `__init__`.""" + source = _CountingSource(_records(4)) + seq = RecordSequence(source, batch_size=2) + + assert source.len_calls == 0 and source.reads == [] + + len(seq) + assert source.len_calls == 1 + + +def test_zero_arg_construction_works_and_a_missing_source_is_reported_lazily() -> None: + seq = RecordSequence() + + with pytest.raises(RuntimeError, match="source"): + len(seq) + + +def test_indices_are_computed_once() -> None: + source = _CountingSource(_records(8)) + seq = RecordSequence(source, batch_size=4) + len(seq), seq[0], seq[1] + + assert source.len_calls == 1 + + +# --------------------------------------------------------------------------- # +# PyDataset's prefetch knobs — DECLARED, not smuggled through **kwargs +# --------------------------------------------------------------------------- # + + +def test_the_prefetch_knobs_reach_pydataset() -> None: + """They are Keras's, but reachable only if we forward them — and a form/schema generator + reads the SIGNATURE, so `**kwargs` would have made them invisible as well as unreachable.""" + seq = RecordSequence(Stream(source=_records(4)), workers=3, use_multiprocessing=True, max_queue_size=7) + + assert (seq.workers, seq.use_multiprocessing, seq.max_queue_size) == (3, True, 7) + + +def test_the_knob_defaults_are_keras_own() -> None: + """Restated in our signature, so declaring them changes no behaviour for anyone.""" + seq = RecordSequence(Stream(source=_records(4))) + + assert (seq.workers, seq.use_multiprocessing, seq.max_queue_size) == (1, False, 10) + + +def test_every_knob_is_a_declared_parameter() -> None: + """The regression guard for the rule itself: no `**kwargs` escape hatch may appear here.""" + import inspect + + params = inspect.signature(RecordSequence.__init__).parameters + + assert not [p for p in params.values() if p.kind is inspect.Parameter.VAR_KEYWORD] + assert {"source", "batch_size", "shuffle", "seed", "transform"} <= set(params) + assert {"workers", "use_multiprocessing", "max_queue_size"} <= set(params) + + +def test_threaded_prefetch_produces_the_same_batches() -> None: + """Executed rather than asserted about: Keras drives a worker pool over this object, so the + row order and the collate must survive being read off the calling thread.""" + ordered = RecordSequence(Stream(source=_records(9)), batch_size=3) + threaded = RecordSequence(Stream(source=_records(9)), batch_size=3, workers=2) + + assert [b["idx"] for b in threaded.batches()] == [b["idx"] for b in ordered.batches()] + + +# --------------------------------------------------------------------------- # +# The KERAS_BACKEND ordering — the reason this is a module and not a loose class +# --------------------------------------------------------------------------- # +# Keras 3 reads KERAS_BACKEND at IMPORT time and defaults to `tensorflow`, which the extra does +# not install: a bare `import keras` dies inside keras.src.tree with ModuleNotFoundError. The +# setdefault has to run in the LOWEST layer that imports keras, because import sorters put a +# library import above a first-party one — a consumer's own shim loses the race. + + +def test_the_backend_default_is_in_effect_before_keras_was_imported() -> None: + """The variable and the LOADED backend must agree — Keras consults it once and never again.""" + assert os.environ["KERAS_BACKEND"] == keras_module.keras_backend() + + +def test_the_default_is_the_first_INSTALLED_backend_not_a_hardcoded_one(monkeypatch: pytest.MonkeyPatch) -> None: + """Hard-coding `torch` would fail on a TensorFlow-only install exactly as Keras's own + `tensorflow` default fails on the torch-only one this exists to fix.""" + monkeypatch.setattr(importlib.util, "find_spec", lambda name: object() if name == "jax" else None) + + assert keras_module._first_installed_backend() == "jax" + + +def test_with_no_engine_installed_it_defers_to_keras_own_default(monkeypatch: pytest.MonkeyPatch) -> None: + """Let KERAS raise then — its error names the package to install; ours would guess.""" + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None) + + assert keras_module._first_installed_backend() == "tensorflow" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_optional_torch.py b/tests/test_optional_torch.py index 53ab9ef..95f53f8 100644 --- a/tests/test_optional_torch.py +++ b/tests/test_optional_torch.py @@ -192,3 +192,56 @@ def test_pyproject_declares_torch_as_an_extra_not_a_dependency() -> None: required = [d for d in project["dependencies"] if d.split(">")[0].split("=")[0].strip() == "torch"] assert not required, f"torch must not be a hard dependency, found: {required}" assert any("torch" in d for d in project["optional-dependencies"]["torch"]) + + +# --------------------------------------------------------------------------- # +# 5. keras is an extra too — and DISCOVERY must not pull it either +# --------------------------------------------------------------------------- # + +_SCAN_WITHOUT_KERAS = textwrap.dedent( + """ + import sys + + import recordstream + from recordstream.discovery import scan_module + + scan_module("recordstream") # what a GUI bridge / MCP bootstrap does + scan_module("recordstream.ops") + + leaked = sorted(m for m in sys.modules if m == "keras" or m.startswith("keras.")) + print("LEAKED:" + ",".join(leaked) if leaked else "CLEAN") + """ +) + + +def test_neither_importing_nor_scanning_recordstream_imports_keras(tmp_path: object) -> None: + """Why `RecordSequence` lives in `recordstream.keras` and NOT at the package root. + + `inspect.getmembers` — inside `scan_module`, and in the GUI bridges — getattrs every name a + module advertises, so a PEP 562 lazy root export (the `ops.ToTensor` pattern) would import + keras on every discovery scan, including on installs that never asked for it. Run in a + subprocess because this one has already imported keras via the sequence tests. + """ + result = subprocess.run( + [sys.executable, "-c", _SCAN_WITHOUT_KERAS], + capture_output=True, + text=True, + cwd=str(tmp_path), + ) + + assert result.returncode == 0, f"scanning recordstream failed:\n{result.stderr}" + assert "CLEAN" in result.stdout, f"keras was imported by a discovery scan: {result.stdout.strip()}" + + +def test_pyproject_declares_keras_as_an_extra_naming_no_compute_engine() -> None: + """Keras 3 is an API, not a runtime: the extra must not drag torch/TF/jax in — the consumer's + own extra picks one, and `recordstream.keras` defaults to whichever is installed.""" + import tomllib + from pathlib import Path + + pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" + extras = tomllib.loads(pyproject.read_text())["project"]["optional-dependencies"] + + assert any("keras" in d for d in extras["keras"]) + engines = [d for d in extras["keras"] if d.split(">")[0].split("=")[0].strip() in ("torch", "tensorflow", "jax")] + assert not engines, f"the keras extra must name no compute engine, found: {engines}" From 564ecac628186be77d01c77492973e73131387be Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 31 Jul 2026 07:22:09 +0200 Subject: [PATCH 065/102] fix: match torch 2.13's reworded int32 target error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit torch 2.13 changed the message CrossEntropyLoss raises on int32 class ids from "expected scalar type Long but found Int" to "expected target dtype to be Long or Byte, but got Int". The rejection itself is unchanged, so the dtype= knob on batch_tensor still earns its place — only the evidence quoted for it had gone stale. - test_batch.py matches (scalar type|target dtype).*Long, so the next rewording is a no-op rather than a red suite - batch.py + AGENTS.md requote the current wording --- AGENTS.md | 2 +- recordstream/batch.py | 2 +- tests/test_batch.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 38b1414..252fe71 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected scalar type Long but found Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. +- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. diff --git a/recordstream/batch.py b/recordstream/batch.py index 6771602..13ece8d 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -129,7 +129,7 @@ def batch_tensor(batch: Record, key: str, device: Any = None, dtype: Any = None) when the result feeds a model. dtype: Optional target dtype — a PARAMETER, not an opinion: the caller names the contract its loss requires and this honours it. Pass ``torch.int64`` for class ids - (``CrossEntropyLoss`` raises *"expected scalar type Long but found Int"* on an + (``CrossEntropyLoss`` raises *"expected target dtype to be Long or Byte, but got Int"* on an int32 target, and a dataset yielding int32 label tensors is perfectly legal) or for a pixel-class mask. ``None`` keeps whatever the values carry. diff --git a/tests/test_batch.py b/tests/test_batch.py index b968b2c..f5621bf 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -103,7 +103,10 @@ def test_dtype_matters_because_the_loss_rejects_the_wrong_one() -> None: batch = collate_records([{"class": Label(torch.tensor(i, dtype=torch.int32))} for i in range(3)]) logits = torch.randn(3, 4) - with pytest.raises(RuntimeError, match="expected scalar type Long"): + # Matched loosely on purpose: torch reworded this in 2.13 ("expected scalar type Long but + # found Int" -> "expected target dtype to be Long or Byte, but got Int"). The REJECTION is + # the contract this test pins; the exact phrasing is torch's to change. + with pytest.raises(RuntimeError, match="(scalar type|target dtype).*Long"): nn.CrossEntropyLoss()(logits, batch_tensor(batch, "class")) nn.CrossEntropyLoss()(logits, batch_tensor(batch, "class", dtype=torch.int64)) # no raise From 4693bc377a79755e8cf783ad154adc6013bcc124 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 1 Aug 2026 17:08:14 +0200 Subject: [PATCH 066/102] refactor: split core, flow and sources into packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `core.py`, `flow.py` and `sources.py` had each grown into an oversized module holding several cohesive units. Each becomes a package with one module per unit — `core/{stream,families,mapstyle,wrappers}`, `flow/{steps,parse,graph,execute}` and `sources/{base,huggingface,split,range,concat}` — and the SUBMODULE path becomes the canonical `!class:` spelling, since `cls.__module__` is what every config generator writes. `tests/test_module_layout.py` pins the three invariants the split makes load-bearing, each of which fails silently in production: nobody may pin `__module__` back to the package (which would also break `confluid.registry.key_for`), each `__init__.py`'s `__all__` is what a visual editor's node bridge walks (`discovery.scan_module` filters on `__module__` and now sees nothing in a package), and the import direction inside `core` / `flow` must not close a cycle. --- AGENTS.md | 8 +- docs/architecture.md | 196 ++++++- docs/augmentation.md | 2 +- docs/graph.md | 15 + docs/record-model.md | 2 +- docs/runnable.md | 2 +- docs/sources.md | 35 +- pyproject.toml | 10 +- recordstream/__init__.py | 2 +- recordstream/core/__init__.py | 81 +++ recordstream/core/families.py | 172 ++++++ recordstream/core/mapstyle.py | 41 ++ recordstream/{core.py => core/stream.py} | 310 +--------- recordstream/core/wrappers.py | 88 +++ recordstream/discovery.py | 2 +- recordstream/flow.py | 708 ----------------------- recordstream/flow/__init__.py | 86 +++ recordstream/flow/execute.py | 270 +++++++++ recordstream/flow/graph.py | 237 ++++++++ recordstream/flow/parse.py | 110 ++++ recordstream/flow/steps.py | 90 +++ recordstream/keras.py | 2 +- recordstream/ops/parallel.py | 2 +- recordstream/processing.py | 6 +- recordstream/projection.py | 2 +- recordstream/sources.py | 511 ---------------- recordstream/sources/__init__.py | 35 ++ recordstream/sources/base.py | 14 + recordstream/sources/concat.py | 72 +++ recordstream/sources/huggingface.py | 201 +++++++ recordstream/sources/range.py | 69 +++ recordstream/sources/split.py | 190 ++++++ recordstream/transform.py | 2 +- tests/test_module_layout.py | 166 ++++++ tests/test_typed_flow.py | 9 +- 35 files changed, 2210 insertions(+), 1538 deletions(-) create mode 100644 recordstream/core/__init__.py create mode 100644 recordstream/core/families.py create mode 100644 recordstream/core/mapstyle.py rename recordstream/{core.py => core/stream.py} (59%) create mode 100644 recordstream/core/wrappers.py delete mode 100644 recordstream/flow.py create mode 100644 recordstream/flow/__init__.py create mode 100644 recordstream/flow/execute.py create mode 100644 recordstream/flow/graph.py create mode 100644 recordstream/flow/parse.py create mode 100644 recordstream/flow/steps.py delete mode 100644 recordstream/sources.py create mode 100644 recordstream/sources/__init__.py create mode 100644 recordstream/sources/base.py create mode 100644 recordstream/sources/concat.py create mode 100644 recordstream/sources/huggingface.py create mode 100644 recordstream/sources/range.py create mode 100644 recordstream/sources/split.py create mode 100644 tests/test_module_layout.py diff --git a/AGENTS.md b/AGENTS.md index 252fe71..869a653 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,14 +23,14 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. - **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. -- **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). +- **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.execute.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). - **1→N Expanding Steps Fork the REMAINING Subgraph (2026-07-30, supersedes the flat-engine pending-queue rule):** An op carrying `EXPANDS = True` yields N children from one record; the remaining steps then run ONCE PER CHILD over that child's own shallow copy of the step environment (independent name→result maps, shared values), DEPTH-FIRST so sibling order matches the nested-loop intuition. An empty expansion or a `None` child drops that branch. This works in EVERY route — serial, spawn-parallel (the worker returns a LIST), and inside a `flow:` graph (the old `FlowGraph` raised `NotImplementedError` on an expanding step; that limit is gone). CONSEQUENCES: (1) `__len__`/`__getitem__` RAISE on `Stream` AND `FlowGraph` when any step op expands — the expanded index map is unknowable up front, so the pipeline is ITERABLE-ONLY (iterate, wrap in a torch IterableDataset, window at the SOURCE for random access, or `list(...)`); (2) `run_steps` (the strict 1→1 twin used for indexing) raises rather than silently dropping siblings. Pins: `tests/test_typed_flow.py::TestExpandingSteps`. - **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. -- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core.families._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). - **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. @@ -44,6 +44,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. +- **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. +- **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). diff --git a/docs/architecture.md b/docs/architecture.md index 1318875..2ac6455 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -20,7 +20,7 @@ Maintenance rules: | Data model | `items.py`, `io.py` | A record is a plain `dict` of typed values; one codec serializes any value | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | | Native ops | `transform.py`, `dispatch.py`, `ops/*` | Type-dispatched `Transform`s (kernels, `field=`) + structural/compose/context ops | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | | Library interop | `core._apply_op`, `register_op_family` | External libraries run as-is via the op-family dispatch — no adapters | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | -| Engines | `core.py` (`Stream`/`JointStream`), `flow.py` (`FlowGraph`) | One op-application chokepoint, four routes; a named-step graph engine with pinned lowering parity | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17), [§5](#5-the-engines-own-callable-wrappers-live-in-corepy-2026-07-20) | +| Engines | `core/` (`Stream`/`JointStream`), `flow/` (`FlowGraph`) | One op-application chokepoint, four routes; one per-record kernel behind two authoring forms | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17), [§5](#5-the-engines-own-callable-wrappers-live-in-core-2026-07-20-still-true-after-the-2026-08-01-package-split) | | Graph wiring | `context.py`, `ops/context.py` | Fan-out/fan-in/cross-branch values on the plain sequential engine | [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17) | | Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17) | | Storage & query | `storage/*` | The `typedrecord-v1` key-group layout over the codec; metadata scans without array loads | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) (contracts) + [storage.md](storage.md) | @@ -65,7 +65,7 @@ Collapse to ONE carrier and ONE op engine: `field=` or the first value of the natural type, and raises a `ValueError` naming the record's keys on every miss. - **External libraries run AS-IS through the engine's op-family dispatch** - (`recordstream.core._apply_op`): an albumentations op receives exactly its own kwarg vocabulary + (`recordstream.core.families._apply_op`): an albumentations op receives exactly its own kwarg vocabulary (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record; one call = one joint draw; array outputs re-wrapped in the incoming `NDArrayItem` type so `Image`/`Mask` survive); a torchvision-v2 op is called on the dict as-is; everything else is `op(record)` with @@ -304,7 +304,7 @@ run the lifting pass first to rebuild what was just destroyed. ### Decision **One execution model: the step graph.** Both spellings parse to the same `FlowStep` list and run -through the same per-record kernel (`recordstream.flow.run_steps_multi`). +through the same per-record kernel (`recordstream.flow.execute.run_steps_multi`). - An `ops:` list compiles to positional steps (`core.linear_steps` — `s0`, `s1`, …) whose names never surface. A sequence IS a graph; no lifting is involved. @@ -447,20 +447,26 @@ schemas = scan_module("recordstream.ops.numpy") # one JSON schema per op defin --- -## 5. The engine's own callable wrappers live in `core.py` (2026-07-20) +## 5. The engine's own callable wrappers live in `core` (2026-07-20; still true after the 2026-08-01 package split) + +> **Note (2026-08-01).** `core.py` became the `core/` PACKAGE (§11). Everything below holds +> unchanged — the split preserved exactly the property this record protects. `FilterOp` and +> `WrappedOp` moved to `core/wrappers.py` and `JointStream` sits beside `Stream` in +> `core/stream.py`, still inside `core`, still importing one way. What a config SPELLS changed +> (`!class:recordstream.core.wrappers.FilterOp`); what depends on whom did not. ### Context -Three classes sit in `core.py` next to the `Stream` engine that look, at first glance, like they +Three classes sit in `core` next to the `Stream` engine that look, at first glance, like they belong elsewhere: `FilterOp` and `WrappedOp` (op-shaped, so why not `ops/`?) and `JointStream` (a second engine in the engine module). ### Decision -They stay in `core.py` because of **who constructs them and which way imports flow**. All three +They stay in `core` because of **who constructs them and which way imports flow**. All three are the construction targets of `Stream`'s own fluent API — `.filter(pred)` appends a `FilterOp`, `.map(fn)` appends a `WrappedOp`, `Stream.joint([...])` wraps a `JointStream` — so the engine itself -instantiates them. And `core.py` is the *bottom* of the op-facing layer: every composing op in +instantiates them. And `core` is the *bottom* of the op-facing layer: every composing op in `ops/` imports `core._apply_op` (the op-family dispatch chokepoint); moving `FilterOp`/`WrappedOp` into `ops/` would make `core` import from `ops` and close an import cycle. `JointStream` is `Stream`'s iteration-only fan-in sibling (`category="engine"`), 20 lines that exist to be @@ -477,8 +483,8 @@ off visual canvases. - `ops/` stays a pure consumer of `core` — the layering is one-directional. - `WrappedOp` is a package-root export (the public "lift a plain function" surface, and its stored-string `f` is the reference use of the discovery serialization half); `FilterOp` is not - root-exported (normally reached via `Stream.filter`; importable as `recordstream.core.FilterOp`). -- `JointStream` is YAML-addressable (`!class:recordstream.core.JointStream()`) and canvas-composable as + root-exported (normally reached via `Stream.filter`; importable as `recordstream.core.wrappers.FilterOp`). +- `JointStream` is YAML-addressable (`!class:recordstream.core.stream.JointStream()`) and canvas-composable as an engine node; its indexable counterpart for raw sources is `ConcatSource`. ### Example @@ -494,7 +500,7 @@ both = Stream.joint([stream_a, stream_b]) # Stream(source=JointS ### What you may change (and where it's documented) -- **A new engine-constructed helper** (another fluent-API target) belongs in `core.py` for the +- **A new engine-constructed helper** (another fluent-API target) belongs in `core` for the same import-direction reason; an op users wire *directly* (YAML/canvas) belongs in `ops/` with a category and group. - **Do not add a discovery category to `FilterOp`/`WrappedOp`** — surfacing a raw-callable @@ -1020,3 +1026,173 @@ loader = DataLoader(cast(Any, stream), batch_size=32, shuffle=True, collate_fn=c `test_every_knob_is_a_declared_parameter`. - **Usage** is [docs/kinds.md](kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you); what the extra provides is the `pyproject.toml` comment beside it. + +## 11. Sources are a package, one class per module — and the module path is the contract (2026-08-01) + +### Context + +`recordstream/sources.py` had grown to four unrelated `@configurable` classes in 511 lines: a +concrete loader (`HuggingFaceSource`, which reaches the network) and three pure view sources that +only do index arithmetic (`DatasetSplit`, `RangeSource`, `ConcatSource`). Nothing tied them +together beyond the word "source" — editing the HF metadata resolution meant scrolling past the +split partitioner, and a reader looking for the concat offsets had to know it was the last class +in the file. `recordstream.ops` had already been a package for exactly this reason. + +The split is not free, because in this workspace a class's **module path is a published +contract**. `confluid.pydantic_export._qualname` builds it as `f"{cls.__module__}.{cls.__qualname__}"`, +and that string is what a generated config emits as its `!class:` tag, what a form-spec / MCP +schema reports, and what keys the discovery-service enrichment table. Moving a class to a +submodule therefore changes the tag every generator writes. + +### Decision + +**One class per module under `recordstream/sources/`, and the submodule path is canonical.** +`huggingface.py` / `split.py` / `range.py` / `concat.py`, plus a `base.py` holding the one helper +the three view sources share. `__init__.py` re-exports every public name so +`from recordstream.sources import DatasetSplit` is unchanged. + +The alternative — pinning `__module__` back to `recordstream.sources` in `__init__.py` so no +downstream string moves — was measured and rejected. It breaks +`confluid.registry.key_for()`: `_entry_for_object` finds an entry by recomputing +`f"{cls.__module__}.{cls.__qualname__}"` and comparing against the key stored when +`@configurable` ran, so a rewritten `__module__` misses. The fallout is silent — with a namesake +registered, a pinned class dumps the ambiguous `!class:Thing()` while an unpinned twin correctly +dumps its disambiguated `!class:__main__.Thing~2()`. It also breaks `inspect.getsource`, which +searches `__init__.py` and raises `OSError: could not find class definition`. + +### Consequences + +- **Both spellings still resolve.** `confluid.resolve_class` falls back to + `importlib.import_module(module_path)` + `getattr`, and the package re-exports every name, so a + hand-written `!class:recordstream.sources.HuggingFaceSource` in an old config keeps loading. + What *changed* is what generators WRITE, so every such string in the workspace was updated in + the same change — including the discovery-service enrichment key, whose miss would have silently + dropped a field alias rather than failing. +- **`__init__.py`'s `__all__` became load-bearing.** `recordstream.discovery.scan_module` filters + members on `member.__module__ == mod_name`, so it now returns `[]` for the package. A visual + editor's node bridge survives only through its second pass, which walks `__all__` — verified by + running that bridge before and after and diffing the registered node keys (identical). +- **One entry point for the package, not one per submodule.** Unlike `recordstream.ops.*`, where + each module is entry-pointed, `recordstream/sources/__init__.py` imports all four submodules, so + importing the package registers every `@configurable`. Adding per-submodule entry points would + only re-scan the same classes. +- **A new source is a new file.** There is no longer a "where in the file" question, and a source + that needs a heavy import keeps it out of its siblings' import path. + +### Example + +```yaml +# canonical — what a generator emits, matching cls.__module__ +train_set: !class:recordstream.sources.huggingface.HuggingFaceSource + path: mnist + split: train + +my_split: !class:recordstream.sources.split.DatasetSplit() + source: !ref:train_set + val_fraction: 0.1 + seed: 42 +``` + +```python +# the import surface is the package, unchanged by the split +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource +``` + +### What you may change (and where it's documented) + +- **Add a source**: one new module under `recordstream/sources/`, its class re-exported from + `__init__.py` AND listed in `__all__` — the second half is what puts it in a visual editor's + palette, and forgetting it fails silently. +- **Share code between view sources**: `base.py`. It is private to the package; a helper a + consumer should call belongs at the package root instead. +- **Usage** is [docs/sources.md](sources.md). + +## 12. `core` and `flow` are packages too — layered by import direction (2026-08-01) + +### Context + +`core.py` was 713 lines holding four unrelated things: the op-family registry and the +`_apply_op` chokepoint, the `MapStyle` Protocol, the two fluent-API callable wrappers, and the +`Stream` engine. `flow.py` was 708: the step model, the document parser, the per-record kernel, +and `FlowGraph`. Both had crossed the line where a reader looking for one thing scrolls past +three others — the same threshold that made `sources.py` a package (§11). + +Unlike `sources.py`, neither file is class-dominated: roughly 45% of `core.py` was free +functions. A literal one-class-per-module rule would have produced a 30-line `joint_stream.py` +and two ~35-line wrapper modules, which §5 had already considered and rejected as "structure for +structure's sake". And the split had to preserve something load-bearing: §5's whole argument is +that `core` is the BOTTOM of the op-facing layer, so a naive split risked closing the very +import cycle that record exists to prevent. + +### Decision + +**One module per cohesive unit, layered so imports run strictly one way.** A class gets its own +module when it dominates one (`Stream`, `FlowGraph`); otherwise the unit is the boundary. + + core/ families.py → mapstyle.py → wrappers.py → stream.py + flow/ steps.py → parse.py → execute.py → graph.py + +`core/families.py` is the bottom: the registry, the built-in families, `_apply_op`, and the +`EXPANDS` protocol — everything about applying ONE op to ONE record, importing nothing from its +siblings. `flow/execute.py` imports it directly, which is what keeps `flow` from reaching back +into `core.stream`; `core.stream` reaches `flow` only through body-local imports, exactly as +`core.py` did. + +Three functions sit where their DEPENDENCY puts them rather than where their name suggests: +`ensure_record_dataset` is in `stream.py` (its whole body builds a `Stream`, and putting it in +`mapstyle.py` beside the `RecordSource` type it consumes would have made a pure type module +import the engine); `linear_steps` and `_worker_task` likewise, because both compile or run an +ops LIST, which is `Stream`'s spelling of a pipeline. + +### Consequences + +- **The canonical `!class:` path moved** — `recordstream.core.stream.Stream`, + `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`. The package + spelling still resolves (§11), so hand-written configs keep loading, but generators emit the + new one, so all 64 downstream files were updated in the same change. +- **A monkeypatch must now name the module that USES a symbol, not the one that defines it.** + This is the one behaviour change with teeth. `flow/graph.py` does + `from recordstream.flow.execute import _result_readers`, which BINDS the name — so patching + `recordstream.flow` (which worked when both lived in one module) silently misses. The suite + caught it immediately; a test that had been asserting a performance property was suddenly + asserting nothing. `recordstream/core/__init__.py` carries a note saying so. +- **`_OP_FAMILIES` is the exception, and only because it is mutable.** The registry list is + re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry. + Rebinding it (`core._OP_FAMILIES = []`) would not. +- **`core/__init__.py` re-exports PRIVATE names deliberately** (`_apply_op` and friends), marked + `# noqa: F401`. They are the engine's internal cross-module surface — every composing op in + `ops/` imports `_apply_op` from `recordstream.core` — so the package boundary has to carry them + even though `__all__` (and therefore a visual editor's palette) must not. +- **`__all__` became load-bearing in both packages**, for the reason §11 gives. `core.py` had + none at all, so `Stream` and `JointStream` reached the palette purely through + `scan_module`'s `__module__` filter; after the split that pass returns `[]`. Verified by + diffing the registered node keys before and after (identical). + +### Example + +```yaml +# canonical — what a generator emits, matching cls.__module__ +train_set: !class:recordstream.core.stream.Stream() + source: !ref:hf_train + ops: !ref:preprocess +``` + +```python +# the import surface is the package, unchanged by the split +from recordstream.core import Stream, ensure_record_dataset +from recordstream.flow import FlowGraph, run_steps_multi + +# ...but a test double names the USER of a symbol, not its definition: +monkeypatch.setattr(recordstream.flow.graph, "_result_readers", counting) # ✓ +monkeypatch.setattr(recordstream.flow, "_result_readers", counting) # ✗ silently misses +``` + +### What you may change (and where it's documented) + +- **Add an engine-constructed helper**: `core/wrappers.py` if it wraps a raw callable, else the + module whose layer it belongs to — never a new module above `stream.py`, which would invert the + direction the layering protects (§5). +- **Add an op family**: `register_op_family` from anywhere; `core/families.py` only holds the + built-ins, and they register through the same public API (§1). +- **Split `flow/execute.py` further** if a third route appears — but `is_linear` must stay the + single gate, and the routes must keep agreeing record-for-record (§3). diff --git a/docs/augmentation.md b/docs/augmentation.md index be1fc2f..f6e8c0f 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -4,7 +4,7 @@ RecordStream does not reimplement augmentations, and it does not wrap them eithe [albumentations](https://albumentations.ai) transform or a bare torchvision `transforms.v2` transform drops **as-is** into any ops list — `Stream(ops=[...])`, a `Pipeline`, a `flow:` step, inside `RandomApply` / `Enable` — and the engine's op-family dispatch -(`recordstream.core._apply_op`) invokes it the way its own library expects. There are no adapter +(`recordstream.core.families._apply_op`) invokes it the way its own library expects. There are no adapter classes and no generated per-transform op families. ```python diff --git a/docs/graph.md b/docs/graph.md index 2f37063..0293184 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -4,6 +4,21 @@ A pipeline is a **graph of named steps**. There is ONE engine and ONE execution and `flow:` are two spellings of it, and which one you write is purely about whether the pipeline branches. +`recordstream.core` and `recordstream.flow` are packages with **one module per cohesive unit**. +Import from the package — `from recordstream.core import Stream` / `from recordstream.flow import +FlowGraph` — but spell the **submodule** path in a config, because that is what `cls.__module__` +says and what a generated config emits: + +| class | module | `!class:` path | +| --- | --- | --- | +| `Stream`, `JointStream` | `core/stream.py` | `recordstream.core.stream.Stream` | +| `FilterOp`, `WrappedOp` | `core/wrappers.py` | `recordstream.core.wrappers.FilterOp` | +| `FlowGraph` | `flow/graph.py` | `recordstream.flow.graph.FlowGraph` | + +The shorter `!class:recordstream.core.Stream` still resolves (Confluid falls back to a +module-path import, and each package re-exports every name), so an older config keeps loading. +Rationale: [docs/architecture.md §12](architecture.md#12-core-and-flow-are-packages-too--layered-by-import-direction-2026-08-01). + ## `ops:` — the linear spelling A straight chain is a graph where every step reads the one before it, so it needs no names: diff --git a/docs/record-model.md b/docs/record-model.md index 090f777..1a7e4b8 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -239,7 +239,7 @@ no library covers them. ### Mixing libraries — as-is, no adapters -The engine's single op-application chokepoint, `recordstream.core._apply_op(record, op)`, dispatches +The engine's single op-application chokepoint, `recordstream.core.families._apply_op(record, op)`, dispatches on the op's FAMILY (by MRO module name, no eager import) and invokes each family the way its own library expects: diff --git a/docs/runnable.md b/docs/runnable.md index cf419b2..9e16c03 100644 --- a/docs/runnable.md +++ b/docs/runnable.md @@ -23,7 +23,7 @@ key broadcasts into the same-named constructor parameter, with no nesting and no runnable: !class:mypkg.Classifier model: !lazy:mypkg.Backbone { name: resnet18 } -train_set: !class:recordstream.sources.HuggingFaceSource { path: mnist, split: train } +train_set: !class:recordstream.sources.huggingface.HuggingFaceSource { path: mnist, split: train } max_epochs: 3 # -> Classifier(max_epochs=3) batch_size: 32 # -> Classifier(batch_size=32) ``` diff --git a/docs/sources.md b/docs/sources.md index 9d2132a..b5203a8 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -1,5 +1,22 @@ # Sources — HuggingFace, splits, ranges, concatenation (`recordstream.sources`) +`recordstream.sources` is a package with **one class per module**. Import from the package — +`from recordstream.sources import HuggingFaceSource, DatasetSplit, RangeSource, ConcatSource` — +but spell the **submodule** path in a config, because that is what `cls.__module__` says and +what a generated config emits: + +| class | module | `!class:` path | +| --- | --- | --- | +| `HuggingFaceSource` | `huggingface.py` | `recordstream.sources.huggingface.HuggingFaceSource` | +| `DatasetSplit` | `split.py` | `recordstream.sources.split.DatasetSplit` | +| `RangeSource` | `range.py` | `recordstream.sources.range.RangeSource` | +| `ConcatSource` | `concat.py` | `recordstream.sources.concat.ConcatSource` | + +The shorter `!class:recordstream.sources.HuggingFaceSource` still resolves (Confluid falls back to +a module-path import, and the package re-exports every name), so an older config keeps loading — +but a generated one will use the submodule spelling. Rationale: +[docs/architecture.md §11](architecture.md#11-sources-are-a-package-one-class-per-module--and-the-module-path-is-the-contract-2026-08-01). + ## Hugging Face datasets `HuggingFaceSource` turns any `datasets.Dataset` (a Hub repo id or a local imagefolder path) into plain record dicts of typed values: the `input_feature` column becomes an `Image` under the record key `"image"`, the `target_feature` column a `Label` under `"class"`, and each kept metadata column its own `Label` entry keyed by the column name (plus the source-provenance `hf_path` / `hf_split` entries) — traceability that often goes missing in bare dictionary loading. @@ -7,7 +24,7 @@ - **`metadata_features` (which extra columns become record entries):** the sentinel **`"*"`** (or `["*"]`, the default) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load; an explicit list keeps exactly those columns; `None` / `[]` keep none. ```yaml -hf_train: !class:recordstream.sources.HuggingFaceSource() +hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() path: mnist input_feature: image target_feature: label @@ -31,19 +48,19 @@ split.train # ≈80% — the remainder split.val # ≈10% split.te The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: ```yaml -hf_train: !class:recordstream.sources.HuggingFaceSource() +hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() path: mnist split: train -my_split: !class:recordstream.sources.DatasetSplit() +my_split: !class:recordstream.sources.split.DatasetSplit() source: !ref:hf_train val_fraction: 0.1 test_fraction: 0.1 seed: 42 -train_set: !class:recordstream.core.Stream() { source: !ref:my_split.train } -val_set: !class:recordstream.core.Stream() { source: !ref:my_split.val } -test_set: !class:recordstream.core.Stream() { source: !ref:my_split.test } +train_set: !class:recordstream.core.stream.Stream() { source: !ref:my_split.train } +val_set: !class:recordstream.core.stream.Stream() { source: !ref:my_split.val } +test_set: !class:recordstream.core.stream.Stream() { source: !ref:my_split.test } ``` Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). @@ -51,7 +68,7 @@ Omit `test_fraction` for a plain two-way train/val split; omit both fractions an **Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `recordstream.SplitName`. ```yaml -val_set: !class:recordstream.sources.DatasetSplit() +val_set: !class:recordstream.sources.split.DatasetSplit() source: !ref:hf_train split: val val_fraction: 0.1 @@ -63,7 +80,7 @@ val_set: !class:recordstream.sources.DatasetSplit() - **`RangeSource(source, start, stop)`** — a contiguous index slice `[start:stop)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. ```yaml - first_half: !class:recordstream.sources.RangeSource() + first_half: !class:recordstream.sources.range.RangeSource() source: !ref:hf_train start: 0 stop: 5000 @@ -72,7 +89,7 @@ val_set: !class:recordstream.sources.DatasetSplit() - **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointStream`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. ```yaml - combined: !class:recordstream.sources.ConcatSource() + combined: !class:recordstream.sources.concat.ConcatSource() sources: - !ref:train_main - !ref:extra_shard diff --git a/pyproject.toml b/pyproject.toml index 3d51ca6..74b25b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ notebook = [ vision = [ "scipy", # Bare torchvision transforms.v2 ops run as-is through the engine's op-family dispatch - # (recordstream.core._apply_op); the extra makes `pip install recordstream[vision]` the + # (recordstream.core.families._apply_op); the extra makes `pip install recordstream[vision]` the # documented way to enable them. "torchvision", ] @@ -70,6 +70,11 @@ build-backend = "setuptools.build_meta" # every ``@configurable`` on bootstrap without hardcoding the list. [project.entry-points."confluid.configurables"] recordstream = "recordstream" +# ONE entry point per engine PACKAGE (core / flow / sources are each one class per cohesive +# module, all re-exported by the package __init__, so importing it registers every +# @configurable). Unlike recordstream.ops.*, do NOT add per-submodule entry points: scan_module's +# __module__ filter finds nothing in a package __init__, so a visual editor's node bridge +# surfaces these through its second pass over ``__all__`` — which is why that list is load-bearing. recordstream-core = "recordstream.core" recordstream-sources = "recordstream.sources" recordstream-ops-parallel = "recordstream.ops.parallel" @@ -79,7 +84,8 @@ recordstream-ops-random-apply = "recordstream.ops.random_apply" # changes need an editable reinstall before StreamStudio/navigaitor discovery sees the module. recordstream-ops-configure = "recordstream.ops.configure" recordstream-ops-formula = "recordstream.ops.formula" -# The FlowGraph engine (flow: named-step documents + the flow<->ops converters) +# The FlowGraph engine (flow: named-step documents) + the per-record kernel both authoring +# forms run on. There is no flow<->ops lowering pass — it was deleted 2026-07-30 (architecture §3). recordstream-flow = "recordstream.flow" # The queryable-metadata scan protocol + MetadataFilterSource view source recordstream-storage-query = "recordstream.storage.query" diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 7df91ab..aaaaeaf 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -5,7 +5,7 @@ owning its metadata — an ``Image`` its layout, a ``Label`` its classes), and ops dispatch on value TYPE (the torchvision-v2 model). Bare albumentations / torchvision ``transforms.v2`` transforms drop into any ops list AS-IS — the engine invokes each op family natively -(``recordstream.core._apply_op``). Import the whole surface from the package top level +(``recordstream.core.families._apply_op``). Import the whole surface from the package top level (``from recordstream import Record, Image, Transform, Pipeline, ...``). """ diff --git a/recordstream/core/__init__.py b/recordstream/core/__init__.py new file mode 100644 index 0000000..86aa594 --- /dev/null +++ b/recordstream/core/__init__.py @@ -0,0 +1,81 @@ +""" +RecordStream's engine core — the ``Stream`` facade and the machinery it runs on. + +One module per cohesive unit, re-exported here so ``from recordstream.core import Stream`` +is unchanged; the canonical dotted path a config / form-spec / MCP schema spells out is the +SUBMODULE one (``!class:recordstream.core.stream.Stream``), because that is what +``cls.__module__`` says. Both resolve — ``confluid.resolve_class`` falls back to a +module-path import, and this package re-exports every name — but a GENERATED config uses the +submodule spelling. + +Submodules, bottom of the layer first (imports run strictly one way): + + - recordstream.core.families: the op-family registry + ``_apply_op``, the single + op-application chokepoint every composing op routes through, plus the ``EXPANDS`` + protocol. Imports nothing from the rest of core. + - recordstream.core.mapstyle: the ``MapStyle`` Protocol + the ``RecordSource`` union — + "a dataset", said structurally so the engine never imports a framework. + - recordstream.core.wrappers: ``FilterOp`` / ``WrappedOp``, the targets of ``Stream``'s + fluent ``.filter()`` / ``.map()`` (``docs/architecture.md`` §5). + - recordstream.core.stream: ``Stream`` + ``JointStream``, the ops-list plumbing + (``linear_steps`` / ``_worker_task``) and ``ensure_record_dataset``. + +``__all__`` below is load-bearing, not decoration: a visual editor's node bridge scans this +module in two passes, and the first (``recordstream.discovery.scan_module``) filters on +``member.__module__``, so it sees NOTHING here now that the classes live in submodules. The +second pass — the one that surfaces ``Stream`` / ``JointStream`` as engine nodes — walks +exactly this ``__all__``. +""" + +from recordstream.core.families import ( # noqa: F401 — see the internal-surface note below + _ALB_KEYS, + _OP_FAMILIES, + OpInvoker, + OpMatcher, + _apply_op, + _expand, + _extra_op_families, + _is_albumentations, + _is_torchvision_v2, + _op_expands, + _sync_op_families, + register_op_family, + registered_op_families, +) +from recordstream.core.mapstyle import MapStyle, RecordSource +from recordstream.core.stream import ( # noqa: F401 — see the internal-surface note below + JointStream, + Stream, + _check_ops_materialized, + _worker_task, + ensure_record_dataset, + linear_steps, +) +from recordstream.core.wrappers import FilterOp, WrappedOp + +__all__ = [ + "FilterOp", + "JointStream", + "MapStyle", + "OpInvoker", + "OpMatcher", + "RecordSource", + "Stream", + "WrappedOp", + "ensure_record_dataset", + "linear_steps", + "register_op_family", + "registered_op_families", +] + +# The private names re-exported above are the engine's INTERNAL cross-module surface: the op +# dispatch `_apply_op` that every composing op in `recordstream.ops` imports, the spawn-worker +# helpers, and the registry list / family predicates the suite snapshots. They stay OUT of +# `__all__` (a leading underscore already keeps them off a visual editor's palette), but +# `from recordstream.core import _apply_op` must keep working — so they are re-exported +# deliberately, with the `noqa` marking that as intent rather than a stray unused import. +# +# NOTE for test doubles: these are BOUND NAMES, not views of the defining module. Patching +# `recordstream.core._apply_op` does NOT affect the copy `flow.execute` already imported — +# patch the module that USES it. The one exception is `_OP_FAMILIES`, a mutable list whose +# identity is shared, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry. diff --git a/recordstream/core/families.py b/recordstream/core/families.py new file mode 100644 index 0000000..8c03b1a --- /dev/null +++ b/recordstream/core/families.py @@ -0,0 +1,172 @@ +"""Applying ONE op to ONE record — the op-family registry and the dispatch chokepoint. + +The bottom of the op-facing layer: every composing op in :mod:`recordstream.ops` reaches +:func:`_apply_op` from here, and nothing here imports back out (see ``docs/architecture.md`` +§5 on the import direction this protects). Also home to the ``EXPANDS`` protocol +(:func:`_op_expands` / :func:`_expand`), because "does this op expand, and how do I run it?" +is the same question as "how do I apply this op" — the graph kernel asks all three together. +""" + +from typing import Any, Callable, List, Optional, Tuple, cast + +from loggair import get_logger + +from recordstream.items import NDArrayItem, Record, with_data + +logger = get_logger(__name__) + + +def _op_expands(op: Any) -> bool: + """True when an op is a 1→N expanding op (explicit ``EXPANDS = True`` class attribute).""" + return bool(getattr(op, "EXPANDS", False)) + + +def _expand(op: Any, record: Any) -> List[Any]: + """Run a 1→N EXPANDING op and return its flattened children.""" + raw = op(record) + if raw is None: + return [] + return [child for child in raw if child is not None] + + +#: An op-family MATCHER recognises a library's op objects. Keep it IMPORT-FREE — inspect +#: ``type(op).__mro__`` module names rather than importing the library. +OpMatcher = Callable[[Any], bool] +#: An op-family INVOKER applies one foreign op with its library's native calling +#: convention: ``(record, op) -> Optional[Record]`` (``None`` = drop the record). +OpInvoker = Callable[[Record, Any], Optional[Record]] + +#: The registered op families, in registration order. Dispatch checks LAST-registered +#: first, so a later (more specific) family can shadow an earlier one. +_OP_FAMILIES: List[Tuple[str, OpMatcher, OpInvoker]] = [] + + +def register_op_family(name: str, matcher: OpMatcher, invoker: OpInvoker) -> None: + """Teach the engine to invoke a NEW library's ops natively — the open extension point. + + ``matcher(op) -> bool`` recognises the family's op objects (keep it import-free — + inspect ``type(op).__mro__`` module names); ``invoker(record, op)`` applies one op with + the library's own calling convention and returns the new record (``None`` drops it). + Re-registering a ``name`` REPLACES that family in place; otherwise the family is + appended, and dispatch checks last-registered first (a more specific family shadows an + earlier one — register yours after the built-ins to win an overlap). + + Both callables MUST be module-level functions (picklable by reference): the engine's + spawn-parallel routes ship non-builtin families to worker processes by pickling them. + + Example — kornia augmentations (``nn.Module``s over batched BCHW tensors):: + + def is_kornia(op) -> bool: + return any(c.__module__.startswith("kornia.augmentation") for c in type(op).__mro__) + + def invoke_kornia(record, op): + img = record["image"] # a CHW torch.Tensor (e.g. after ToTensor) + out = op(img.unsqueeze(0)).squeeze(0) # kornia draws once per batch call + return {**record, "image": out} + + register_op_family("kornia", is_kornia, invoke_kornia) + """ + entry = (str(name), matcher, invoker) + for i, (existing, _, _) in enumerate(_OP_FAMILIES): + if existing == name: + _OP_FAMILIES[i] = entry + return + _OP_FAMILIES.append(entry) + + +def registered_op_families() -> Tuple[str, ...]: + """The registered op-family names, in registration/dispatch-precedence order.""" + return tuple(name for name, _, _ in _OP_FAMILIES) + + +def _sync_op_families(families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]]) -> None: + """Merge families shipped from the parent process into this process's registry. + + Spawn workers import this module (built-ins present) but never re-run the user's + registration side effects — the parallel routes therefore pass the parent's + non-builtin entries along and merge them here (idempotent by name). + """ + for name, matcher, invoker in families or []: + register_op_family(name, matcher, invoker) + + +def _extra_op_families() -> List[Tuple[str, OpMatcher, OpInvoker]]: + """The non-builtin registry entries — what a spawn worker cannot rebuild by import alone.""" + return [entry for entry in _OP_FAMILIES if entry[0] not in _BUILTIN_FAMILIES] + + +#: The record keys albumentations understands — its OWN target vocabulary. An albumentations +#: op receives exactly these keys (the ones present) and nothing else, so extra record +#: entries (scalars, domain items) never reach a library that would reject them. +_ALB_KEYS: Tuple[str, ...] = ("image", "mask", "masks", "bboxes", "keypoints", "labels") + + +def _is_albumentations(op: Any) -> bool: + """True for an albumentations transform / ``Compose`` — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(op).__mro__) + + +def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: + """albumentations dispatches by KWARG NAME: hand the op exactly its own target keys + present in the record (one call = one joint draw across them); array outputs are + re-wrapped in the incoming value's item type (``with_data``) so ``Image``/``Mask`` + keep their type and metadata. Box-carrying augmentation belongs in albumentations' own + ``A.Compose(..., bbox_params=...)`` — format handling is Compose's job in that library. + """ + kwargs = {k: record[k] for k in _ALB_KEYS if k in record} + if not kwargs: + logger.debug( + f"albumentations op {type(op).__name__} received no known keys " + f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." + ) + return record + out = op(**kwargs) + merged = dict(record) + for key, value in out.items(): + original = record.get(key) + if isinstance(original, NDArrayItem) and not isinstance(value, NDArrayItem): + value = with_data(original, value) + merged[key] = value + return merged + + +def _is_torchvision_v2(op: Any) -> bool: + """True for a torchvision ``transforms.v2`` transform — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(op).__mro__) + + +def _invoke_torchvision_v2(record: Record, op: Any) -> Optional[Record]: + """torchvision v2 natively walks a dict: params sampled once, tensor/tv_tensor/PIL + leaves transformed, everything else passed through — called as-is.""" + return cast(Record, op(record)) + + +# The built-in families register through the SAME open registry third parties use — +# one mechanism, no privileged code path. Registered at import, so spawn workers +# rebuild them by importing this module. +register_op_family("albumentations", _is_albumentations, _invoke_albumentations) +register_op_family("torchvision_v2", _is_torchvision_v2, _invoke_torchvision_v2) +_BUILTIN_FAMILIES: Tuple[str, ...] = ("albumentations", "torchvision_v2") + + +def _apply_op(record: Record, op: Any) -> Optional[Record]: + """Apply one op to the record dict — the engine's op-FAMILY dispatch. + + The single op-application chokepoint shared by the sequential, parallel (via + :func:`~recordstream.core.stream._worker_task`), streamed, and random-access + (``__getitem__``) paths; composing ops (``Pipeline`` / ``Parallel`` / ``Enable`` / + ``RandomApply`` / ``ConfigureOp``) route their inner ops through here so every op is + applied identically. Each op family is invoked the way its library expects — no + wrapper/adapter classes: the registered families (:func:`register_op_family`; built-ins + ``albumentations`` / ``torchvision_v2``) are checked LAST-registered first, and an op + matching none of them is a native/wiring op called ``op(record) -> Optional[Record]`` + (``None`` drops the record — filter semantics). + """ + for _name, matcher, invoker in reversed(_OP_FAMILIES): + try: + matched = matcher(op) + except Exception: # pragma: no cover - a defensive matcher never breaks dispatch + matched = False + if matched: + return invoker(record, op) + return cast(Optional[Record], op(record)) diff --git a/recordstream/core/mapstyle.py b/recordstream/core/mapstyle.py new file mode 100644 index 0000000..112035c --- /dev/null +++ b/recordstream/core/mapstyle.py @@ -0,0 +1,41 @@ +"""What recordstream MEANS by "a dataset", said structurally — no framework import. + +Pure types: the :class:`MapStyle` Protocol and the :data:`RecordSource` union built on it. +Deliberately depends on nothing but ``typing`` + the record alias, so every other core +module can import it without ordering constraints. The function that NORMALIZES a wired slot +into one of these (``ensure_record_dataset``) lives beside ``Stream`` instead, because +building a ``Stream`` is its whole body. +""" + +from typing import Any, Iterable, Protocol, Union, runtime_checkable + +from recordstream.items import Record + + +@runtime_checkable +class MapStyle(Protocol): + """A map-style dataset: ``len(ds)`` and ``ds[i]``. + + What recordstream MEANS by "a dataset", said structurally so the engine never imports a + framework to express it. ``Stream`` used to inherit ``torch.utils.data.Dataset``, which made + torch a hard dependency of a package whose own work is numpy — for nothing: that base is not + load-bearing. ``DataLoader`` duck-types its argument (a plain object with these two methods + works), nothing in the workspace does ``isinstance(x, Dataset)``, and the annotation is the + only thing the inheritance ever bought. + """ + + def __len__(self) -> int: ... + + def __getitem__(self, index: int) -> Any: ... + + +#: What a wired dataset slot may hold — the contract ``ensure_record_dataset`` enforces, +#: named ONCE here rather than restated by every consumer: anything MAP-STYLE (``__len__`` + +#: ``__getitem__`` — which a ``Stream`` is), or any iterable of records (a recordstream +#: source, a plain list of record dicts). Consumers annotate their slots +#: ``Optional[Lazy[RecordSource]]`` — ``Lazy`` because they flow the slot at run time. +#: +#: Expressed with the structural :class:`MapStyle` rather than ``torch.utils.data.Dataset`` so the +#: engine can say "a dataset" without importing a framework; torch's ``DataLoader`` is itself +#: duck-typed and consumes either. +RecordSource = Union[MapStyle, Iterable[Record]] diff --git a/recordstream/core.py b/recordstream/core/stream.py similarity index 59% rename from recordstream/core.py rename to recordstream/core/stream.py index 76b83cf..e699804 100644 --- a/recordstream/core.py +++ b/recordstream/core/stream.py @@ -1,22 +1,27 @@ +"""The ``Stream`` engine, its fan-in sibling ``JointStream``, and the ops-list plumbing. + +``Stream`` is the dataset-surface facade over the one step-graph kernel: an ``ops:`` list is +compiled to POSITIONAL steps by :func:`linear_steps` and run by +:mod:`recordstream.flow`, exactly as a ``flow:`` document's author-named steps are. The two +spellings are the same thing (``docs/architecture.md`` §3). + +What else lives here and why: + +* ``JointStream`` — 20 lines that exist to be ``Stream.joint``'s return value (§5). +* :func:`linear_steps` / :func:`_worker_task` — both compile or run an ops LIST, which is + ``Stream``'s spelling of a pipeline. +* :func:`ensure_record_dataset` — its whole body is "already a Stream? else wrap in one". +* the deferred-source guidance helpers — they phrase ``Stream``'s own Fluid errors. + +The imports of :mod:`recordstream.flow` are body-local ON PURPOSE: flow imports the op +dispatch from :mod:`recordstream.core.families` at module level, so a top-level import back +would close the cycle. +""" + import concurrent.futures import multiprocessing from contextlib import nullcontext -from typing import ( - Any, - Callable, - Collection, - Dict, - Iterable, - Iterator, - List, - Optional, - Protocol, - Sequence, - Tuple, - Union, - cast, - runtime_checkable, -) +from typing import Any, Callable, Collection, Iterable, Iterator, List, Optional, Sequence, Tuple, Union, cast from confluid import configurable from confluid import load as _confluid_load @@ -24,176 +29,14 @@ from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger -from recordstream.items import NDArrayItem, Record, item_data, with_data +from recordstream.core.families import OpInvoker, OpMatcher, _extra_op_families, _op_expands, _sync_op_families +from recordstream.core.mapstyle import RecordSource +from recordstream.core.wrappers import FilterOp, WrappedOp +from recordstream.items import Record logger = get_logger(__name__) -def _op_expands(op: Any) -> bool: - """True when an op is a 1→N expanding op (explicit ``EXPANDS = True`` class attribute).""" - return bool(getattr(op, "EXPANDS", False)) - - -#: An op-family MATCHER recognises a library's op objects. Keep it IMPORT-FREE — inspect -#: ``type(op).__mro__`` module names rather than importing the library. -OpMatcher = Callable[[Any], bool] -#: An op-family INVOKER applies one foreign op with its library's native calling -#: convention: ``(record, op) -> Optional[Record]`` (``None`` = drop the record). -OpInvoker = Callable[[Record, Any], Optional[Record]] - -#: The registered op families, in registration order. Dispatch checks LAST-registered -#: first, so a later (more specific) family can shadow an earlier one. -_OP_FAMILIES: List[Tuple[str, OpMatcher, OpInvoker]] = [] - - -def register_op_family(name: str, matcher: OpMatcher, invoker: OpInvoker) -> None: - """Teach the engine to invoke a NEW library's ops natively — the open extension point. - - ``matcher(op) -> bool`` recognises the family's op objects (keep it import-free — - inspect ``type(op).__mro__`` module names); ``invoker(record, op)`` applies one op with - the library's own calling convention and returns the new record (``None`` drops it). - Re-registering a ``name`` REPLACES that family in place; otherwise the family is - appended, and dispatch checks last-registered first (a more specific family shadows an - earlier one — register yours after the built-ins to win an overlap). - - Both callables MUST be module-level functions (picklable by reference): the engine's - spawn-parallel routes ship non-builtin families to worker processes by pickling them. - - Example — kornia augmentations (``nn.Module``s over batched BCHW tensors):: - - def is_kornia(op) -> bool: - return any(c.__module__.startswith("kornia.augmentation") for c in type(op).__mro__) - - def invoke_kornia(record, op): - img = record["image"] # a CHW torch.Tensor (e.g. after ToTensor) - out = op(img.unsqueeze(0)).squeeze(0) # kornia draws once per batch call - return {**record, "image": out} - - register_op_family("kornia", is_kornia, invoke_kornia) - """ - entry = (str(name), matcher, invoker) - for i, (existing, _, _) in enumerate(_OP_FAMILIES): - if existing == name: - _OP_FAMILIES[i] = entry - return - _OP_FAMILIES.append(entry) - - -def registered_op_families() -> Tuple[str, ...]: - """The registered op-family names, in registration/dispatch-precedence order.""" - return tuple(name for name, _, _ in _OP_FAMILIES) - - -def _sync_op_families(families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]]) -> None: - """Merge families shipped from the parent process into this process's registry. - - Spawn workers import this module (built-ins present) but never re-run the user's - registration side effects — the parallel routes therefore pass the parent's - non-builtin entries along and merge them here (idempotent by name). - """ - for name, matcher, invoker in families or []: - register_op_family(name, matcher, invoker) - - -def _extra_op_families() -> List[Tuple[str, OpMatcher, OpInvoker]]: - """The non-builtin registry entries — what a spawn worker cannot rebuild by import alone.""" - return [entry for entry in _OP_FAMILIES if entry[0] not in _BUILTIN_FAMILIES] - - -#: The record keys albumentations understands — its OWN target vocabulary. An albumentations -#: op receives exactly these keys (the ones present) and nothing else, so extra record -#: entries (scalars, domain items) never reach a library that would reject them. -_ALB_KEYS: Tuple[str, ...] = ("image", "mask", "masks", "bboxes", "keypoints", "labels") - - -def _is_albumentations(op: Any) -> bool: - """True for an albumentations transform / ``Compose`` — by MRO module name (no import here).""" - return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(op).__mro__) - - -def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: - """albumentations dispatches by KWARG NAME: hand the op exactly its own target keys - present in the record (one call = one joint draw across them); array outputs are - re-wrapped in the incoming value's item type (``with_data``) so ``Image``/``Mask`` - keep their type and metadata. Box-carrying augmentation belongs in albumentations' own - ``A.Compose(..., bbox_params=...)`` — format handling is Compose's job in that library. - """ - kwargs = {k: record[k] for k in _ALB_KEYS if k in record} - if not kwargs: - logger.debug( - f"albumentations op {type(op).__name__} received no known keys " - f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." - ) - return record - out = op(**kwargs) - merged = dict(record) - for key, value in out.items(): - original = record.get(key) - if isinstance(original, NDArrayItem) and not isinstance(value, NDArrayItem): - value = with_data(original, value) - merged[key] = value - return merged - - -def _is_torchvision_v2(op: Any) -> bool: - """True for a torchvision ``transforms.v2`` transform — by MRO module name (no import here).""" - return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(op).__mro__) - - -def _invoke_torchvision_v2(record: Record, op: Any) -> Optional[Record]: - """torchvision v2 natively walks a dict: params sampled once, tensor/tv_tensor/PIL - leaves transformed, everything else passed through — called as-is.""" - return cast(Record, op(record)) - - -# The built-in families register through the SAME open registry third parties use — -# one mechanism, no privileged code path. Registered at import, so spawn workers -# rebuild them by importing this module. -register_op_family("albumentations", _is_albumentations, _invoke_albumentations) -register_op_family("torchvision_v2", _is_torchvision_v2, _invoke_torchvision_v2) -_BUILTIN_FAMILIES: Tuple[str, ...] = ("albumentations", "torchvision_v2") - - -def _apply_op(record: Record, op: Any) -> Optional[Record]: - """Apply one op to the record dict — the engine's op-FAMILY dispatch. - - The single op-application chokepoint shared by the sequential, parallel (via - :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths; composing - ops (``Pipeline`` / ``Parallel`` / ``Enable`` / ``RandomApply`` / ``ConfigureOp``) - route their inner ops through here so every op is applied identically. Each op family - is invoked the way its library expects — no wrapper/adapter classes: the registered - families (:func:`register_op_family`; built-ins ``albumentations`` / - ``torchvision_v2``) are checked LAST-registered first, and an op matching none of - them is a native/wiring op called ``op(record) -> Optional[Record]`` (``None`` drops - the record — filter semantics). - """ - for _name, matcher, invoker in reversed(_OP_FAMILIES): - try: - matched = matcher(op) - except Exception: # pragma: no cover - a defensive matcher never breaks dispatch - matched = False - if matched: - return invoker(record, op) - return cast(Optional[Record], op(record)) - - -@runtime_checkable -class MapStyle(Protocol): - """A map-style dataset: ``len(ds)`` and ``ds[i]``. - - What recordstream MEANS by "a dataset", said structurally so the engine never imports a - framework to express it. ``Stream`` used to inherit ``torch.utils.data.Dataset``, which made - torch a hard dependency of a package whose own work is numpy — for nothing: that base is not - load-bearing. ``DataLoader`` duck-types its argument (a plain object with these two methods - works), nothing in the workspace does ``isinstance(x, Dataset)``, and the annotation is the - only thing the inheritance ever bought. - """ - - def __len__(self) -> int: ... - - def __getitem__(self, index: int) -> Any: ... - - def _describe_deferred_source(source: Any) -> str: """Return a human-friendly description of a still-deferred Confluid source. @@ -245,88 +88,6 @@ def _check_ops_materialized(ops: List[Any]) -> None: raise TypeError(_fluid_op_guidance(op, i)) from exc -@configurable -class FilterOp: - """Configurable filter operation. - - The op form of :meth:`Stream.filter` — a predicate gate over the stream: the record - passes when the predicate returns ``True`` and is dropped otherwise (``__call__`` - returns ``None``, which every engine route treats as "skip this record"). - - Args: - p: Predicate ``record -> bool``; the record passes through when it returns ``True``, else is dropped. - Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. - """ - - def __init__(self, p: Optional[Callable[[Record], bool]] = None): - # Lazy / zero-arg: store config only; a missing predicate is validated lazily in __call__. - self.p = p - - def __call__(self, record: Record) -> Optional[Record]: - if self.p is None: - raise ValueError("FilterOp.p (predicate) is not set — provide a record->bool callable before use.") - return record if self.p(record) else None - - -@configurable -class WrappedOp: - """Configurable transformation wrapper with smart mapping. - - The op form of :meth:`Stream.map` — lifts a plain function over one record value. The - callable is ALWAYS stored as its importable ``module:function`` path (via - :mod:`recordstream.discovery`), so the op pickles across ``spawn`` workers and - serializes into Confluid YAML verbatim; the live function resolves lazily on first - call. - - Args: - f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). - Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. - key: The record key whose value payload the function transforms (item metadata preserved). - ``None`` (default) = the function receives the WHOLE record dict and returns the new record. - kw: Extra keyword arguments forwarded to the wrapped callable on every call (defaults to none). - """ - - def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: Optional[Dict[str, Any]] = None): - from recordstream.discovery import get_callable_path - - # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` - # property). EXPLICIT: always store the string path for serialization. - self.f = get_callable_path(f) if callable(f) else f - self.key = key - self.kw = dict(kw) if kw else {} - # Internal cache for the live callable - self._func_cache: Optional[Callable] = None - - @property - def func(self) -> Callable: - if self._func_cache is None: - from recordstream.discovery import resolve_callable - - self._func_cache = resolve_callable(self.f) - return self._func_cache - - def __call__(self, record: Record) -> Optional[Record]: - if self.key is None: - return cast(Optional[Record], self.func(record, **self.kw)) - if self.key not in record: - raise KeyError(f"WrappedOp: record has no key {self.key!r} (keys: {list(record)})") - value = record[self.key] - new_data = self.func(item_data(value), **self.kw) - try: - new_value = with_data(value, new_data) - except TypeError: - new_value = new_data # a plain (non-item) value is replaced verbatim - return {**record, self.key: new_value} - - -def _expand(op: Any, record: Any) -> List[Any]: - """Run a 1→N EXPANDING op and return its flattened children.""" - raw = op(record) - if raw is None: - return [] - return [child for child in raw if child is not None] - - def linear_steps(ops: Sequence[Any]) -> Tuple[List[Any], str]: """Compile a flat op list into the linear step graph the engine executes. @@ -396,10 +157,11 @@ class Stream: Wraps any iterable or indexed dataset and provides a functional API. Every carrier is a plain record ``dict`` of typed values, and every op is applied - through the op-FAMILY dispatch (:func:`_apply_op`) — so native recordstream ops, - bare albumentations transforms, and bare torchvision ``transforms.v2`` transforms - all sit in ONE ``ops`` list as-is. ``source`` is duck-typed (any iterable; the - Indexable protocol if ``__getitem__``/``__len__`` are present). + through the op-FAMILY dispatch (:func:`~recordstream.core.families._apply_op`) — so + native recordstream ops, bare albumentations transforms, and bare torchvision + ``transforms.v2`` transforms all sit in ONE ``ops`` list as-is. ``source`` is + duck-typed (any iterable; the Indexable protocol if ``__getitem__``/``__len__`` are + present). Args: source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. @@ -661,18 +423,6 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: yield {k: v for k, v in record.items() if k in want} -#: What a wired dataset slot may hold — the contract :func:`ensure_record_dataset` enforces, -#: What a wired dataset slot may hold, named ONCE here rather than restated by every consumer: -#: anything MAP-STYLE (``__len__`` + ``__getitem__`` — which a :class:`Stream` is), or any -#: iterable of records (a recordstream source, a plain list of record dicts). Consumers annotate -#: their slots ``Optional[Lazy[RecordSource]]`` — ``Lazy`` because they flow the slot at run time. -#: -#: Expressed with the structural :class:`MapStyle` rather than ``torch.utils.data.Dataset`` so the -#: engine can say "a dataset" without importing a framework; torch's ``DataLoader`` is itself -#: duck-typed and consumes either. -RecordSource = Union[MapStyle, Iterable[Record]] - - def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> "Stream": """Normalize any wired source into a map-style ``Dataset`` that yields record dicts. diff --git a/recordstream/core/wrappers.py b/recordstream/core/wrappers.py new file mode 100644 index 0000000..83e673c --- /dev/null +++ b/recordstream/core/wrappers.py @@ -0,0 +1,88 @@ +"""The engine's own callable wrappers — ``FilterOp`` and ``WrappedOp``. + +Op-shaped, but NOT in :mod:`recordstream.ops`: both are construction targets of ``Stream``'s +fluent API (``.filter`` appends a ``FilterOp``, ``.map`` a ``WrappedOp``), and moving them +into ``ops/`` would make core import from a package that imports core. They carry no +discovery ``category`` on purpose — they wrap a RAW Python callable, which no GUI can wire. +Rationale: ``docs/architecture.md`` §5. +""" + +from typing import Any, Callable, Dict, Optional, Union, cast + +from confluid import configurable + +from recordstream.items import Record, item_data, with_data + + +@configurable +class FilterOp: + """Configurable filter operation. + + The op form of :meth:`~recordstream.core.Stream.filter` — a predicate gate over the + stream: the record passes when the predicate returns ``True`` and is dropped otherwise + (``__call__`` returns ``None``, which every engine route treats as "skip this record"). + + Args: + p: Predicate ``record -> bool``; the record passes through when it returns ``True``, else is dropped. + Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. + """ + + def __init__(self, p: Optional[Callable[[Record], bool]] = None): + # Lazy / zero-arg: store config only; a missing predicate is validated lazily in __call__. + self.p = p + + def __call__(self, record: Record) -> Optional[Record]: + if self.p is None: + raise ValueError("FilterOp.p (predicate) is not set — provide a record->bool callable before use.") + return record if self.p(record) else None + + +@configurable +class WrappedOp: + """Configurable transformation wrapper with smart mapping. + + The op form of :meth:`~recordstream.core.Stream.map` — lifts a plain function over one + record value. The callable is ALWAYS stored as its importable ``module:function`` path + (via :mod:`recordstream.discovery`), so the op pickles across ``spawn`` workers and + serializes into Confluid YAML verbatim; the live function resolves lazily on first + call. + + Args: + f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). + Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. + key: The record key whose value payload the function transforms (item metadata preserved). + ``None`` (default) = the function receives the WHOLE record dict and returns the new record. + kw: Extra keyword arguments forwarded to the wrapped callable on every call (defaults to none). + """ + + def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: Optional[Dict[str, Any]] = None): + from recordstream.discovery import get_callable_path + + # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` + # property). EXPLICIT: always store the string path for serialization. + self.f = get_callable_path(f) if callable(f) else f + self.key = key + self.kw = dict(kw) if kw else {} + # Internal cache for the live callable + self._func_cache: Optional[Callable] = None + + @property + def func(self) -> Callable: + if self._func_cache is None: + from recordstream.discovery import resolve_callable + + self._func_cache = resolve_callable(self.f) + return self._func_cache + + def __call__(self, record: Record) -> Optional[Record]: + if self.key is None: + return cast(Optional[Record], self.func(record, **self.kw)) + if self.key not in record: + raise KeyError(f"WrappedOp: record has no key {self.key!r} (keys: {list(record)})") + value = record[self.key] + new_data = self.func(item_data(value), **self.kw) + try: + new_value = with_data(value, new_data) + except TypeError: + new_value = new_data # a plain (non-item) value is replaced verbatim + return {**record, self.key: new_value} diff --git a/recordstream/discovery.py b/recordstream/discovery.py index 5bdc904..c769ca2 100644 --- a/recordstream/discovery.py +++ b/recordstream/discovery.py @@ -16,7 +16,7 @@ their property panels. The serialization half doubles as the workspace's generic string-callable hook -pattern (:class:`~recordstream.core.WrappedOp` stores its ``f`` this way; consuming +pattern (:class:`~recordstream.core.wrappers.WrappedOp` stores its ``f`` this way; consuming packages reuse it for their own dotted-path hooks). Curated discovery (MCP form-specs, option pickers) builds on the Confluid registry instead — which registers classes AND builder functions, but only opt-in by name; this module is diff --git a/recordstream/flow.py b/recordstream/flow.py deleted file mode 100644 index f0d9fd9..0000000 --- a/recordstream/flow.py +++ /dev/null @@ -1,708 +0,0 @@ -"""The ``flow:`` document and the :class:`FlowGraph` engine. - -A **flow document** is the named-step form of a pipeline: a mapping of ``step-name → op``, -where a step's name is how later steps reference its result. It is the spelling to reach for -when a pipeline BRANCHES; a straight chain is written as a plain ``ops:`` list, which the -engine compiles to positional steps (``recordstream.core.linear_steps``). Both parse to the -same :class:`FlowStep` list and run through the same per-record kernel — there is ONE -execution model, and no lowering pass between the two forms (the flow⇄ops converters and the -per-record context ops they emitted were deleted 2026-07-30; see ``docs/architecture.md`` §3). - -.. code-block:: yaml - - flow: - spec: !class:mypkg.MakeSpectrogram() # input: the source record - rescaled: !class:recordstream.ops.numpy.Threshold() # input: previous step - masked: !class:mypkg.Segment() {from: spec} # 2nd reader of spec = fan-out - out: {from: masked, merge_from: [rescaled]} # fan-in (no op) - outputs: out - -Step grammar (the three RESERVED step keys, stripped before the op is built): - -- ``from:`` — the step supplying this step's input record. Omitted = the previous step - (the first step reads the source record). Must name an EARLIER step: document order is - the schedule, so forward references are errors and cycles are inexpressible. -- ``merge_from:`` — fan-in: UNION the named steps' record entries into this step's incoming - record before the op runs (listed order, last-write-wins on a key collision). -- ``bind:`` — ``{param: ref}`` per-record parameters: ``ref`` is a step name (the step's - whole result record), ``step[key]`` (one entry of it), or ``step.attr`` (the step op's - live ``@output`` after it ran — read through wrapper chains by :func:`_read_output`). - -A step may be a plain mapping with no op (``out: {from: a, merge_from: [b]}``) — a pure -fan-in/identity step; ``{}`` is the identity (used to give the source a referable name). -``outputs:`` names the step whose result the pipeline yields (default: the last step). - -Step results are freed automatically: :func:`_result_readers` counts each step's readers -slot-granularly and the kernel drops a result after its last one. A straight chain needs no -environment at all — :func:`is_linear` routes it to :func:`_run_linear`. -""" - -import concurrent.futures -import inspect -import multiprocessing -from copy import deepcopy -from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Sequence, Tuple, Union, cast - -from confluid import configurable, flow -from confluid import resolve as _confluid_resolve -from confluid.fluid import Fluid as _ConfluidFluid -from loggair import get_logger - -from recordstream.core import ( - OpInvoker, - OpMatcher, - _apply_op, - _expand, - _extra_op_families, - _op_expands, - _sync_op_families, -) -from recordstream.items import Record - -logger = get_logger(__name__) - -RESERVED_STEP_KEYS = ("from", "merge_from", "bind") -"""Step-grammar keys stripped from a step mapping before the op is constructed.""" - -__all__ = ["FlowGraph", "FlowStep", "parse_flow", "run_steps", "run_steps_multi", "RESERVED_STEP_KEYS"] - -_MISSING = object() - - -def _read_output(op: Any, name: str) -> Any: - """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. - - Backs the ``bind: {param: "step.attr"}`` grammar — the step op's live ``@output`` after it - ran. The wrapper walk matters because a step op may be a composing op (``ConfigureOp`` - wrapping the real op in ``target``). Returns ``_MISSING`` when absent. - """ - cur, seen = op, set() - while cur is not None and id(cur) not in seen: - seen.add(id(cur)) - value = getattr(cur, name, _MISSING) - if value is not _MISSING: - return value - cur = getattr(cur, "target", None) or getattr(cur, "op", None) - return _MISSING - - -class FlowStep(NamedTuple): - """One parsed step of a flow document.""" - - name: str - op: Optional[Any] # live op callable; None = pure fan-in / identity step - from_: Optional[str] # None = previous step (first step: the source record) - bind: Dict[str, str] # param -> "step" | "step.attr" | "step[key]" - merge_from: Tuple[str, ...] = () # typed fan-in: union these steps' FIELDS, in slot order - - -class _BindRef(NamedTuple): - """A parsed ``bind:`` reference.""" - - step: str - attr: Optional[str] # "step.attr" = the step op's @output attribute - key: Optional[str] # "step[key]" = the named ENTRY of the step's record result - - -def _split_bind_ref(ref: str) -> _BindRef: - """Split a bind reference into its three shapes: ``step`` / ``step.attr`` / ``step[key]``.""" - text = str(ref) - if text.endswith("]") and "[" in text: - head, _, inner = text[:-1].partition("[") - if head and inner and "." not in head: - return _BindRef(head, None, inner) - head, dot, attr = text.partition(".") - return _BindRef(head, attr if dot else None, None) - - -def _parse_bind_ref(ref: str, known: Sequence[str]) -> _BindRef: - parsed = _split_bind_ref(ref) - if parsed.step not in known: - raise ValueError( - f"flow: bind reference {ref!r} does not name an earlier step " - f"(known steps at this point: {list(known)!r})" - ) - return parsed - - -def _check_reserved_collision(op: Any, step_name: str) -> None: - """Raise if the op's constructor has a param named like a reserved step key. - - Reserved keys are stripped from the step mapping before the op is built, so such a - param could never be configured inline — fail loudly instead of silently stealing it. - """ - try: - params = inspect.signature(type(op).__init__).parameters - except (TypeError, ValueError): # pragma: no cover - C-extension ctor - return - clash = [k for k in RESERVED_STEP_KEYS if k in params] - if clash: - raise ValueError( - f"flow step {step_name!r}: op {type(op).__name__!r} has constructor parameter(s) " - f"{clash!r} that collide with reserved flow step keys {RESERVED_STEP_KEYS!r} — " - "such an op cannot be configured in a flow document; rename the parameter or " - "wire the op in the flat ops form instead." - ) - - -def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[List[FlowStep], str]: - """Parse a flow mapping into ordered :class:`FlowStep`\\ s + the resolved output step name. - - ``flow_doc`` is the ``flow:`` mapping — step values may be confluid markers (from - ``resolve()``/``load()``), plain dicts (pure fan-in steps, or programmatic - ``{"op": , "from": ...}`` form), or live op callables. Reserved keys are popped; - markers are flowed per step (confluid does not auto-flow two-levels-nested markers). - Validates: step names carry no dots, every reference points to an EARLIER step. - - ``build=False`` keeps a marker step UNBUILT (the op stays a Fluid marker) — for - structural consumers (converters/importers) that must not materialize ops. - """ - if not isinstance(flow_doc, dict) or not flow_doc: - raise ValueError("flow: expected a non-empty mapping of step-name -> op") - - steps: List[FlowStep] = [] - seen: List[str] = [] - for name, value in flow_doc.items(): - name = str(name) - if "." in name: - raise ValueError(f"flow: step name {name!r} may not contain '.' (reserved for @output refs)") - if name in seen: - raise ValueError(f"flow: duplicate step name {name!r}") - - reserved: Dict[str, Any] = {} - op: Optional[Any] - if isinstance(value, _ConfluidFluid): - for key in RESERVED_STEP_KEYS: - if key in value.kwargs: - reserved[key] = value.kwargs.pop(key) - op = flow(value) if build else value - elif isinstance(value, dict): - extra = value.get("op") - reserved = {k: v for k, v in value.items() if k in RESERVED_STEP_KEYS} - unknown = [k for k in value if k not in RESERVED_STEP_KEYS and k != "op"] - if unknown: - raise ValueError( - f"flow step {name!r}: unknown step key(s) {unknown!r} — a plain-mapping step " - f"accepts only {RESERVED_STEP_KEYS!r} and 'op'" - ) - op = flow(extra) if (build and isinstance(extra, _ConfluidFluid)) else extra - elif callable(value): - op = value - elif value is None: - op = None - else: - raise TypeError(f"flow step {name!r}: expected an op, a marker, or a mapping — got {type(value).__name__}") - - if op is not None and not isinstance(op, _ConfluidFluid) and not callable(op): - raise TypeError(f"flow step {name!r}: op is not callable ({type(op).__name__})") - if op is not None and not isinstance(op, _ConfluidFluid): - _check_reserved_collision(op, name) - - from_ = reserved.get("from") - if from_ is not None and str(from_) not in seen: - raise ValueError( - f"flow step {name!r}: from: {from_!r} does not name an EARLIER step " - f"(document order is the schedule; steps so far: {seen!r})" - ) - merge_raw = reserved.get("merge_from") - merge_from: Tuple[str, ...] = () - if merge_raw is not None: - merge_from = (str(merge_raw),) if isinstance(merge_raw, str) else tuple(str(r) for r in merge_raw) - for ref in merge_from: - if ref not in seen: - raise ValueError( - f"flow step {name!r}: merge_from: {ref!r} does not name an EARLIER step " - f"(document order is the schedule; steps so far: {seen!r})" - ) - bind_raw = reserved.get("bind") or {} - if not isinstance(bind_raw, dict): - raise TypeError(f"flow step {name!r}: bind must be a mapping of param -> step[.output]") - bind: Dict[str, str] = {} - for param, ref in bind_raw.items(): - _parse_bind_ref(str(ref), seen) # validates - bind[str(param)] = str(ref) - if bind and op is None: - raise ValueError(f"flow step {name!r}: bind requires an op to configure") - - steps.append( - FlowStep( - name=name, - op=op, - from_=None if from_ is None else str(from_), - bind=bind, - merge_from=merge_from, - ) - ) - seen.append(name) - - out = str(outputs) if outputs else steps[-1].name - if out not in seen: - raise ValueError(f"flow: outputs {out!r} does not name a step (steps: {seen!r})") - return steps, out - - -def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[Tuple[int, str]]]: - """Step-result cell -> ordered ``(consumer_index, slot)`` reads. - - Slot granularity matters: one consumer step may read the SAME producer through several - slots (its input AND a ``bind`` param), and only the ``"in"`` slot of the immediately - following step can ride the linear stream. Slots: ``"in"`` (input), ``"merge"``, - ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A ``bind`` step-result - reference counts; an ``@output`` (``step.attr``) reference does NOT. - - NO steps is the identity graph (a bare ``ops: []``): nothing is produced, so nothing is - read — and there is no output step to account for. - """ - if not steps: - return {} - readers: Dict[str, List[Tuple[int, str]]] = {s.name: [] for s in steps} - for i, step in enumerate(steps): - implicit = steps[i - 1].name if i > 0 else None - source = step.from_ or implicit - if source is not None: - readers[source].append((i, "in")) - for ref in step.merge_from: - readers[ref].append((i, "merge")) - for ref in step.bind.values(): - parsed = _split_bind_ref(ref) - if parsed.attr is None: - readers[parsed.step].append((i, "bind")) - readers[outputs].append((len(steps), "out")) - return readers - - -# --------------------------------------------------------------------------- -# The FlowGraph engine -# --------------------------------------------------------------------------- - - -def run_steps_multi( - seed: Any, - steps: Sequence[FlowStep], - outputs: str, - readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, -) -> List[Record]: - """Run ONE source record through the parsed steps, returning EVERY resulting record. - - The engine's per-record kernel, module-level so a spawn worker can pickle a reference to - it. Usually one record back, zero when a step filtered (an op returned ``None``), several - when a 1→N EXPANDING step fired. - - ``readers`` is the slot-granular reader accounting from :func:`_result_readers`; it - depends only on ``(steps, outputs)``, so a caller running many records MUST compute it - once and pass it in — recomputing per record is an O(steps²) tax on every record (it was - measured at 3.4 µs/record on a 23-step pipeline, roughly half the graph engine's total - overhead over a flat op list). - - EXPANSION semantics: a step whose op carries ``EXPANDS`` yields N children, and the - REMAINING subgraph runs once per child over its own shallow copy of the step environment - (independent name→result maps, shared values — the graph twin of ``Context.copy()``). - Traversal is DEPTH-FIRST, so sibling order matches the nested-loop intuition and the flat - engine's documented order. An empty expansion or a ``None`` child just drops that branch. - - NO steps is the IDENTITY graph — the seed comes straight back. That is what makes a bare - ``Stream(source=..., ops=[])`` yield its source unchanged once the flat engine routes - through this kernel. - """ - if not steps: - return [] if seed is None else [cast(Record, seed)] - out: List[Record] = [] - if is_linear(steps, outputs): - _run_linear(seed, steps, 0, out) - return out - if readers is None: - readers = _result_readers(steps, outputs) - base_remaining = {name: len(idx) for name, idx in readers.items()} - _run_from(0, seed, steps, outputs, {}, base_remaining, None, out) - return out - - -def is_linear(steps: Sequence[FlowStep], outputs: str) -> bool: - """True when the graph is a straight chain — no named reference reaches back. - - Every step reads the one before it, nothing binds, nothing merges, and the yielded step - is the last one. Such a graph needs no step ENVIRONMENT at all: the record can ride a - local variable exactly as it did in the flat op loop, which is what keeps an ``ops:`` - list as cheap to run as before it became a graph (the env bookkeeping measured ~33% - of engine overhead on a 23-step chain). - """ - if not steps or outputs != steps[-1].name: - return False - return all(s.from_ is None and not s.bind and not s.merge_from for s in steps) - - -def _run_linear( - record: Any, - steps: Sequence[FlowStep], - index: int, - out: List[Record], -) -> None: - """Run a straight chain from ``steps[index:]`` — the env-free path (see :func:`is_linear`). - - Same expansion contract as :func:`_run_from`: a 1→N step forks the remaining chain, - depth-first, so sibling order matches the nested-loop intuition. - """ - for i in range(index, len(steps)): - op = steps[i].op - if op is None: - continue - if _op_expands(op): - for child in _expand(op, record): - _run_linear(child, steps, i + 1, out) - return - result = _apply_op(record, op) - if result is None: - return - record = result - out.append(cast(Record, record)) - - -def _run_from( - index: int, - seed: Any, - steps: Sequence[FlowStep], - outputs: str, - env: Dict[str, Any], - remaining: Dict[str, int], - prev: Optional[str], - out: List[Record], -) -> None: - """Run ``steps[index:]`` over ``env``, appending every surviving result to ``out``. - - Recurses ONCE PER CHILD at an expanding step (recursion depth = the number of expanding - steps on the path, not the record count), which is what gives depth-first sibling order - for free. - - Each expansion branch gets its OWN shallow copy of the step environment (independent - name→result maps, shared values), so siblings cannot see each other's results. - """ - - def read_result(name: str, *, copy: bool) -> Any: - value = env[name] - remaining[name] -= 1 - if remaining[name] <= 0: - del env[name] - elif copy: - value = deepcopy(value) - return value - - for i in range(index, len(steps)): - step = steps[i] - # 1. the input record (implicit stream reads move; explicit fan-out reads copy) - if step.from_ is not None: - record = read_result(step.from_, copy=True) - elif prev is not None: - record = read_result(prev, copy=False) - else: - record = seed - - # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) - if step.merge_from: - if not isinstance(record, dict): - raise TypeError( - f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " - f"{type(record).__name__} — expected a record dict." - ) - merged = dict(record) - for ref in step.merge_from: - value = read_result(ref, copy=True) - if not isinstance(value, dict): - raise TypeError( - f"flow step {step.name!r}: merge_from step {ref!r} holds " - f"{type(value).__name__}, expected a record" - ) - merged.update(value) - record = merged - - # 3. per-record parameter binds - if step.op is not None: - op = step.op - for param, ref in step.bind.items(): - parsed = _split_bind_ref(ref) - if parsed.attr is not None: - producer = next(s for s in steps if s.name == parsed.step) - value = _read_output(producer.op, parsed.attr) - if value is _MISSING: - raise AttributeError( - f"flow step {step.name!r}: bind {param}={ref!r} — " - f"step {parsed.step!r} op has no @output attribute {parsed.attr!r}" - ) - else: - value = read_result(parsed.step, copy=False) - if isinstance(value, dict) and parsed.key: - # "step[key]" = the named entry; bare "step" = the whole record. - value = value[parsed.key] - setattr(op, param, value) - - # 4. a 1→N step forks the REMAINING subgraph, one branch per child - if _op_expands(op): - for child in _expand(op, record): - child_env = dict(env) - child_env[step.name] = child - _run_from(i + 1, seed, steps, outputs, child_env, dict(remaining), step.name, out) - return - - result = _apply_op(record, op) - if result is None: - return - record = result - - env[step.name] = record - prev = step.name - - if outputs in env: - out.append(cast(Record, env[outputs])) - - -def run_steps( - seed: Any, - steps: Sequence[FlowStep], - outputs: str, - readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, -) -> Optional[Record]: - """Strictly 1→1 twin of :func:`run_steps_multi` — one result back, or ``None``. - - For callers that need exactly one carrier (indexing, a single-record probe). An expanding - step RAISES here rather than silently dropping its siblings; route those through - :func:`run_steps_multi`. - """ - for step in steps: - if step.op is not None and _op_expands(step.op): - raise TypeError( - f"flow step {step.name!r}: {type(step.op).__name__!r} is a 1→N expanding op, which " - "this strictly 1→1 route cannot carry — iterate the graph instead." - ) - results = run_steps_multi(seed, steps, outputs, readers) - return results[0] if results else None - - -def _graph_worker_task( - seed: Any, - steps: Sequence[FlowStep], - outputs: str, - families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, -) -> List[Record]: - """Spawn-worker entry point: re-register third-party op families, then run one record. - - Module-level for pickling (the same constraint :func:`recordstream.core._worker_task_multi` - obeys). Returns a LIST because an expanding step makes one seed yield several records. - ``readers`` is deliberately NOT passed across the boundary — it is cheap to derive once per - worker call relative to the process hop, and shipping it would add a second pickled - structure that must stay in sync with ``steps``. - """ - _sync_op_families(families) - return run_steps_multi(seed, steps, outputs) - - -@configurable(category="engine") -class FlowGraph: - """Named-step graph engine — executes a ``flow:`` document natively. - - The named-step twin of :class:`~recordstream.core.Stream`, over the SAME kernel: steps run - in document order against a per-record environment of named results, with fan-out isolation - (copy-on-read, move on last read) and automatic result lifetimes. A LINEAR graph converts - to a Stream (:meth:`to_stream`); a branchy one has no flat spelling by design. - - Args: - source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. - flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. - outputs: Name of the step whose result is yielded. Blank (default) = the last step. - chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single records. - """ - - def __init__( - self, - source: Optional[Any] = None, - flow: Optional[Union[Dict[str, Any], List[FlowStep]]] = None, - outputs: str = "", - chunk_size: int = 0, - ) -> None: - # Lazy / zero-arg: store config only; parsing/validation happen in the cached property. - self.source = source - self.flow = flow - self.outputs = str(outputs) - self._chunk_size = int(chunk_size) - self._workers = 1 - self._parsed: Optional[Tuple[List[FlowStep], str]] = None - self._readers: Optional[Dict[str, List[Tuple[int, str]]]] = None - - # -- parsing ----------------------------------------------------------- - - @property - def steps(self) -> List[FlowStep]: - """The parsed, validated steps (cached; recomputed only if ``flow`` is reassigned).""" - return self._ensure_parsed()[0] - - @property - def output_step(self) -> str: - """The resolved output step name.""" - return self._ensure_parsed()[1] - - def _ensure_parsed(self) -> Tuple[List[FlowStep], str]: - if self._parsed is None: - if self.flow is None: - raise ValueError("FlowGraph.flow is not set — provide a flow mapping or FlowStep list.") - if isinstance(self.flow, list) and all(isinstance(s, FlowStep) for s in self.flow): - names = [s.name for s in self.flow] - out = self.outputs or (names[-1] if names else "") - if out not in names: - raise ValueError(f"FlowGraph: outputs {out!r} does not name a step ({names!r})") - self._parsed = (list(self.flow), out) - else: - self._parsed = parse_flow(cast(Dict[str, Any], self.flow), self.outputs) - return self._parsed - - def _ensure_readers(self) -> Dict[str, List[Tuple[int, str]]]: - """The reader accounting, computed ONCE per graph (see :func:`run_steps`).""" - if self._readers is None: - steps, outputs = self._ensure_parsed() - self._readers = _result_readers(steps, outputs) - return self._readers - - @classmethod - def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": - """Build a FlowGraph from a ``{flow: {...}, outputs: ...}`` YAML document (or inline string). - - Uses ``confluid.resolve`` so step markers stay UNbuilt until :func:`parse_flow` - pops the reserved step keys and flows each op itself. - """ - doc = _confluid_resolve(path) - if not isinstance(doc, dict) or "flow" not in doc: - raise ValueError(f"FlowGraph.from_yaml: {path!r} has no 'flow:' mapping") - return cls(source=source, flow=doc["flow"], outputs=str(doc.get("outputs", "") or "")) - - @classmethod - def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": - """Load a flat ``{ops: [...]}`` YAML document as a LINEAR step graph. - - No lifting is involved: a sequence IS a graph, so the op list becomes positional - steps (``recordstream.core.linear_steps``) — the same compilation a ``Stream``'s - ``ops`` list goes through, because they are the same thing spelled two ways. - """ - from recordstream.core import Stream, linear_steps - - stream = Stream.from_ops_yaml(path, source=source) - steps, outputs = linear_steps(stream.ops) - return cls(source=source, flow=steps, outputs=outputs) - - # -- execution --------------------------------------------------------- - - def _run(self, seed: Any) -> Optional[Any]: - """Run one record through the steps; ``None`` = filtered (an op returned None).""" - steps, outputs = self._ensure_parsed() - return run_steps(seed, steps, outputs, self._ensure_readers()) - - def __iter__(self) -> Iterator[Any]: - if self.source is None: - return - it = self._iter_records() - if self._chunk_size > 0: - batch: List[Record] = [] - for record in it: - batch.append(record) - if len(batch) == self._chunk_size: - yield batch - batch = [] - if batch: - yield batch - else: - yield from it - - def _iter_records(self) -> Iterator[Record]: - if self._workers > 1: - yield from self._iter_parallel() - return - assert self.source is not None - steps, outputs = self._ensure_parsed() - readers = self._ensure_readers() - for item in self.source: - yield from run_steps_multi(item, steps, outputs, readers) - - def _iter_parallel(self) -> Iterator[Record]: - """Multiprocess execution — the graph's OWN spawn pool, one future per source record. - - Mirrors :meth:`recordstream.core.Stream._iter_parallel`: ``spawn`` (consistent with - Loggair, no CI deadlocks), third-party op families shipped to the workers by - reference. The steps pickle because their ops already must; the source never crosses - the boundary (only the seed record does). - """ - assert self.source is not None - steps, outputs = self._ensure_parsed() - ctx = multiprocessing.get_context("spawn") - - with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: - extra_families = _extra_op_families() - futures = [ - executor.submit(_graph_worker_task, item, steps, outputs, extra_families) for item in self.source - ] - for future in futures: - yield from future.result() - - @property - def _expands(self) -> bool: - """True when any step op is 1→N — the length/index map is then unknowable.""" - return any(step.op is not None and _op_expands(step.op) for step in self._ensure_parsed()[0]) - - def _guard_not_expanding(self, operation: str) -> None: - if self._expands: - raise TypeError( - f"FlowGraph.{operation} is unavailable: a step op is 1→N EXPANDING, so the " - "expanded length/index map is unknowable. Iterate the graph, wrap it in a torch " - "IterableDataset, window at the SOURCE for random access, or call .collect()." - ) - - def __len__(self) -> int: - from collections.abc import Sized - - self._guard_not_expanding("__len__") - if isinstance(self.source, Sized): - return len(self.source) - return 0 - - def __getitem__(self, index: int) -> Any: - if self.source is None: - raise TypeError("FlowGraph source is None — cannot index.") - self._guard_not_expanding("__getitem__") - if hasattr(self.source, "__getitem__"): - raw = self.source[index] - else: - raise TypeError( - f"FlowGraph source {type(self.source).__name__} does not support indexing; " - "wrap it in a list or use iteration." - ) - result = self._run(raw) - if result is None: - raise IndexError(f"Record {index} filtered out by the flow") - return result - - def parallel(self, workers: int = 4) -> "FlowGraph": - """Enable multiprocess execution on the graph's own spawn pool.""" - self._workers = workers - return self - - def batch(self, chunk_size: int) -> "FlowGraph": - """Group yielded records into lists of ``chunk_size``.""" - self._chunk_size = chunk_size - return self - - def collect(self) -> List[Any]: - """Materialize the full stream into a list.""" - return list(self) - - def to_stream(self) -> Any: - """The ``Stream`` twin of a LINEAR graph — same source, same ops, same engine. - - Only a straight chain converts: a `Stream` carries an op LIST, which cannot express - fan-out. A branchy graph has no flat spelling (that is what the deleted lowering pass - manufactured, at the cost of destroying the structure), so it raises. - """ - from recordstream.core import Stream - - steps, outputs = self._ensure_parsed() - if not is_linear(steps, outputs): - raise TypeError( - "FlowGraph.to_stream: this graph is not a straight chain (it forks or merges), " - "and a Stream's ops list cannot express that. Iterate the FlowGraph directly — " - "it is the same engine." - ) - return Stream(source=self.source, ops=[s.op for s in steps if s.op is not None]) diff --git a/recordstream/flow/__init__.py b/recordstream/flow/__init__.py new file mode 100644 index 0000000..7fcfde2 --- /dev/null +++ b/recordstream/flow/__init__.py @@ -0,0 +1,86 @@ +"""The ``flow:`` document and the :class:`FlowGraph` engine. + +A **flow document** is the named-step form of a pipeline: a mapping of ``step-name → op``, +where a step's name is how later steps reference its result. It is the spelling to reach for +when a pipeline BRANCHES; a straight chain is written as a plain ``ops:`` list, which the +engine compiles to positional steps (``recordstream.core.linear_steps``). Both parse to the +same :class:`FlowStep` list and run through the same per-record kernel — there is ONE +execution model, and no lowering pass between the two forms (the flow⇄ops converters and the +per-record context ops they emitted were deleted 2026-07-30; see ``docs/architecture.md`` §3). + +.. code-block:: yaml + + flow: + spec: !class:mypkg.MakeSpectrogram() # input: the source record + rescaled: !class:recordstream.ops.numpy.Threshold() # input: previous step + masked: !class:mypkg.Segment() {from: spec} # 2nd reader of spec = fan-out + out: {from: masked, merge_from: [rescaled]} # fan-in (no op) + outputs: out + +Step grammar (the three RESERVED step keys, stripped before the op is built): + +- ``from:`` — the step supplying this step's input record. Omitted = the previous step + (the first step reads the source record). Must name an EARLIER step: document order is + the schedule, so forward references are errors and cycles are inexpressible. +- ``merge_from:`` — fan-in: UNION the named steps' record entries into this step's incoming + record before the op runs (listed order, last-write-wins on a key collision). +- ``bind:`` — ``{param: ref}`` per-record parameters: ``ref`` is a step name (the step's + whole result record), ``step[key]`` (one entry of it), or ``step.attr`` (the step op's + live ``@output`` after it ran — read through wrapper chains by ``_read_output``). + +A step may be a plain mapping with no op (``out: {from: a, merge_from: [b]}``) — a pure +fan-in/identity step; ``{}`` is the identity (used to give the source a referable name). +``outputs:`` names the step whose result the pipeline yields (default: the last step). + +Step results are freed automatically: ``_result_readers`` counts each step's readers +slot-granularly and the kernel drops a result after its last one. A straight chain needs no +environment at all — :func:`is_linear` routes it to ``_run_linear``. + +Submodules, bottom of the layer first (imports run strictly one way): + + - recordstream.flow.steps: the step MODEL — ``FlowStep``, the ``bind:`` reference + grammar, ``RESERVED_STEP_KEYS``. Pure data; imports nothing from its siblings. + - recordstream.flow.parse: ``parse_flow`` — the only module that knows the DOCUMENT form. + - recordstream.flow.execute: the per-record kernel (``run_steps_multi`` / ``run_steps`` / + ``is_linear`` + the two routes) and the spawn-worker entry point. + - recordstream.flow.graph: ``FlowGraph``, the engine facade. + +The canonical dotted path for a config is the SUBMODULE one +(``!class:recordstream.flow.graph.FlowGraph``); the package re-export keeps +``from recordstream.flow import FlowGraph`` and the older spelling working. ``__all__`` is +load-bearing — a visual editor's node bridge surfaces ``FlowGraph`` through it, because +``scan_module``'s ``__module__`` filter no longer sees anything in this package. +""" + +from recordstream.flow.execute import ( # noqa: F401 — see the internal-surface note below + _graph_worker_task, + _result_readers, + _run_from, + _run_linear, + is_linear, + run_steps, + run_steps_multi, +) +from recordstream.flow.graph import FlowGraph +from recordstream.flow.parse import parse_flow +from recordstream.flow.steps import ( # noqa: F401 — see the internal-surface note below + _MISSING, + RESERVED_STEP_KEYS, + FlowStep, + _BindRef, + _read_output, + _split_bind_ref, +) + +__all__ = ["FlowGraph", "FlowStep", "parse_flow", "run_steps", "run_steps_multi", "is_linear", "RESERVED_STEP_KEYS"] + +# The private names re-exported above are the engine's INTERNAL cross-module surface: the +# kernel's two routes, the reader accounting `core.stream` imports, the spawn-worker entry +# point, and the bind-grammar helpers the suite pins. They stay OUT of `__all__` (a leading +# underscore already keeps them off a visual editor's palette), but +# `from recordstream.flow import _result_readers` must keep working — so they are re-exported +# deliberately, with the `noqa` marking that as intent rather than a stray unused import. +# +# NOTE for test doubles: these are BOUND NAMES, not views of the defining module. Patching +# `recordstream.flow._result_readers` does NOT affect the copy `flow.graph` already imported — +# patch the module that USES it (see `tests/test_typed_flow.py`). diff --git a/recordstream/flow/execute.py b/recordstream/flow/execute.py new file mode 100644 index 0000000..482b5b5 --- /dev/null +++ b/recordstream/flow/execute.py @@ -0,0 +1,270 @@ +"""The per-record KERNEL — the one execution model, shared by both authoring forms. + +``Stream``'s positional steps and a ``flow:`` document's author-named steps arrive here as +the same :class:`FlowStep` list (``docs/architecture.md`` §3). Two routes, and they MUST +agree record-for-record: :func:`_run_linear` for a straight chain (no step environment at +all — what keeps an ``ops:`` list cheap) and :func:`_run_from` for a graph that forks, +merges or binds. :func:`is_linear` is the gate between them. + +Result lifetimes are automatic: :func:`_result_readers` counts each step's readers +slot-granularly and the kernel drops a result after its last one. +""" + +from copy import deepcopy +from typing import Any, Dict, List, Optional, Sequence, Tuple, cast + +from loggair import get_logger + +from recordstream.core.families import OpInvoker, OpMatcher, _apply_op, _expand, _op_expands, _sync_op_families +from recordstream.flow.steps import _MISSING, FlowStep, _read_output, _split_bind_ref +from recordstream.items import Record + +logger = get_logger(__name__) + + +def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[Tuple[int, str]]]: + """Step-result cell -> ordered ``(consumer_index, slot)`` reads. + + Slot granularity matters: one consumer step may read the SAME producer through several + slots (its input AND a ``bind`` param), and only the ``"in"`` slot of the immediately + following step can ride the linear stream. Slots: ``"in"`` (input), ``"merge"``, + ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A ``bind`` step-result + reference counts; an ``@output`` (``step.attr``) reference does NOT. + + NO steps is the identity graph (a bare ``ops: []``): nothing is produced, so nothing is + read — and there is no output step to account for. + """ + if not steps: + return {} + readers: Dict[str, List[Tuple[int, str]]] = {s.name: [] for s in steps} + for i, step in enumerate(steps): + implicit = steps[i - 1].name if i > 0 else None + source = step.from_ or implicit + if source is not None: + readers[source].append((i, "in")) + for ref in step.merge_from: + readers[ref].append((i, "merge")) + for ref in step.bind.values(): + parsed = _split_bind_ref(ref) + if parsed.attr is None: + readers[parsed.step].append((i, "bind")) + readers[outputs].append((len(steps), "out")) + return readers + + +def run_steps_multi( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, +) -> List[Record]: + """Run ONE source record through the parsed steps, returning EVERY resulting record. + + The engine's per-record kernel, module-level so a spawn worker can pickle a reference to + it. Usually one record back, zero when a step filtered (an op returned ``None``), several + when a 1→N EXPANDING step fired. + + ``readers`` is the slot-granular reader accounting from :func:`_result_readers`; it + depends only on ``(steps, outputs)``, so a caller running many records MUST compute it + once and pass it in — recomputing per record is an O(steps²) tax on every record (it was + measured at 3.4 µs/record on a 23-step pipeline, roughly half the graph engine's total + overhead over a flat op list). + + EXPANSION semantics: a step whose op carries ``EXPANDS`` yields N children, and the + REMAINING subgraph runs once per child over its own shallow copy of the step environment + (independent name→result maps, shared values — the graph twin of ``Context.copy()``). + Traversal is DEPTH-FIRST, so sibling order matches the nested-loop intuition and the flat + engine's documented order. An empty expansion or a ``None`` child just drops that branch. + + NO steps is the IDENTITY graph — the seed comes straight back. That is what makes a bare + ``Stream(source=..., ops=[])`` yield its source unchanged once the flat engine routes + through this kernel. + """ + if not steps: + return [] if seed is None else [cast(Record, seed)] + out: List[Record] = [] + if is_linear(steps, outputs): + _run_linear(seed, steps, 0, out) + return out + if readers is None: + readers = _result_readers(steps, outputs) + base_remaining = {name: len(idx) for name, idx in readers.items()} + _run_from(0, seed, steps, outputs, {}, base_remaining, None, out) + return out + + +def is_linear(steps: Sequence[FlowStep], outputs: str) -> bool: + """True when the graph is a straight chain — no named reference reaches back. + + Every step reads the one before it, nothing binds, nothing merges, and the yielded step + is the last one. Such a graph needs no step ENVIRONMENT at all: the record can ride a + local variable exactly as it did in the flat op loop, which is what keeps an ``ops:`` + list as cheap to run as before it became a graph (the env bookkeeping measured ~33% + of engine overhead on a 23-step chain). + """ + if not steps or outputs != steps[-1].name: + return False + return all(s.from_ is None and not s.bind and not s.merge_from for s in steps) + + +def _run_linear( + record: Any, + steps: Sequence[FlowStep], + index: int, + out: List[Record], +) -> None: + """Run a straight chain from ``steps[index:]`` — the env-free path (see :func:`is_linear`). + + Same expansion contract as :func:`_run_from`: a 1→N step forks the remaining chain, + depth-first, so sibling order matches the nested-loop intuition. + """ + for i in range(index, len(steps)): + op = steps[i].op + if op is None: + continue + if _op_expands(op): + for child in _expand(op, record): + _run_linear(child, steps, i + 1, out) + return + result = _apply_op(record, op) + if result is None: + return + record = result + out.append(cast(Record, record)) + + +def _run_from( + index: int, + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + env: Dict[str, Any], + remaining: Dict[str, int], + prev: Optional[str], + out: List[Record], +) -> None: + """Run ``steps[index:]`` over ``env``, appending every surviving result to ``out``. + + Recurses ONCE PER CHILD at an expanding step (recursion depth = the number of expanding + steps on the path, not the record count), which is what gives depth-first sibling order + for free. + + Each expansion branch gets its OWN shallow copy of the step environment (independent + name→result maps, shared values), so siblings cannot see each other's results. + """ + + def read_result(name: str, *, copy: bool) -> Any: + value = env[name] + remaining[name] -= 1 + if remaining[name] <= 0: + del env[name] + elif copy: + value = deepcopy(value) + return value + + for i in range(index, len(steps)): + step = steps[i] + # 1. the input record (implicit stream reads move; explicit fan-out reads copy) + if step.from_ is not None: + record = read_result(step.from_, copy=True) + elif prev is not None: + record = read_result(prev, copy=False) + else: + record = seed + + # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) + if step.merge_from: + if not isinstance(record, dict): + raise TypeError( + f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " + f"{type(record).__name__} — expected a record dict." + ) + merged = dict(record) + for ref in step.merge_from: + value = read_result(ref, copy=True) + if not isinstance(value, dict): + raise TypeError( + f"flow step {step.name!r}: merge_from step {ref!r} holds " + f"{type(value).__name__}, expected a record" + ) + merged.update(value) + record = merged + + # 3. per-record parameter binds + if step.op is not None: + op = step.op + for param, ref in step.bind.items(): + parsed = _split_bind_ref(ref) + if parsed.attr is not None: + producer = next(s for s in steps if s.name == parsed.step) + value = _read_output(producer.op, parsed.attr) + if value is _MISSING: + raise AttributeError( + f"flow step {step.name!r}: bind {param}={ref!r} — " + f"step {parsed.step!r} op has no @output attribute {parsed.attr!r}" + ) + else: + value = read_result(parsed.step, copy=False) + if isinstance(value, dict) and parsed.key: + # "step[key]" = the named entry; bare "step" = the whole record. + value = value[parsed.key] + setattr(op, param, value) + + # 4. a 1→N step forks the REMAINING subgraph, one branch per child + if _op_expands(op): + for child in _expand(op, record): + child_env = dict(env) + child_env[step.name] = child + _run_from(i + 1, seed, steps, outputs, child_env, dict(remaining), step.name, out) + return + + result = _apply_op(record, op) + if result is None: + return + record = result + + env[step.name] = record + prev = step.name + + if outputs in env: + out.append(cast(Record, env[outputs])) + + +def run_steps( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, +) -> Optional[Record]: + """Strictly 1→1 twin of :func:`run_steps_multi` — one result back, or ``None``. + + For callers that need exactly one carrier (indexing, a single-record probe). An expanding + step RAISES here rather than silently dropping its siblings; route those through + :func:`run_steps_multi`. + """ + for step in steps: + if step.op is not None and _op_expands(step.op): + raise TypeError( + f"flow step {step.name!r}: {type(step.op).__name__!r} is a 1→N expanding op, which " + "this strictly 1→1 route cannot carry — iterate the graph instead." + ) + results = run_steps_multi(seed, steps, outputs, readers) + return results[0] if results else None + + +def _graph_worker_task( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, +) -> List[Record]: + """Spawn-worker entry point: re-register third-party op families, then run one record. + + Module-level for pickling (the same constraint + :func:`recordstream.core.stream._worker_task` obeys). Returns a LIST because an expanding + step makes one seed yield several records. ``readers`` is deliberately NOT passed across + the boundary — it is cheap to derive once per worker call relative to the process hop, and + shipping it would add a second pickled structure that must stay in sync with ``steps``. + """ + _sync_op_families(families) + return run_steps_multi(seed, steps, outputs) diff --git a/recordstream/flow/graph.py b/recordstream/flow/graph.py new file mode 100644 index 0000000..16be4d2 --- /dev/null +++ b/recordstream/flow/graph.py @@ -0,0 +1,237 @@ +"""``FlowGraph`` — the engine facade over a ``flow:`` document. + +The named-step twin of ``Stream``: same kernel (:mod:`recordstream.flow.execute`), different +authoring form. It earns its own module by size and by owning a document grammar of its own +(``docs/architecture.md`` §5); ``Stream`` reaches it only through body-local imports, which +is what keeps the core→flow direction one-way. +""" + +import concurrent.futures +import multiprocessing +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast + +from confluid import configurable +from confluid import resolve as _confluid_resolve +from loggair import get_logger + +from recordstream.core.families import _extra_op_families, _op_expands +from recordstream.flow.execute import _graph_worker_task, _result_readers, is_linear, run_steps, run_steps_multi +from recordstream.flow.parse import parse_flow +from recordstream.flow.steps import FlowStep +from recordstream.items import Record + +logger = get_logger(__name__) + + +@configurable(category="engine") +class FlowGraph: + """Named-step graph engine — executes a ``flow:`` document natively. + + The named-step twin of :class:`~recordstream.core.Stream`, over the SAME kernel: steps run + in document order against a per-record environment of named results, with fan-out isolation + (copy-on-read, move on last read) and automatic result lifetimes. A LINEAR graph converts + to a Stream (:meth:`to_stream`); a branchy one has no flat spelling by design. + + Args: + source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. + flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. + outputs: Name of the step whose result is yielded. Blank (default) = the last step. + chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single records. + """ + + def __init__( + self, + source: Optional[Any] = None, + flow: Optional[Union[Dict[str, Any], List[FlowStep]]] = None, + outputs: str = "", + chunk_size: int = 0, + ) -> None: + # Lazy / zero-arg: store config only; parsing/validation happen in the cached property. + self.source = source + self.flow = flow + self.outputs = str(outputs) + self._chunk_size = int(chunk_size) + self._workers = 1 + self._parsed: Optional[Tuple[List[FlowStep], str]] = None + self._readers: Optional[Dict[str, List[Tuple[int, str]]]] = None + + # -- parsing ----------------------------------------------------------- + + @property + def steps(self) -> List[FlowStep]: + """The parsed, validated steps (cached; recomputed only if ``flow`` is reassigned).""" + return self._ensure_parsed()[0] + + @property + def output_step(self) -> str: + """The resolved output step name.""" + return self._ensure_parsed()[1] + + def _ensure_parsed(self) -> Tuple[List[FlowStep], str]: + if self._parsed is None: + if self.flow is None: + raise ValueError("FlowGraph.flow is not set — provide a flow mapping or FlowStep list.") + if isinstance(self.flow, list) and all(isinstance(s, FlowStep) for s in self.flow): + names = [s.name for s in self.flow] + out = self.outputs or (names[-1] if names else "") + if out not in names: + raise ValueError(f"FlowGraph: outputs {out!r} does not name a step ({names!r})") + self._parsed = (list(self.flow), out) + else: + self._parsed = parse_flow(cast(Dict[str, Any], self.flow), self.outputs) + return self._parsed + + def _ensure_readers(self) -> Dict[str, List[Tuple[int, str]]]: + """The reader accounting, computed ONCE per graph (see :func:`run_steps`).""" + if self._readers is None: + steps, outputs = self._ensure_parsed() + self._readers = _result_readers(steps, outputs) + return self._readers + + @classmethod + def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": + """Build a FlowGraph from a ``{flow: {...}, outputs: ...}`` YAML document (or inline string). + + Uses ``confluid.resolve`` so step markers stay UNbuilt until :func:`parse_flow` + pops the reserved step keys and flows each op itself. + """ + doc = _confluid_resolve(path) + if not isinstance(doc, dict) or "flow" not in doc: + raise ValueError(f"FlowGraph.from_yaml: {path!r} has no 'flow:' mapping") + return cls(source=source, flow=doc["flow"], outputs=str(doc.get("outputs", "") or "")) + + @classmethod + def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": + """Load a flat ``{ops: [...]}`` YAML document as a LINEAR step graph. + + No lifting is involved: a sequence IS a graph, so the op list becomes positional + steps (``recordstream.core.linear_steps``) — the same compilation a ``Stream``'s + ``ops`` list goes through, because they are the same thing spelled two ways. + """ + from recordstream.core import Stream, linear_steps + + stream = Stream.from_ops_yaml(path, source=source) + steps, outputs = linear_steps(stream.ops) + return cls(source=source, flow=steps, outputs=outputs) + + # -- execution --------------------------------------------------------- + + def _run(self, seed: Any) -> Optional[Any]: + """Run one record through the steps; ``None`` = filtered (an op returned None).""" + steps, outputs = self._ensure_parsed() + return run_steps(seed, steps, outputs, self._ensure_readers()) + + def __iter__(self) -> Iterator[Any]: + if self.source is None: + return + it = self._iter_records() + if self._chunk_size > 0: + batch: List[Record] = [] + for record in it: + batch.append(record) + if len(batch) == self._chunk_size: + yield batch + batch = [] + if batch: + yield batch + else: + yield from it + + def _iter_records(self) -> Iterator[Record]: + if self._workers > 1: + yield from self._iter_parallel() + return + assert self.source is not None + steps, outputs = self._ensure_parsed() + readers = self._ensure_readers() + for item in self.source: + yield from run_steps_multi(item, steps, outputs, readers) + + def _iter_parallel(self) -> Iterator[Record]: + """Multiprocess execution — the graph's OWN spawn pool, one future per source record. + + Mirrors :meth:`recordstream.core.Stream._iter_parallel`: ``spawn`` (consistent with + Loggair, no CI deadlocks), third-party op families shipped to the workers by + reference. The steps pickle because their ops already must; the source never crosses + the boundary (only the seed record does). + """ + assert self.source is not None + steps, outputs = self._ensure_parsed() + ctx = multiprocessing.get_context("spawn") + + with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: + extra_families = _extra_op_families() + futures = [ + executor.submit(_graph_worker_task, item, steps, outputs, extra_families) for item in self.source + ] + for future in futures: + yield from future.result() + + @property + def _expands(self) -> bool: + """True when any step op is 1→N — the length/index map is then unknowable.""" + return any(step.op is not None and _op_expands(step.op) for step in self._ensure_parsed()[0]) + + def _guard_not_expanding(self, operation: str) -> None: + if self._expands: + raise TypeError( + f"FlowGraph.{operation} is unavailable: a step op is 1→N EXPANDING, so the " + "expanded length/index map is unknowable. Iterate the graph, wrap it in a torch " + "IterableDataset, window at the SOURCE for random access, or call .collect()." + ) + + def __len__(self) -> int: + from collections.abc import Sized + + self._guard_not_expanding("__len__") + if isinstance(self.source, Sized): + return len(self.source) + return 0 + + def __getitem__(self, index: int) -> Any: + if self.source is None: + raise TypeError("FlowGraph source is None — cannot index.") + self._guard_not_expanding("__getitem__") + if hasattr(self.source, "__getitem__"): + raw = self.source[index] + else: + raise TypeError( + f"FlowGraph source {type(self.source).__name__} does not support indexing; " + "wrap it in a list or use iteration." + ) + result = self._run(raw) + if result is None: + raise IndexError(f"Record {index} filtered out by the flow") + return result + + def parallel(self, workers: int = 4) -> "FlowGraph": + """Enable multiprocess execution on the graph's own spawn pool.""" + self._workers = workers + return self + + def batch(self, chunk_size: int) -> "FlowGraph": + """Group yielded records into lists of ``chunk_size``.""" + self._chunk_size = chunk_size + return self + + def collect(self) -> List[Any]: + """Materialize the full stream into a list.""" + return list(self) + + def to_stream(self) -> Any: + """The ``Stream`` twin of a LINEAR graph — same source, same ops, same engine. + + Only a straight chain converts: a `Stream` carries an op LIST, which cannot express + fan-out. A branchy graph has no flat spelling (that is what the deleted lowering pass + manufactured, at the cost of destroying the structure), so it raises. + """ + from recordstream.core import Stream + + steps, outputs = self._ensure_parsed() + if not is_linear(steps, outputs): + raise TypeError( + "FlowGraph.to_stream: this graph is not a straight chain (it forks or merges), " + "and a Stream's ops list cannot express that. Iterate the FlowGraph directly — " + "it is the same engine." + ) + return Stream(source=self.source, ops=[s.op for s in steps if s.op is not None]) diff --git a/recordstream/flow/parse.py b/recordstream/flow/parse.py new file mode 100644 index 0000000..f9b43cb --- /dev/null +++ b/recordstream/flow/parse.py @@ -0,0 +1,110 @@ +"""Parsing a ``flow:`` mapping into ordered, validated :class:`FlowStep`\\ s. + +The only module that knows the DOCUMENT form. It builds ops (flowing confluid markers per +step, because confluid does not auto-flow two-levels-nested markers) and enforces the +grammar's one structural rule: a reference must name an EARLIER step, so document order IS +the schedule and cycles are inexpressible. +""" + +from typing import Any, Dict, List, Optional, Tuple + +from confluid import flow +from confluid.fluid import Fluid as _ConfluidFluid + +from recordstream.flow.steps import RESERVED_STEP_KEYS, FlowStep, _check_reserved_collision, _parse_bind_ref + + +def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[List[FlowStep], str]: + """Parse a flow mapping into ordered :class:`FlowStep`\\ s + the resolved output step name. + + ``flow_doc`` is the ``flow:`` mapping — step values may be confluid markers (from + ``resolve()``/``load()``), plain dicts (pure fan-in steps, or programmatic + ``{"op": , "from": ...}`` form), or live op callables. Reserved keys are popped; + markers are flowed per step (confluid does not auto-flow two-levels-nested markers). + Validates: step names carry no dots, every reference points to an EARLIER step. + + ``build=False`` keeps a marker step UNBUILT (the op stays a Fluid marker) — for + structural consumers (converters/importers) that must not materialize ops. + """ + if not isinstance(flow_doc, dict) or not flow_doc: + raise ValueError("flow: expected a non-empty mapping of step-name -> op") + + steps: List[FlowStep] = [] + seen: List[str] = [] + for name, value in flow_doc.items(): + name = str(name) + if "." in name: + raise ValueError(f"flow: step name {name!r} may not contain '.' (reserved for @output refs)") + if name in seen: + raise ValueError(f"flow: duplicate step name {name!r}") + + reserved: Dict[str, Any] = {} + op: Optional[Any] + if isinstance(value, _ConfluidFluid): + for key in RESERVED_STEP_KEYS: + if key in value.kwargs: + reserved[key] = value.kwargs.pop(key) + op = flow(value) if build else value + elif isinstance(value, dict): + extra = value.get("op") + reserved = {k: v for k, v in value.items() if k in RESERVED_STEP_KEYS} + unknown = [k for k in value if k not in RESERVED_STEP_KEYS and k != "op"] + if unknown: + raise ValueError( + f"flow step {name!r}: unknown step key(s) {unknown!r} — a plain-mapping step " + f"accepts only {RESERVED_STEP_KEYS!r} and 'op'" + ) + op = flow(extra) if (build and isinstance(extra, _ConfluidFluid)) else extra + elif callable(value): + op = value + elif value is None: + op = None + else: + raise TypeError(f"flow step {name!r}: expected an op, a marker, or a mapping — got {type(value).__name__}") + + if op is not None and not isinstance(op, _ConfluidFluid) and not callable(op): + raise TypeError(f"flow step {name!r}: op is not callable ({type(op).__name__})") + if op is not None and not isinstance(op, _ConfluidFluid): + _check_reserved_collision(op, name) + + from_ = reserved.get("from") + if from_ is not None and str(from_) not in seen: + raise ValueError( + f"flow step {name!r}: from: {from_!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) + merge_raw = reserved.get("merge_from") + merge_from: Tuple[str, ...] = () + if merge_raw is not None: + merge_from = (str(merge_raw),) if isinstance(merge_raw, str) else tuple(str(r) for r in merge_raw) + for ref in merge_from: + if ref not in seen: + raise ValueError( + f"flow step {name!r}: merge_from: {ref!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) + bind_raw = reserved.get("bind") or {} + if not isinstance(bind_raw, dict): + raise TypeError(f"flow step {name!r}: bind must be a mapping of param -> step[.output]") + bind: Dict[str, str] = {} + for param, ref in bind_raw.items(): + _parse_bind_ref(str(ref), seen) # validates + bind[str(param)] = str(ref) + if bind and op is None: + raise ValueError(f"flow step {name!r}: bind requires an op to configure") + + steps.append( + FlowStep( + name=name, + op=op, + from_=None if from_ is None else str(from_), + bind=bind, + merge_from=merge_from, + ) + ) + seen.append(name) + + out = str(outputs) if outputs else steps[-1].name + if out not in seen: + raise ValueError(f"flow: outputs {out!r} does not name a step (steps: {seen!r})") + return steps, out diff --git a/recordstream/flow/steps.py b/recordstream/flow/steps.py new file mode 100644 index 0000000..9974615 --- /dev/null +++ b/recordstream/flow/steps.py @@ -0,0 +1,90 @@ +"""The step MODEL — what a parsed flow step is, and how its references are spelled. + +Pure data + string grammar: no execution, no confluid, no op dispatch. Everything else in +:mod:`recordstream.flow` builds on this, so it deliberately sits at the bottom and imports +nothing from its siblings. +""" + +import inspect +from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple + +RESERVED_STEP_KEYS = ("from", "merge_from", "bind") +"""Step-grammar keys stripped from a step mapping before the op is constructed.""" + +_MISSING = object() + + +def _read_output(op: Any, name: str) -> Any: + """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. + + Backs the ``bind: {param: "step.attr"}`` grammar — the step op's live ``@output`` after it + ran. The wrapper walk matters because a step op may be a composing op (``ConfigureOp`` + wrapping the real op in ``target``). Returns ``_MISSING`` when absent. + """ + cur, seen = op, set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + value = getattr(cur, name, _MISSING) + if value is not _MISSING: + return value + cur = getattr(cur, "target", None) or getattr(cur, "op", None) + return _MISSING + + +class FlowStep(NamedTuple): + """One parsed step of a flow document.""" + + name: str + op: Optional[Any] # live op callable; None = pure fan-in / identity step + from_: Optional[str] # None = previous step (first step: the source record) + bind: Dict[str, str] # param -> "step" | "step.attr" | "step[key]" + merge_from: Tuple[str, ...] = () # typed fan-in: union these steps' FIELDS, in slot order + + +class _BindRef(NamedTuple): + """A parsed ``bind:`` reference.""" + + step: str + attr: Optional[str] # "step.attr" = the step op's @output attribute + key: Optional[str] # "step[key]" = the named ENTRY of the step's record result + + +def _split_bind_ref(ref: str) -> _BindRef: + """Split a bind reference into its three shapes: ``step`` / ``step.attr`` / ``step[key]``.""" + text = str(ref) + if text.endswith("]") and "[" in text: + head, _, inner = text[:-1].partition("[") + if head and inner and "." not in head: + return _BindRef(head, None, inner) + head, dot, attr = text.partition(".") + return _BindRef(head, attr if dot else None, None) + + +def _parse_bind_ref(ref: str, known: Sequence[str]) -> _BindRef: + parsed = _split_bind_ref(ref) + if parsed.step not in known: + raise ValueError( + f"flow: bind reference {ref!r} does not name an earlier step " + f"(known steps at this point: {list(known)!r})" + ) + return parsed + + +def _check_reserved_collision(op: Any, step_name: str) -> None: + """Raise if the op's constructor has a param named like a reserved step key. + + Reserved keys are stripped from the step mapping before the op is built, so such a + param could never be configured inline — fail loudly instead of silently stealing it. + """ + try: + params = inspect.signature(type(op).__init__).parameters + except (TypeError, ValueError): # pragma: no cover - C-extension ctor + return + clash = [k for k in RESERVED_STEP_KEYS if k in params] + if clash: + raise ValueError( + f"flow step {step_name!r}: op {type(op).__name__!r} has constructor parameter(s) " + f"{clash!r} that collide with reserved flow step keys {RESERVED_STEP_KEYS!r} — " + "such an op cannot be configured in a flow document; rename the parameter or " + "wire the op in the flat ops form instead." + ) diff --git a/recordstream/keras.py b/recordstream/keras.py index 0c6d3de..89f8d30 100644 --- a/recordstream/keras.py +++ b/recordstream/keras.py @@ -19,7 +19,7 @@ the rest: hand :func:`~recordstream.collate.collate_records` to a ``DataLoader`` as its ``collate_fn`` and torch owns the row order, the batch slicing and the per-epoch reshuffle (a ``Stream`` is map-style, which is all a ``DataLoader`` needs — see -:class:`~recordstream.core.MapStyle`). Keras 3 has no ``DataLoader``: +:class:`~recordstream.core.mapstyle.MapStyle`). Keras 3 has no ``DataLoader``: ``keras.utils.PyDataset.__getitem__`` must return a whole BATCH, so somebody has to write that loop. :class:`RecordSequence` is that loop and nothing else — row order, slicing, reshuffle, ``collate_records`` — the DataLoader half, kept beside the collate half it calls instead of diff --git a/recordstream/ops/parallel.py b/recordstream/ops/parallel.py index bdc7092..f934e63 100644 --- a/recordstream/ops/parallel.py +++ b/recordstream/ops/parallel.py @@ -1,6 +1,6 @@ """``Parallel`` — explicit parallel sub-pipeline op. -Place inside a :class:`~recordstream.core.Stream`'s ops list to dispatch each +Place inside a :class:`~recordstream.core.stream.Stream`'s ops list to dispatch each upstream record through an inner sub-pipeline (``self.ops``) in a spawn-context worker pool. Bounded prefetch caps outstanding work so the executor queue can't grow unboundedly with source length. diff --git a/recordstream/processing.py b/recordstream/processing.py index 90bdecf..7f8ebb0 100644 --- a/recordstream/processing.py +++ b/recordstream/processing.py @@ -1,6 +1,6 @@ """Generic source→sink pipeline runner. -:class:`DatasetProcessor` orchestrates a :class:`~recordstream.core.Stream` from source +:class:`DatasetProcessor` orchestrates a :class:`~recordstream.core.stream.Stream` from source to sink — a runnable that drives whole-dataset processing (windowing, format conversion, data acquisition) with an optional console progress bar. It is the generic, modality-neutral data-pipeline runner: it iterates the stream and writes @@ -38,7 +38,7 @@ class DatasetProcessor(ProgressReporting): """Orchestrate a RecordStream pipeline from source to sink. Args: - stream: The :class:`~recordstream.core.Stream` to execute. Required to run; + stream: The :class:`~recordstream.core.stream.Stream` to execute. Required to run; defaulted to ``None`` for zero-arg construction (validated in :meth:`run`, the workspace lazy-construction rule). sink: Optional sink; when absent, records are materialized to a list. @@ -87,7 +87,7 @@ def run(self) -> None: if sink: logger.info(f"Streaming data to sink: {sink.__class__.__name__}") - # Replicates recordstream.core.Stream.to_sink so we can iterate through + # Replicates recordstream.core.stream.Stream.to_sink so we can iterate through # our progress wrapper while preserving the Storage context + flush. sink_ctx: Any = sink if isinstance(sink, Storage) else nullcontext() count = 0 diff --git a/recordstream/projection.py b/recordstream/projection.py index a861c12..a49d0a3 100644 --- a/recordstream/projection.py +++ b/recordstream/projection.py @@ -16,7 +16,7 @@ * Every public function is a lazy generator (**Lazy Evaluation** mandate) — nothing materializes the whole source. * :func:`num_classes` (integer class-id semantics) is a free function, *not* a - method on the generic :class:`~recordstream.core.Stream` engine — counting classes is + method on the generic :class:`~recordstream.core.stream.Stream` engine — counting classes is a classification concern, and bolting it onto the task-agnostic engine would make every ``Stream`` look classification-capable to duck-typed consumers. """ diff --git a/recordstream/sources.py b/recordstream/sources.py deleted file mode 100644 index 30e41e2..0000000 --- a/recordstream/sources.py +++ /dev/null @@ -1,511 +0,0 @@ -import bisect -import random -from typing import Any, Collection, Dict, Iterator, List, Literal, Optional, get_args - -from confluid import configurable -from loggair import get_logger - -from recordstream.items import Image, Label, Record - -logger = get_logger(__name__) - - -def _pass_through(item: Any) -> Any: - """Pass a wrapped source's item through verbatim. - - Every carrier is a plain dict; the view sources - (:class:`DatasetSplit` / :class:`RangeSource` / :class:`ConcatSource`) only slice/index, - they never inspect payloads, so a source's records flow through them unchanged. - """ - return item - - -# Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer -# closed Literals over bare strings — self-documenting + machine-introspectable by UIs / -# navigaitor form-spec / MCP schemas via ``typing.get_args``). The runtime-validation tuple -# is derived from the Literal so there is ONE source of truth — never restate the values. -SplitName = Literal["train", "val", "test"] -_SPLIT_NAMES = get_args(SplitName) - -# Sentinel for ``HuggingFaceSource.metadata_features`` meaning "every dataset column except the -# input/target features" — the full-traceability option, kept OPT-IN (``None`` / ``[]`` still = no -# extra metadata) so existing configs are unaffected. Resolved against the loaded dataset's -# ``column_names`` at construction. Accepted bare (``"*"``) or as the one-element list (``["*"]``); -# Visual editors offer it as a selectable "*" entry in a metadata picker. -METADATA_ALL_FEATURES = "*" - - -def _resolve_metadata_features( - requested: Optional[List[str] | str], - column_names: Optional[List[str]], - input_feature: str, - target_feature: str, -) -> List[str]: - """Resolve a ``metadata_features`` spec into a concrete, order-preserving column list. - - ``None`` / ``[]`` -> ``[]`` (no extra metadata — the backward-compatible default). The sentinel - ``"*"`` (bare or inside a list) -> every column in ``column_names`` except ``input_feature`` / - ``target_feature`` (full traceability). An explicit list of names is used verbatim. ``"*"`` may - be combined with extra names (union, order-preserving: the "rest" first, then the extras). - """ - if not requested: - return [] - if isinstance(requested, str): - requested = [requested] - if METADATA_ALL_FEATURES not in requested: - return list(requested) - excluded = {input_feature, target_feature} - rest = [c for c in (column_names or []) if c not in excluded] - extras = [r for r in requested if r != METADATA_ALL_FEATURES and r not in excluded and r not in rest] - return rest + extras - - -@configurable(category="source") -class HuggingFaceSource: - """ - RecordStream Source for Hugging Face Datasets, yielding plain record dicts. - - Key mapping (the record layout): - - * the ``input_feature`` value (image / array) -> an :class:`~recordstream.Image` under the - record key ``"image"``; - * the ``target_feature`` value (label) -> a :class:`~recordstream.Label` under the record key - ``"class"``; - * each ``metadata_features`` column -> its own :class:`~recordstream.Label` keyed by the column - name, plus the source-provenance ``hf_path`` / ``hf_split`` Labels. - - Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md - "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and - does NO functional work — ``HuggingFaceSource()`` is valid, and the dataset is downloaded - only on first access to :attr:`dataset` (cached thereafter; reset ``_dataset`` to reload). - ``path`` is therefore optional at construction and validated lazily when the data is needed. - - Args: - path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. - split: HF split name (``train`` / ``validation`` / ``test`` / etc.). - input_feature: Dataset feature column mapped onto the ``"image"`` record key (an ``Image`` item). - target_feature: Dataset feature column mapped onto the ``"class"`` record key (a ``Label`` item). - metadata_features: Columns -> per-column ``Label`` entries; ``None``=none, ``"*"``=all-but-i/o, else a list. - count: Optional cap on the number of records yielded (useful for fast smoke runs). - name: Optional HF subset/config name (e.g. for multi-config datasets). - """ - - def __init__( - self, - path: str = "", - split: str = "train", - input_feature: str = "image", - target_feature: str = "label", - metadata_features: Optional[List[str] | str] = "*", - count: Optional[int] = None, - name: Optional[str] = None, - **kwargs: Any, - ) -> None: - # Lazy constructor: store config only — never load here. Real work (the network/disk - # download) is deferred to the ``dataset`` property so the object is cheap to build and - # configurable post-construction. - self.path = path - self.split = split - self.input_feature = input_feature - self.target_feature = target_feature - # Stored as the RAW spec (``None`` / ``"*"`` / list) — resolved against the loaded dataset's - # columns lazily by the ``resolved_metadata_features`` property, not eagerly here. - self.metadata_features = metadata_features - self.count = count - self.name = name - # Extra kwargs forwarded verbatim to ``datasets.load_dataset`` at load time (e.g. ``token``, - # ``trust_remote_code``). Captured now, applied lazily in the ``dataset`` property. - self._load_kwargs = dict(kwargs) - # Lazy cache for the materialized dataset (see the ``dataset`` property). - self._dataset: Any = None - - @property - def dataset(self) -> Any: - """The HF dataset, loaded on first access and cached. Resetting ``_dataset`` to None reloads. - - Raises ``ValueError`` if ``path`` was never set — the zero-arg constructor allows building an - unconfigured source, but materializing one without a dataset id cannot succeed. - """ - if self._dataset is None: - if not self.path: - raise ValueError( - "HuggingFaceSource.path is empty — set it (constructor arg, YAML, or configure()) " - "before iterating or indexing the source." - ) - from datasets import load_dataset - - logger.info(f"HuggingFaceSource: Loading {self.path} ({self.split})...") - self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self._load_kwargs) - return self._dataset - - @property - def resolved_metadata_features(self) -> List[str]: - """``metadata_features`` resolved against the live dataset's columns (expands the ``"*"`` sentinel). - - Lazy because the ``"*"`` expansion needs the loaded dataset's ``column_names``; ``None`` / ``[]`` - stays "no extra metadata" (backward-compatible). - """ - return _resolve_metadata_features( - self.metadata_features, getattr(self.dataset, "column_names", None), self.input_feature, self.target_feature - ) - - def _to_record( - self, - item: Any, - metadata_features: List[str], - keys: Optional[Collection[str]] = None, - ) -> Record: - """Assemble one record dict from a raw HF row dict (see the class docstring for the key mapping). - - ``keys`` gates which record entries are built (``None`` = all) — the projection path - (:meth:`project`) passes only the requested ones, so an unwanted image is never decoded. - ``metadata_features`` arrives pre-filtered on the projection path. - """ - record: Record = {} - if keys is None or "image" in keys: - # The input value (image/array) becomes an ``Image`` item; a PIL image / list is coerced - # to an ndarray by ``Image.__new__`` (np.asarray), preserving the default HWC layout. - record["image"] = Image(item.get(self.input_feature)) - if keys is None or "class" in keys: - record["class"] = Label(item.get(self.target_feature)) - # Each requested metadata column rides its OWN Label entry keyed by the column name (the - # metadata a value needs travels WITH it). Source provenance follows the same shape. - for feature in metadata_features: - record[feature] = Label(item.get(feature)) - if keys is None or "hf_path" in keys: - record["hf_path"] = Label(self.path) - if keys is None or "hf_split" in keys: - record["hf_split"] = Label(self.split) - return record - - def __iter__(self) -> Iterator[Record]: - dataset = self.dataset - metadata_features = self.resolved_metadata_features - limit = self.count or len(dataset) - - for counter, item in enumerate(dataset): - if counter >= limit: - break - yield self._to_record(item, metadata_features) - - def __getitem__(self, index: int) -> Record: - return self._to_record(self.dataset[index], self.resolved_metadata_features) - - def project(self, keys: Collection[str]) -> Iterator[Record]: - """Yield key-restricted records — the ``SupportsProjection`` efficient path. - - Only the requested keys are built, so a label-only walk (e.g. :func:`~recordstream.num_classes`) - skips decoding the image entirely: ``"image"`` -> the input feature, ``"class"`` -> the target - Label, plus any requested metadata-column / provenance keys. - """ - want = frozenset(keys) - dataset = self.dataset - # Resolve (and pre-filter) the metadata columns only when a key beyond the fixed image/class - # pair is requested — the "*" expansion needs the loaded dataset's columns. - meta_requested = bool(want - {"image", "class"}) - metadata_features = [f for f in self.resolved_metadata_features if f in want] if meta_requested else [] - limit = self.count or len(dataset) - for counter, item in enumerate(dataset): - if counter >= limit: - break - yield self._to_record(item, metadata_features, keys=want) - - def __len__(self) -> int: - # A ``count`` of 0 (or None) means "all records", matching __iter__'s - # ``limit = self.count or len(...)``. Returning a bare ``self.count`` here - # would report 0 for the common "0 == unlimited" case, making the source - # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) - # even though iteration yields every record. - return self.count or len(self.dataset) - - -@configurable(category="source") -class DatasetSplit: - """ - Splits an indexable source into reproducible ``train`` / ``val`` / ``test`` views. - - A ``source`` (it yields records and is wired into a trainer's ``source:`` slot), - not an engine — it applies no ops, it just exposes a reproducible partition of another - source. (For a contiguous index slice use :class:`RangeSource`; to concatenate several - sources use :class:`ConcatSource`.) - - **Property API (preferred).** Configure ONE ``DatasetSplit`` with ``seed`` and the - held-out fraction(s) (``val_fraction`` and/or ``test_fraction``) and read the three - cached view sources off it:: - - split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) - split.train # ≈80% — the remainder - split.val # ≈10% - split.test # ≈10% - - The views are disjoint and complementary, computed once (cached) over a single - deterministic shuffle, so the underlying source is consumed once. In Confluid YAML the - views are reachable by **attribute reference** — ``!ref:my_split.train`` / ``.val`` / - ``.test`` — and because two ``!ref:`` to the same key flow the *same* instance, the - partition and the source load are shared across all three references:: - - my_split: !class:recordstream.sources.DatasetSplit() - source: !ref:hf_train - val_fraction: 0.1 - test_fraction: 0.1 - seed: 42 - - train_set: !class:recordstream.core.Stream() - source: !ref:my_split.train - val_set: !class:recordstream.core.Stream() - source: !ref:my_split.val - - **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one - view (``split=None`` ⇒ ``train``), so it is directly usable as a single ``source:``. - - Omit ``test_fraction`` for a plain two-way train/val split; omit both fractions for a - degenerate split where ``train`` is the whole source and ``val`` / ``test`` are empty. - - The wrapped source must implement ``__len__`` and ``__getitem__``. Lazy: only index - arithmetic happens up front; records are produced on demand. - - Args: - source: The underlying indexable source (defaults to ``None``; validated lazily on first use). - split: View this iterates as a source — ``train`` / ``val`` / ``test`` (``None`` ⇒ ``train``). - val_fraction: Fraction of records assigned to the ``val`` view. Must be in ``(0, 1)``. - test_fraction: Fraction of records assigned to the ``test`` view. Must be in ``(0, 1)``. - seed: Seed for the deterministic shuffle. Required when any fraction is set. - """ - - def __init__( - self, - source: Any = None, - split: Optional[SplitName] = None, - val_fraction: Optional[float] = None, - test_fraction: Optional[float] = None, - seed: Optional[int] = None, - ) -> None: - # Lazy / zero-arg: store config only. All validation is deferred to first materialization - # (``_validate``, invoked from ``_view``) so the source can be configured post-construction. - self.source = source - self.split = split - self.val_fraction = val_fraction - self.test_fraction = test_fraction - self.seed = seed - # Cache of materialized split views. Underscore-prefixed so confluid's - # vars(obj)-based discovery / dump ignores it (the `train`/`val`/`test` - # @property descriptors live on the class, not in vars(obj), so they never - # surface as configurable attributes either). - self._views: Dict[str, "_SplitView"] = {} - - def _validate(self) -> None: - """Validate the (post-construction) configuration. Called lazily before the first partition.""" - source = self.source - if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): - raise TypeError( - "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" - ) - if self.split is not None and self.split not in _SPLIT_NAMES: - raise ValueError(f"split must be one of {_SPLIT_NAMES}; got {self.split!r}") - if (self.val_fraction is not None or self.test_fraction is not None) and self.seed is None: - raise ValueError("DatasetSplit requires `seed` when a fraction is set, so the partition is reproducible.") - if self.val_fraction is not None and not (0.0 < self.val_fraction < 1.0): - raise ValueError(f"val_fraction must be in (0, 1); got {self.val_fraction}") - if self.test_fraction is not None and not (0.0 < self.test_fraction < 1.0): - raise ValueError(f"test_fraction must be in (0, 1); got {self.test_fraction}") - if (self.val_fraction or 0.0) + (self.test_fraction or 0.0) >= 1.0: - raise ValueError( - "val_fraction + test_fraction must be < 1 (to leave a non-empty train split); " - f"got val_fraction={self.val_fraction}, test_fraction={self.test_fraction}" - ) - - def _partition(self) -> Dict[str, List[int]]: - """Deterministically partition the source indices into ``train`` / ``val`` / ``test``. - - One shuffle seeded by ``seed`` (skipped when no fraction is set, so the degenerate - "all train" case keeps source order); layout is ``[train | val | test]``. ``max(1, …)`` - guarantees a held-out split gets at least one record on tiny sources. - """ - n = len(self.source) - val_fraction = self.val_fraction or 0.0 - test_fraction = self.test_fraction or 0.0 - shuffled = list(range(n)) - if val_fraction or test_fraction: - random.Random(self.seed).shuffle(shuffled) - val_count = max(1, int(round(n * val_fraction))) if val_fraction else 0 - test_count = max(1, int(round(n * test_fraction))) if test_fraction else 0 - train_count = max(0, n - val_count - test_count) - return { - "train": shuffled[:train_count], - "val": shuffled[train_count : train_count + val_count], - "test": shuffled[train_count + val_count :], - } - - def _view(self, split: SplitName) -> "_SplitView": - if split not in self._views: - self._validate() - self._views[split] = _SplitView(self.source, self._partition()[split]) - return self._views[split] - - @property - def train(self) -> "_SplitView": - """Cached training-split view (the remainder after ``val`` / ``test`` are held out).""" - return self._view("train") - - @property - def val(self) -> "_SplitView": - """Cached validation-split view (≈ ``val_fraction`` of the source).""" - return self._view("val") - - @property - def test(self) -> "_SplitView": - """Cached test-split view (≈ ``test_fraction`` of the source).""" - return self._view("test") - - def __iter__(self) -> Iterator[Record]: - return iter(self._view(self.split or "train")) - - def __getitem__(self, index: int) -> Any: - return self._view(self.split or "train")[index] - - def __len__(self) -> int: - return len(self._view(self.split or "train")) - - -class _SplitView: - """An indexable view of ``source`` restricted (and reordered) to ``indices``. - - Internal to :class:`DatasetSplit` — produced by its ``train`` / ``val`` / ``test`` - properties (and reachable in Confluid YAML via ``!ref:my_split.train``). Deliberately - NOT a ``@configurable``: it is never constructed directly in a config, only read off a - live ``DatasetSplit`` instance, so it carries no discovery surface of its own. - """ - - def __init__(self, source: Any, indices: List[int]) -> None: - self.source = source - self.indices = indices - - def __iter__(self) -> Iterator[Record]: - for idx in self.indices: - yield _pass_through(self.source[idx]) - - def __getitem__(self, index: int) -> Any: - return _pass_through(self.source[self.indices[index]]) - - def __len__(self) -> int: - return len(self.indices) - - -@configurable(category="source") -class RangeSource: - """A contiguous index slice ``[start:stop)`` over an indexable source. - - The plain-slice counterpart to :class:`DatasetSplit` (which shuffles + partitions) — - extracted from DatasetSplit's old "range mode". Negative ``start`` / ``stop`` count from - the end; both are clamped to ``[0, len(source)]``. Lazy: only index arithmetic happens - up front; records are produced on demand. - - The wrapped source must implement ``__len__`` and ``__getitem__``. - - Args: - source: The underlying indexable source (defaults to ``None``; validated lazily on first use). - start: Inclusive start index (``None`` ⇒ 0; a negative value counts from the end). - stop: Exclusive stop index (``None`` ⇒ len(source); a negative value counts from the end). - """ - - def __init__(self, source: Any = None, start: Optional[int] = None, stop: Optional[int] = None) -> None: - # Lazy / zero-arg: store config only; the index arithmetic (and source validation) is deferred - # to the ``indices`` property so the source can be configured post-construction. - self.source = source - self.start = start - self.stop = stop - self._indices: Optional[List[int]] = None - - @property - def indices(self) -> List[int]: - """The contiguous ``[start:stop)`` source indices, computed lazily on first access and cached.""" - if self._indices is None: - source = self.source - if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): - raise TypeError( - "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" - ) - n = len(source) - s = 0 if self.start is None else self.start - e = n if self.stop is None else self.stop - if s < 0: - s = max(0, n + s) - if e < 0: - e = max(0, n + e) - s = max(0, min(s, n)) - e = max(s, min(e, n)) - self._indices = list(range(s, e)) - logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) - return self._indices - - def __iter__(self) -> Iterator[Record]: - for idx in self.indices: - yield _pass_through(self.source[idx]) - - def __getitem__(self, index: int) -> Any: - return _pass_through(self.source[self.indices[index]]) - - def __len__(self) -> int: - return len(self.indices) - - -@configurable(category="source") -class ConcatSource: - """Concatenates multiple indexable sources into one longer indexable source. - - The indexable counterpart to :class:`recordstream.core.JointStream` (which is iteration-only): - ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning - sub-source, so a ``ConcatSource`` can itself be wrapped by :class:`DatasetSplit` / - :class:`RangeSource`. (Distinct from :class:`waivefront.paired.AnnotationJoinSource`, which - *column-joins* annotations onto records — this one *concatenates* sequences end to end.) - - Each sub-source must implement ``__len__`` and ``__getitem__``. - - Args: - sources: The indexable sources to concatenate, walked in order (defaults to ``None`` ⇒ empty). - """ - - def __init__(self, sources: Optional[List[Any]] = None) -> None: - # Lazy / zero-arg: store config only; sub-source validation + the cumulative-offset precompute - # are deferred to the ``offsets`` property so sources can be configured post-construction. - self.sources = list(sources) if sources else [] - self._offsets: Optional[List[int]] = None - - @property - def offsets(self) -> List[int]: - """Cumulative END offsets per sub-source, computed lazily on first access and cached. - - Computing them validates each sub-source (``__len__`` / ``__getitem__``); enables an - O(log k) global-index → (sub-source, local index) map. - """ - if self._offsets is None: - offsets: List[int] = [] - total = 0 - for i, src in enumerate(self.sources): - if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): - raise TypeError( - "ConcatSource requires sources supporting __len__ and __getitem__; " - f"source[{i}] is {type(src).__name__}" - ) - total += len(src) - offsets.append(total) - self._offsets = offsets - return self._offsets - - def __len__(self) -> int: - return self.offsets[-1] if self.offsets else 0 - - def __getitem__(self, index: int) -> Any: - n = len(self) - if index < 0: - index += n - if not 0 <= index < n: - raise IndexError(index) - j = bisect.bisect_right(self.offsets, index) - start = self.offsets[j - 1] if j > 0 else 0 - return _pass_through(self.sources[j][index - start]) - - def __iter__(self) -> Iterator[Record]: - for src in self.sources: - for item in src: - yield _pass_through(item) diff --git a/recordstream/sources/__init__.py b/recordstream/sources/__init__.py new file mode 100644 index 0000000..b6e092d --- /dev/null +++ b/recordstream/sources/__init__.py @@ -0,0 +1,35 @@ +""" +RecordStream sources — the classes that yield (or derive a view of) record dicts. + +One class per module, re-exported here so ``from recordstream.sources import X`` keeps +working; the canonical dotted path a config / form-spec / MCP schema spells out is the +SUBMODULE one (``!class:recordstream.sources.huggingface.HuggingFaceSource``), exactly as +for :mod:`recordstream.ops`. Both spellings resolve — ``confluid.resolve_class`` falls back +to a module-path import, and this package re-exports every name — but generated configs and +the enrichment table key on the submodule path, because that is what ``cls.__module__`` says. + +Submodules: + - recordstream.sources.huggingface: HuggingFaceSource (+ the METADATA_ALL_FEATURES sentinel) + - recordstream.sources.split: DatasetSplit (+ the SplitName Literal) + - recordstream.sources.range: RangeSource (a contiguous ``[start:stop)`` slice) + - recordstream.sources.concat: ConcatSource (several indexable sources end to end) + +``__all__`` below is load-bearing, not decoration: a visual editor's node bridge scans this +module in two passes, and the first (``recordstream.discovery.scan_module``) filters on +``member.__module__``, so it sees NOTHING here now that the classes live in submodules. The +second pass — the one that surfaces these nodes — walks exactly this ``__all__``. +""" + +from recordstream.sources.concat import ConcatSource +from recordstream.sources.huggingface import METADATA_ALL_FEATURES, HuggingFaceSource +from recordstream.sources.range import RangeSource +from recordstream.sources.split import DatasetSplit, SplitName + +__all__ = [ + "ConcatSource", + "DatasetSplit", + "HuggingFaceSource", + "METADATA_ALL_FEATURES", + "RangeSource", + "SplitName", +] diff --git a/recordstream/sources/base.py b/recordstream/sources/base.py new file mode 100644 index 0000000..4578a80 --- /dev/null +++ b/recordstream/sources/base.py @@ -0,0 +1,14 @@ +"""Shared internals for the view sources (``split`` / ``range`` / ``concat``).""" + +from typing import Any + + +def _pass_through(item: Any) -> Any: + """Pass a wrapped source's item through verbatim. + + Every carrier is a plain dict; the view sources + (:class:`~recordstream.sources.DatasetSplit` / :class:`~recordstream.sources.RangeSource` / + :class:`~recordstream.sources.ConcatSource`) only slice/index, they never inspect payloads, + so a source's records flow through them unchanged. + """ + return item diff --git a/recordstream/sources/concat.py b/recordstream/sources/concat.py new file mode 100644 index 0000000..3b93bd6 --- /dev/null +++ b/recordstream/sources/concat.py @@ -0,0 +1,72 @@ +"""``ConcatSource`` — several indexable sources presented end to end as one.""" + +import bisect +from typing import Any, Iterator, List, Optional + +from confluid import configurable + +from recordstream.items import Record +from recordstream.sources.base import _pass_through + + +@configurable(category="source") +class ConcatSource: + """Concatenates multiple indexable sources into one longer indexable source. + + The indexable counterpart to :class:`recordstream.core.JointStream` (which is iteration-only): + ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning + sub-source, so a ``ConcatSource`` can itself be wrapped by + :class:`~recordstream.sources.DatasetSplit` / :class:`~recordstream.sources.RangeSource`. + (Distinct from an annotation-join source, which *column-joins* annotations onto records — + this one *concatenates* sequences end to end.) + + Each sub-source must implement ``__len__`` and ``__getitem__``. + + Args: + sources: The indexable sources to concatenate, walked in order (defaults to ``None`` ⇒ empty). + """ + + def __init__(self, sources: Optional[List[Any]] = None) -> None: + # Lazy / zero-arg: store config only; sub-source validation + the cumulative-offset precompute + # are deferred to the ``offsets`` property so sources can be configured post-construction. + self.sources = list(sources) if sources else [] + self._offsets: Optional[List[int]] = None + + @property + def offsets(self) -> List[int]: + """Cumulative END offsets per sub-source, computed lazily on first access and cached. + + Computing them validates each sub-source (``__len__`` / ``__getitem__``); enables an + O(log k) global-index → (sub-source, local index) map. + """ + if self._offsets is None: + offsets: List[int] = [] + total = 0 + for i, src in enumerate(self.sources): + if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): + raise TypeError( + "ConcatSource requires sources supporting __len__ and __getitem__; " + f"source[{i}] is {type(src).__name__}" + ) + total += len(src) + offsets.append(total) + self._offsets = offsets + return self._offsets + + def __len__(self) -> int: + return self.offsets[-1] if self.offsets else 0 + + def __getitem__(self, index: int) -> Any: + n = len(self) + if index < 0: + index += n + if not 0 <= index < n: + raise IndexError(index) + j = bisect.bisect_right(self.offsets, index) + start = self.offsets[j - 1] if j > 0 else 0 + return _pass_through(self.sources[j][index - start]) + + def __iter__(self) -> Iterator[Record]: + for src in self.sources: + for item in src: + yield _pass_through(item) diff --git a/recordstream/sources/huggingface.py b/recordstream/sources/huggingface.py new file mode 100644 index 0000000..7319a79 --- /dev/null +++ b/recordstream/sources/huggingface.py @@ -0,0 +1,201 @@ +"""``HuggingFaceSource`` — a Hugging Face dataset as a stream of record dicts.""" + +from typing import Any, Collection, Iterator, List, Optional + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Image, Label, Record + +logger = get_logger(__name__) + +# Sentinel for ``HuggingFaceSource.metadata_features`` meaning "every dataset column except the +# input/target features" — the full-traceability option, kept OPT-IN (``None`` / ``[]`` still = no +# extra metadata) so existing configs are unaffected. Resolved against the loaded dataset's +# ``column_names`` at construction. Accepted bare (``"*"``) or as the one-element list (``["*"]``); +# Visual editors offer it as a selectable "*" entry in a metadata picker. +METADATA_ALL_FEATURES = "*" + + +def _resolve_metadata_features( + requested: Optional[List[str] | str], + column_names: Optional[List[str]], + input_feature: str, + target_feature: str, +) -> List[str]: + """Resolve a ``metadata_features`` spec into a concrete, order-preserving column list. + + ``None`` / ``[]`` -> ``[]`` (no extra metadata — the backward-compatible default). The sentinel + ``"*"`` (bare or inside a list) -> every column in ``column_names`` except ``input_feature`` / + ``target_feature`` (full traceability). An explicit list of names is used verbatim. ``"*"`` may + be combined with extra names (union, order-preserving: the "rest" first, then the extras). + """ + if not requested: + return [] + if isinstance(requested, str): + requested = [requested] + if METADATA_ALL_FEATURES not in requested: + return list(requested) + excluded = {input_feature, target_feature} + rest = [c for c in (column_names or []) if c not in excluded] + extras = [r for r in requested if r != METADATA_ALL_FEATURES and r not in excluded and r not in rest] + return rest + extras + + +@configurable(category="source") +class HuggingFaceSource: + """ + RecordStream Source for Hugging Face Datasets, yielding plain record dicts. + + Key mapping (the record layout): + + * the ``input_feature`` value (image / array) -> an :class:`~recordstream.Image` under the + record key ``"image"``; + * the ``target_feature`` value (label) -> a :class:`~recordstream.Label` under the record key + ``"class"``; + * each ``metadata_features`` column -> its own :class:`~recordstream.Label` keyed by the column + name, plus the source-provenance ``hf_path`` / ``hf_split`` Labels. + + Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md + "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and + does NO functional work — ``HuggingFaceSource()`` is valid, and the dataset is downloaded + only on first access to :attr:`dataset` (cached thereafter; reset ``_dataset`` to reload). + ``path`` is therefore optional at construction and validated lazily when the data is needed. + + Args: + path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. + split: HF split name (``train`` / ``validation`` / ``test`` / etc.). + input_feature: Dataset feature column mapped onto the ``"image"`` record key (an ``Image`` item). + target_feature: Dataset feature column mapped onto the ``"class"`` record key (a ``Label`` item). + metadata_features: Columns -> per-column ``Label`` entries; ``None``=none, ``"*"``=all-but-i/o, else a list. + count: Optional cap on the number of records yielded (useful for fast smoke runs). + name: Optional HF subset/config name (e.g. for multi-config datasets). + """ + + def __init__( + self, + path: str = "", + split: str = "train", + input_feature: str = "image", + target_feature: str = "label", + metadata_features: Optional[List[str] | str] = "*", + count: Optional[int] = None, + name: Optional[str] = None, + **kwargs: Any, + ) -> None: + # Lazy constructor: store config only — never load here. Real work (the network/disk + # download) is deferred to the ``dataset`` property so the object is cheap to build and + # configurable post-construction. + self.path = path + self.split = split + self.input_feature = input_feature + self.target_feature = target_feature + # Stored as the RAW spec (``None`` / ``"*"`` / list) — resolved against the loaded dataset's + # columns lazily by the ``resolved_metadata_features`` property, not eagerly here. + self.metadata_features = metadata_features + self.count = count + self.name = name + # Extra kwargs forwarded verbatim to ``datasets.load_dataset`` at load time (e.g. ``token``, + # ``trust_remote_code``). Captured now, applied lazily in the ``dataset`` property. + self._load_kwargs = dict(kwargs) + # Lazy cache for the materialized dataset (see the ``dataset`` property). + self._dataset: Any = None + + @property + def dataset(self) -> Any: + """The HF dataset, loaded on first access and cached. Resetting ``_dataset`` to None reloads. + + Raises ``ValueError`` if ``path`` was never set — the zero-arg constructor allows building an + unconfigured source, but materializing one without a dataset id cannot succeed. + """ + if self._dataset is None: + if not self.path: + raise ValueError( + "HuggingFaceSource.path is empty — set it (constructor arg, YAML, or configure()) " + "before iterating or indexing the source." + ) + from datasets import load_dataset + + logger.info(f"HuggingFaceSource: Loading {self.path} ({self.split})...") + self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self._load_kwargs) + return self._dataset + + @property + def resolved_metadata_features(self) -> List[str]: + """``metadata_features`` resolved against the live dataset's columns (expands the ``"*"`` sentinel). + + Lazy because the ``"*"`` expansion needs the loaded dataset's ``column_names``; ``None`` / ``[]`` + stays "no extra metadata" (backward-compatible). + """ + return _resolve_metadata_features( + self.metadata_features, getattr(self.dataset, "column_names", None), self.input_feature, self.target_feature + ) + + def _to_record( + self, + item: Any, + metadata_features: List[str], + keys: Optional[Collection[str]] = None, + ) -> Record: + """Assemble one record dict from a raw HF row dict (see the class docstring for the key mapping). + + ``keys`` gates which record entries are built (``None`` = all) — the projection path + (:meth:`project`) passes only the requested ones, so an unwanted image is never decoded. + ``metadata_features`` arrives pre-filtered on the projection path. + """ + record: Record = {} + if keys is None or "image" in keys: + # The input value (image/array) becomes an ``Image`` item; a PIL image / list is coerced + # to an ndarray by ``Image.__new__`` (np.asarray), preserving the default HWC layout. + record["image"] = Image(item.get(self.input_feature)) + if keys is None or "class" in keys: + record["class"] = Label(item.get(self.target_feature)) + # Each requested metadata column rides its OWN Label entry keyed by the column name (the + # metadata a value needs travels WITH it). Source provenance follows the same shape. + for feature in metadata_features: + record[feature] = Label(item.get(feature)) + if keys is None or "hf_path" in keys: + record["hf_path"] = Label(self.path) + if keys is None or "hf_split" in keys: + record["hf_split"] = Label(self.split) + return record + + def __iter__(self) -> Iterator[Record]: + dataset = self.dataset + metadata_features = self.resolved_metadata_features + limit = self.count or len(dataset) + + for counter, item in enumerate(dataset): + if counter >= limit: + break + yield self._to_record(item, metadata_features) + + def __getitem__(self, index: int) -> Record: + return self._to_record(self.dataset[index], self.resolved_metadata_features) + + def project(self, keys: Collection[str]) -> Iterator[Record]: + """Yield key-restricted records — the ``SupportsProjection`` efficient path. + + Only the requested keys are built, so a label-only walk (e.g. :func:`~recordstream.num_classes`) + skips decoding the image entirely: ``"image"`` -> the input feature, ``"class"`` -> the target + Label, plus any requested metadata-column / provenance keys. + """ + want = frozenset(keys) + dataset = self.dataset + # Resolve (and pre-filter) the metadata columns only when a key beyond the fixed image/class + # pair is requested — the "*" expansion needs the loaded dataset's columns. + meta_requested = bool(want - {"image", "class"}) + metadata_features = [f for f in self.resolved_metadata_features if f in want] if meta_requested else [] + limit = self.count or len(dataset) + for counter, item in enumerate(dataset): + if counter >= limit: + break + yield self._to_record(item, metadata_features, keys=want) + + def __len__(self) -> int: + # A ``count`` of 0 (or None) means "all records", matching __iter__'s + # ``limit = self.count or len(...)``. Returning a bare ``self.count`` here + # would report 0 for the common "0 == unlimited" case, making the source + # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) + # even though iteration yields every record. + return self.count or len(self.dataset) diff --git a/recordstream/sources/range.py b/recordstream/sources/range.py new file mode 100644 index 0000000..0e5771b --- /dev/null +++ b/recordstream/sources/range.py @@ -0,0 +1,69 @@ +"""``RangeSource`` — a contiguous ``[start:stop)`` index slice over an indexable source.""" + +from typing import Any, Iterator, List, Optional + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record +from recordstream.sources.base import _pass_through + +logger = get_logger(__name__) + + +@configurable(category="source") +class RangeSource: + """A contiguous index slice ``[start:stop)`` over an indexable source. + + The plain-slice counterpart to :class:`~recordstream.sources.DatasetSplit` (which shuffles + + partitions) — extracted from DatasetSplit's old "range mode". Negative ``start`` / + ``stop`` count from the end; both are clamped to ``[0, len(source)]``. Lazy: only index + arithmetic happens up front; records are produced on demand. + + The wrapped source must implement ``__len__`` and ``__getitem__``. + + Args: + source: The underlying indexable source (defaults to ``None``; validated lazily on first use). + start: Inclusive start index (``None`` ⇒ 0; a negative value counts from the end). + stop: Exclusive stop index (``None`` ⇒ len(source); a negative value counts from the end). + """ + + def __init__(self, source: Any = None, start: Optional[int] = None, stop: Optional[int] = None) -> None: + # Lazy / zero-arg: store config only; the index arithmetic (and source validation) is deferred + # to the ``indices`` property so the source can be configured post-construction. + self.source = source + self.start = start + self.stop = stop + self._indices: Optional[List[int]] = None + + @property + def indices(self) -> List[int]: + """The contiguous ``[start:stop)`` source indices, computed lazily on first access and cached.""" + if self._indices is None: + source = self.source + if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + n = len(source) + s = 0 if self.start is None else self.start + e = n if self.stop is None else self.stop + if s < 0: + s = max(0, n + s) + if e < 0: + e = max(0, n + e) + s = max(0, min(s, n)) + e = max(s, min(e, n)) + self._indices = list(range(s, e)) + logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) + return self._indices + + def __iter__(self) -> Iterator[Record]: + for idx in self.indices: + yield _pass_through(self.source[idx]) + + def __getitem__(self, index: int) -> Any: + return _pass_through(self.source[self.indices[index]]) + + def __len__(self) -> int: + return len(self.indices) diff --git a/recordstream/sources/split.py b/recordstream/sources/split.py new file mode 100644 index 0000000..827cd6e --- /dev/null +++ b/recordstream/sources/split.py @@ -0,0 +1,190 @@ +"""``DatasetSplit`` — reproducible train/val/test views over an indexable source.""" + +import random +from typing import Any, Dict, Iterator, List, Literal, Optional, get_args + +from confluid import configurable + +from recordstream.items import Record +from recordstream.sources.base import _pass_through + +# Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer +# closed Literals over bare strings — self-documenting + machine-introspectable by UIs / +# navigaitor form-spec / MCP schemas via ``typing.get_args``). The runtime-validation tuple +# is derived from the Literal so there is ONE source of truth — never restate the values. +SplitName = Literal["train", "val", "test"] +_SPLIT_NAMES = get_args(SplitName) + + +@configurable(category="source") +class DatasetSplit: + """ + Splits an indexable source into reproducible ``train`` / ``val`` / ``test`` views. + + A ``source`` (it yields records and is wired into a trainer's ``source:`` slot), + not an engine — it applies no ops, it just exposes a reproducible partition of another + source. (For a contiguous index slice use :class:`~recordstream.sources.RangeSource`; to + concatenate several sources use :class:`~recordstream.sources.ConcatSource`.) + + **Property API (preferred).** Configure ONE ``DatasetSplit`` with ``seed`` and the + held-out fraction(s) (``val_fraction`` and/or ``test_fraction``) and read the three + cached view sources off it:: + + split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) + split.train # ≈80% — the remainder + split.val # ≈10% + split.test # ≈10% + + The views are disjoint and complementary, computed once (cached) over a single + deterministic shuffle, so the underlying source is consumed once. In Confluid YAML the + views are reachable by **attribute reference** — ``!ref:my_split.train`` / ``.val`` / + ``.test`` — and because two ``!ref:`` to the same key flow the *same* instance, the + partition and the source load are shared across all three references:: + + my_split: !class:recordstream.sources.split.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + + train_set: !class:recordstream.core.Stream() + source: !ref:my_split.train + val_set: !class:recordstream.core.Stream() + source: !ref:my_split.val + + **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one + view (``split=None`` ⇒ ``train``), so it is directly usable as a single ``source:``. + + Omit ``test_fraction`` for a plain two-way train/val split; omit both fractions for a + degenerate split where ``train`` is the whole source and ``val`` / ``test`` are empty. + + The wrapped source must implement ``__len__`` and ``__getitem__``. Lazy: only index + arithmetic happens up front; records are produced on demand. + + Args: + source: The underlying indexable source (defaults to ``None``; validated lazily on first use). + split: View this iterates as a source — ``train`` / ``val`` / ``test`` (``None`` ⇒ ``train``). + val_fraction: Fraction of records assigned to the ``val`` view. Must be in ``(0, 1)``. + test_fraction: Fraction of records assigned to the ``test`` view. Must be in ``(0, 1)``. + seed: Seed for the deterministic shuffle. Required when any fraction is set. + """ + + def __init__( + self, + source: Any = None, + split: Optional[SplitName] = None, + val_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + seed: Optional[int] = None, + ) -> None: + # Lazy / zero-arg: store config only. All validation is deferred to first materialization + # (``_validate``, invoked from ``_view``) so the source can be configured post-construction. + self.source = source + self.split = split + self.val_fraction = val_fraction + self.test_fraction = test_fraction + self.seed = seed + # Cache of materialized split views. Underscore-prefixed so confluid's + # vars(obj)-based discovery / dump ignores it (the `train`/`val`/`test` + # @property descriptors live on the class, not in vars(obj), so they never + # surface as configurable attributes either). + self._views: Dict[str, "_SplitView"] = {} + + def _validate(self) -> None: + """Validate the (post-construction) configuration. Called lazily before the first partition.""" + source = self.source + if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + if self.split is not None and self.split not in _SPLIT_NAMES: + raise ValueError(f"split must be one of {_SPLIT_NAMES}; got {self.split!r}") + if (self.val_fraction is not None or self.test_fraction is not None) and self.seed is None: + raise ValueError("DatasetSplit requires `seed` when a fraction is set, so the partition is reproducible.") + if self.val_fraction is not None and not (0.0 < self.val_fraction < 1.0): + raise ValueError(f"val_fraction must be in (0, 1); got {self.val_fraction}") + if self.test_fraction is not None and not (0.0 < self.test_fraction < 1.0): + raise ValueError(f"test_fraction must be in (0, 1); got {self.test_fraction}") + if (self.val_fraction or 0.0) + (self.test_fraction or 0.0) >= 1.0: + raise ValueError( + "val_fraction + test_fraction must be < 1 (to leave a non-empty train split); " + f"got val_fraction={self.val_fraction}, test_fraction={self.test_fraction}" + ) + + def _partition(self) -> Dict[str, List[int]]: + """Deterministically partition the source indices into ``train`` / ``val`` / ``test``. + + One shuffle seeded by ``seed`` (skipped when no fraction is set, so the degenerate + "all train" case keeps source order); layout is ``[train | val | test]``. ``max(1, …)`` + guarantees a held-out split gets at least one record on tiny sources. + """ + n = len(self.source) + val_fraction = self.val_fraction or 0.0 + test_fraction = self.test_fraction or 0.0 + shuffled = list(range(n)) + if val_fraction or test_fraction: + random.Random(self.seed).shuffle(shuffled) + val_count = max(1, int(round(n * val_fraction))) if val_fraction else 0 + test_count = max(1, int(round(n * test_fraction))) if test_fraction else 0 + train_count = max(0, n - val_count - test_count) + return { + "train": shuffled[:train_count], + "val": shuffled[train_count : train_count + val_count], + "test": shuffled[train_count + val_count :], + } + + def _view(self, split: SplitName) -> "_SplitView": + if split not in self._views: + self._validate() + self._views[split] = _SplitView(self.source, self._partition()[split]) + return self._views[split] + + @property + def train(self) -> "_SplitView": + """Cached training-split view (the remainder after ``val`` / ``test`` are held out).""" + return self._view("train") + + @property + def val(self) -> "_SplitView": + """Cached validation-split view (≈ ``val_fraction`` of the source).""" + return self._view("val") + + @property + def test(self) -> "_SplitView": + """Cached test-split view (≈ ``test_fraction`` of the source).""" + return self._view("test") + + def __iter__(self) -> Iterator[Record]: + return iter(self._view(self.split or "train")) + + def __getitem__(self, index: int) -> Any: + return self._view(self.split or "train")[index] + + def __len__(self) -> int: + return len(self._view(self.split or "train")) + + +class _SplitView: + """An indexable view of ``source`` restricted (and reordered) to ``indices``. + + Internal to :class:`DatasetSplit` — produced by its ``train`` / ``val`` / ``test`` + properties (and reachable in Confluid YAML via ``!ref:my_split.train``). Deliberately + NOT a ``@configurable``: it is never constructed directly in a config, only read off a + live ``DatasetSplit`` instance, so it carries no discovery surface of its own. It stays + in this module for the same reason — it is DatasetSplit's own return type, not a + separately-exported source. + """ + + def __init__(self, source: Any, indices: List[int]) -> None: + self.source = source + self.indices = indices + + def __iter__(self) -> Iterator[Record]: + for idx in self.indices: + yield _pass_through(self.source[idx]) + + def __getitem__(self, index: int) -> Any: + return _pass_through(self.source[self.indices[index]]) + + def __len__(self) -> int: + return len(self.indices) diff --git a/recordstream/transform.py b/recordstream/transform.py index f1517bb..69fa097 100644 --- a/recordstream/transform.py +++ b/recordstream/transform.py @@ -17,7 +17,7 @@ recordstream ships NO native augmentation ops — geometric/photometric augmentation comes from the libraries (torchvision ``transforms.v2`` / albumentations) dropped into an ops list -AS-IS; the engine invokes each op family natively (see ``recordstream.core._apply_op``). +AS-IS; the engine invokes each op family natively (see ``recordstream.core.families._apply_op``). There are no wrapper/adapter classes. """ diff --git a/tests/test_module_layout.py b/tests/test_module_layout.py new file mode 100644 index 0000000..34c2cbc --- /dev/null +++ b/tests/test_module_layout.py @@ -0,0 +1,166 @@ +"""Layout invariants of the engine PACKAGES — `sources`, `core`, `flow`. + +Each was one oversized module and is now one module per cohesive unit (architecture §11/§12). +That split made three things load-bearing that a casual tidy-up would undo, and every one of +them fails SILENTLY in production: + +* the SUBMODULE path is the canonical ``!class:`` spelling, because ``cls.__module__`` is what + every generator writes — so nobody may pin ``__module__`` back to the package (that also + breaks ``confluid.registry.key_for``, whose miss only surfaces once a namesake registers); +* ``__init__.py``'s ``__all__`` is what a visual editor's node bridge walks, because + ``discovery.scan_module`` filters on ``__module__`` and now sees nothing in a package; +* the IMPORT DIRECTION inside ``core`` / ``flow`` — ``core`` is the bottom of the op-facing + layer (architecture §5), and a top-level import pointing the wrong way closes a cycle. +""" + +import ast +import importlib +import inspect +from pathlib import Path +from typing import Any, List, Set + +import pytest +from confluid.pydantic_export import _qualname +from confluid.registry import get_registry, resolve_class + +import recordstream.core as core_pkg +import recordstream.flow as flow_pkg +import recordstream.sources as sources_pkg +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp +from recordstream.discovery import scan_module +from recordstream.flow import FlowGraph +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource + +#: Every public engine class and the module it must be DEFINED in (not merely re-exported from). +CLASS_MODULES = { + HuggingFaceSource: "recordstream.sources.huggingface", + DatasetSplit: "recordstream.sources.split", + RangeSource: "recordstream.sources.range", + ConcatSource: "recordstream.sources.concat", + Stream: "recordstream.core.stream", + JointStream: "recordstream.core.stream", + FilterOp: "recordstream.core.wrappers", + WrappedOp: "recordstream.core.wrappers", + FlowGraph: "recordstream.flow.graph", +} + +#: package -> the submodules whose @configurable classes it must re-export. +PACKAGES = { + sources_pkg: ["huggingface", "split", "range", "concat"], + core_pkg: ["families", "mapstyle", "wrappers", "stream"], + flow_pkg: ["steps", "parse", "execute", "graph"], +} + + +def _module_level_imports(module_name: str) -> Set[str]: + """The modules ``module_name`` imports at MODULE level (body-local imports excluded). + + Body-local imports are the sanctioned seam for the one direction that must stay lazy, so + only top-level statements count here. + """ + source = Path(inspect.getsourcefile(importlib.import_module(module_name)) or "").read_text() + imported: Set[str] = set() + for node in ast.parse(source).body: # top level only — never recurse into function bodies + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + imported.add(node.module) + return imported + + +@pytest.mark.parametrize("cls,module", list(CLASS_MODULES.items()), ids=lambda v: getattr(v, "__name__", v)) +def test_each_class_lives_in_its_declared_module(cls: type, module: str) -> None: + """``__module__`` reports the defining submodule — never the package.""" + assert cls.__module__ == module + # Pinning __module__ back to the package would make the assert above pass by lying; + # getsource is what catches that (it searches the named module's file for the definition). + assert inspect.getsource(cls).lstrip().startswith("@configurable") + + +@pytest.mark.parametrize("cls,module", list(CLASS_MODULES.items()), ids=lambda v: getattr(v, "__name__", v)) +def test_the_canonical_class_path_is_the_submodule_one(cls: type, module: str) -> None: + """``_qualname`` is what a generated config / form-spec / enrichment key spells out.""" + assert _qualname(cls) == f"{module}.{cls.__name__}" + assert resolve_class(_qualname(cls)) is cls + + +@pytest.mark.parametrize("cls", list(CLASS_MODULES), ids=lambda v: v.__name__) +def test_the_package_re_export_still_resolves(cls: type) -> None: + """An older hand-written config spelling the PACKAGE path keeps loading.""" + package = cls.__module__.rsplit(".", 1)[0] + assert resolve_class(f"{package}.{cls.__name__}") is cls + + +@pytest.mark.parametrize("cls", list(CLASS_MODULES), ids=lambda v: v.__name__) +def test_pinning_module_would_break_the_registry_lookup(cls: type) -> None: + """``key_for`` re-derives ``module.qualname``; a rewritten ``__module__`` misses the entry. + + This is the failure mode that ruled out keeping the old paths by pinning ``__module__`` — + it is silent, surfacing only as an ambiguous ``!class:`` tag once a namesake registers. + """ + assert get_registry().key_for(cls) is not None + + +@pytest.mark.parametrize("package,submodules", list(PACKAGES.items()), ids=lambda v: getattr(v, "__name__", "")) +def test_every_configurable_is_in_all(package: Any, submodules: List[str]) -> None: + """``__all__`` is the ONLY pass that surfaces these as palette nodes — see the module docstring.""" + exported = getattr(package, "__all__", []) + for name in submodules: + module_path = f"{package.__name__}.{name}" + module = importlib.import_module(module_path) + for attr, member in vars(module).items(): + if attr.startswith("_") or not isinstance(member, type): + continue + if getattr(member, "__confluid_configurable__", False) and member.__module__ == module_path: + assert attr in exported, f"{module_path}.{attr} is @configurable but not in __all__" + + +@pytest.mark.parametrize("package", list(PACKAGES), ids=lambda v: getattr(v, "__name__", "")) +def test_scan_module_no_longer_sees_the_package(package: Any) -> None: + """The reason ``__all__`` matters: the ``__module__`` filter finds nothing in a package.""" + assert scan_module(package.__name__) == [] + + +def test_the_engine_internals_stay_importable_from_the_package() -> None: + """``ops/`` reaches the op dispatch as ``from recordstream.core import _apply_op``. + + The private re-exports in ``core/__init__.py`` are a deliberate cross-module surface, not + stray imports — dropping one breaks every composing op at run time, not at import. + """ + for name in ("_apply_op", "_op_expands", "_expand", "_extra_op_families", "_sync_op_families", "_worker_task"): + assert hasattr(core_pkg, name), f"recordstream.core.{name} is the engine's internal surface" + # A mutable list re-exported by IDENTITY — this is what lets the suite's registry fixture + # restore the real families with `core._OP_FAMILIES[:] = snapshot`. + from recordstream.core.families import _OP_FAMILIES + + assert core_pkg._OP_FAMILIES is _OP_FAMILIES + + +def test_core_families_is_the_bottom_of_the_op_facing_layer() -> None: + """``core.families`` imports NOTHING from its siblings — architecture §5's whole argument. + + Every composing op imports ``_apply_op`` from here; a top-level import back into the engine + (or into ``flow``) would close the cycle that record exists to prevent. + """ + imported = _module_level_imports("recordstream.core.families") + offenders = [m for m in imported if m.startswith(("recordstream.core.", "recordstream.flow", "recordstream.ops"))] + assert offenders == [], f"core.families must not import {offenders} at module level" + + +def test_flow_reaches_core_only_through_the_dispatch_layer() -> None: + """``flow`` may import ``core.families``; importing ``core.stream`` would invert the layering. + + ``core.stream`` reaches ``flow`` through BODY-LOCAL imports precisely so this direction can + stay a module-level one. + """ + for name in ("steps", "parse", "execute", "graph"): + imported = _module_level_imports(f"recordstream.flow.{name}") + assert "recordstream.core.stream" not in imported, f"flow.{name} must not import core.stream at module level" + assert "recordstream.core" not in imported, f"flow.{name} must reach the dispatch via core.families" + + +def test_core_stream_defers_its_flow_imports() -> None: + """The one direction that MUST stay lazy: ``core.stream`` -> ``flow`` is body-local only.""" + imported = _module_level_imports("recordstream.core.stream") + offenders = [m for m in imported if m.startswith("recordstream.flow")] + assert offenders == [], f"core.stream must import flow inside function bodies, not {offenders}" diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index 48a3e25..83f5f01 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -243,16 +243,19 @@ class TestNativeExecution: def test_reader_accounting_is_computed_once_per_graph(self, monkeypatch: Any) -> None: # _result_readers depends only on (steps, outputs); recomputing it per record was an # O(steps^2) tax measured at ~half the graph engine's overhead over a flat op list. - import recordstream.flow as flow_mod + # Patch the module that USES the name, not the one that defines it: `flow.graph` + # imports `_result_readers` from `flow.execute`, so it holds its own binding and + # patching `flow.execute` (or the `recordstream.flow` package) would miss. + import recordstream.flow.graph as graph_mod calls = {"n": 0} - real = flow_mod._result_readers + real = graph_mod._result_readers def counting(steps: Any, outputs: str) -> Any: calls["n"] += 1 return real(steps, outputs) - monkeypatch.setattr(flow_mod, "_result_readers", counting) + monkeypatch.setattr(graph_mod, "_result_readers", counting) graph = FlowGraph(source=[_seed(1.0) for _ in range(25)], flow={"plus": _AddOffset(offset=2.0)}) assert len(list(graph)) == 25 assert calls["n"] == 1 From 5bf95b96b2b665abf700a2c425f20b8237094b50 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 1 Aug 2026 17:16:37 +0200 Subject: [PATCH 067/102] docs(sources): spell the canonical Stream path in DatasetSplit's YAML example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package-split sweep excluded recordstream/sources/, so DatasetSplit's docstring still showed `!class:recordstream.core.Stream()`. Both spellings resolve, but an example is what gets copied — show the submodule path a generated config emits. --- recordstream/sources/split.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recordstream/sources/split.py b/recordstream/sources/split.py index 827cd6e..609f4ac 100644 --- a/recordstream/sources/split.py +++ b/recordstream/sources/split.py @@ -47,9 +47,9 @@ class DatasetSplit: test_fraction: 0.1 seed: 42 - train_set: !class:recordstream.core.Stream() + train_set: !class:recordstream.core.stream.Stream() source: !ref:my_split.train - val_set: !class:recordstream.core.Stream() + val_set: !class:recordstream.core.stream.Stream() source: !ref:my_split.val **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one From b3bbade1135b5210c6cbf9c4678e218ccd279622 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 1 Aug 2026 17:16:45 +0200 Subject: [PATCH 068/102] docs(architecture): drop the deleted context modules from the overview table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'Graph wiring' row still named `context.py` / `ops/context.py` and the 'Native ops' row still advertised context ops — all deleted 2026-07-30 with the lowering pass. Wiring is step GRAMMAR now (`from:`/`merge_from:`/`bind:`), parsed in flow/steps.py + flow/parse.py. Both rows also linked a stale §3 anchor (the section was retitled in the same 2026-07-30 change), so the two links 404'd. Audited every in-document anchor across the repo's markdown while here; this was the only broken one. --- docs/architecture.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 2ac6455..a13a24b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,10 +18,10 @@ Maintenance rules: | Layer | Modules | What it is | Where the *why* lives | |---|---|---|---| | Data model | `items.py`, `io.py` | A record is a plain `dict` of typed values; one codec serializes any value | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | -| Native ops | `transform.py`, `dispatch.py`, `ops/*` | Type-dispatched `Transform`s (kernels, `field=`) + structural/compose/context ops | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Native ops | `transform.py`, `dispatch.py`, `ops/*` | Type-dispatched `Transform`s (kernels, `field=`) + structural/compose ops | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | | Library interop | `core._apply_op`, `register_op_family` | External libraries run as-is via the op-family dispatch — no adapters | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | -| Engines | `core/` (`Stream`/`JointStream`), `flow/` (`FlowGraph`) | One op-application chokepoint, four routes; one per-record kernel behind two authoring forms | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17), [§5](#5-the-engines-own-callable-wrappers-live-in-core-2026-07-20-still-true-after-the-2026-08-01-package-split) | -| Graph wiring | `context.py`, `ops/context.py` | Fan-out/fan-in/cross-branch values on the plain sequential engine | [§3](#3-the-per-record-context-is-an-ambient-wiring-plane-recordstreamcontext-2026-07-17) | +| Engines | `core/` (`Stream`/`JointStream`), `flow/` (`FlowGraph`) | One op-application chokepoint, four routes; one per-record kernel behind two authoring forms | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-graph-is-the-execution-model--the-lowering-pass-was-deleted-2026-07-30), [§5](#5-the-engines-own-callable-wrappers-live-in-core-2026-07-20-still-true-after-the-2026-08-01-package-split) | +| Graph wiring | `flow/steps.py`, `flow/parse.py` | Fan-out/fan-in/cross-step values are step GRAMMAR (`from:`/`merge_from:`/`bind:`), never ops | [§3](#3-the-graph-is-the-execution-model--the-lowering-pass-was-deleted-2026-07-30) | | Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17) | | Storage & query | `storage/*` | The `typedrecord-v1` key-group layout over the codec; metadata scans without array loads | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) (contracts) + [storage.md](storage.md) | | Introspection & serialization | `discovery.py` | Callable↔string identity + registration-free module scans | [§4](#4-callablestring-serialization--passive-introspection-recordstreamdiscovery-2026-07-20) | From d32f34c15d39b57b2c8ffa732c3074c1ef6207c7 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 1 Aug 2026 19:41:09 +0200 Subject: [PATCH 069/102] docs: clarify the runnable-protocol mandate in AGENTS.md --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 869a653..7c3bd38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → ONE step-graph engine behind two facades (`Stream`/`JointStream` for the dataset surface, `FlowGraph` for a `flow:` document) → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config; the context ops + the flow⇄ops lowering pass were DELETED 2026-07-30 (one step-graph engine, see the mandate below). Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair lightning train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and marainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. - **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). From f884d23b80d7790d744a88509fd58bd6bf34adff Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 2 Aug 2026 14:15:13 +0200 Subject: [PATCH 070/102] feat(sources): a source can name the data it reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `recordstream.uri` is the opt-in identity protocol: `dataset_uri` is the canonical, machine-parseable handle (stable across machines) and `dataset_url` the human one, with the same shape as `recordstream.projection` — a Protocol rather than a base class, free helpers that materialize a deferred source first, and `None` for "this source has no such handle", which is not an error. `HuggingFaceSource` implements both: a Hub repo id resolves to the hub URI, a local path to its `file://` form, and the split/count/feature selection rides as a query so two views of the same dataset stay distinguishable. A run record that says which data produced it needs exactly this, and the source is the only thing that knows it. --- recordstream/sources/huggingface.py | 77 ++++++++++- recordstream/uri.py | 168 +++++++++++++++++++++++ tests/test_dataset_uri.py | 200 ++++++++++++++++++++++++++++ 3 files changed, 443 insertions(+), 2 deletions(-) create mode 100644 recordstream/uri.py create mode 100644 tests/test_dataset_uri.py diff --git a/recordstream/sources/huggingface.py b/recordstream/sources/huggingface.py index 7319a79..b85bfc3 100644 --- a/recordstream/sources/huggingface.py +++ b/recordstream/sources/huggingface.py @@ -1,6 +1,8 @@ """``HuggingFaceSource`` — a Hugging Face dataset as a stream of record dicts.""" -from typing import Any, Collection, Iterator, List, Optional +from pathlib import Path +from typing import Any, Collection, Dict, Iterator, List, Optional +from urllib.parse import quote, urlencode from confluid import configurable from loggair import get_logger @@ -9,6 +11,18 @@ logger = get_logger(__name__) +#: Scheme of the canonical identifier for a Hub dataset. Matches the convention hosted +#: tracking services already use for a dataset source, so a URI recorded here is the one +#: their UI expects rather than a spelling invented for this package. +HF_URI_PREFIX = "hf://datasets/" + +#: Where a Hub dataset is browsable. The ``/viewer//`` suffix opens the +#: dataset viewer on exactly the rows this source reads. +HF_BROWSE_PREFIX = "https://huggingface.co/datasets/" + +#: The config name the Hub viewer uses when a dataset declares no named configs. +HF_DEFAULT_CONFIG = "default" + # Sentinel for ``HuggingFaceSource.metadata_features`` meaning "every dataset column except the # input/target features" — the full-traceability option, kept OPT-IN (``None`` / ``[]`` still = no # extra metadata) so existing configs are unaffected. Resolved against the loaded dataset's @@ -62,6 +76,11 @@ class HuggingFaceSource: only on first access to :attr:`dataset` (cached thereafter; reset ``_dataset`` to reload). ``path`` is therefore optional at construction and validated lazily when the data is needed. + It also carries its own IDENTITY (:mod:`recordstream.uri`): :attr:`dataset_uri` names the + dataset canonically (``hf://datasets/ylecun/mnist?split=train``, or a ``file://`` URI for a + local imagefolder) and :attr:`dataset_url` links to the Hub viewer for the same rows. Both + read stored configuration only — asking either never loads anything. + Args: path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. split: HF split name (``train`` / ``validation`` / ``test`` / etc.). @@ -116,10 +135,64 @@ def dataset(self) -> Any: ) from datasets import load_dataset - logger.info(f"HuggingFaceSource: Loading {self.path} ({self.split})...") + # The identifier goes in the LOAD line: it is the one moment a reader of the log + # can tie the run to a specific dataset, and the browsable URL is what makes that + # tie followable rather than merely recorded. + logger.info(f"HuggingFaceSource: Loading {self.dataset_url or self.dataset_uri}...") self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self._load_kwargs) return self._dataset + # -- identity (see recordstream.uri) ------------------------------------------------------ + + @property + def _identity_query(self) -> str: + """The ``name`` / ``revision`` / ``split`` selection as a sorted query string. + + Sorted so two identically-configured sources produce the SAME string — a URI whose + parameter order depended on insertion would not compare equal to itself. + """ + parts: Dict[str, str] = {} + if self.name: + parts["name"] = str(self.name) + revision = self._load_kwargs.get("revision") + if revision: + parts["revision"] = str(revision) + if self.split: + parts["split"] = str(self.split) + return urlencode(sorted(parts.items())) + + @property + def dataset_uri(self) -> Optional[str]: + """Canonical identifier for the dataset this source reads — ``None`` without a ``path``. + + A Hub repo id becomes ``hf://datasets/?…``; a local directory becomes its + ``file://`` URI. Which one applies is decided by whether ``path`` exists on disk — + the same question ``datasets.load_dataset`` itself answers. Pure string work over the + stored configuration: nothing is loaded, so an unconsumed source still answers. + """ + if not self.path: + return None + local = Path(self.path) + base = local.resolve().as_uri() if local.exists() else HF_URI_PREFIX + quote(str(self.path).strip("/")) + query = self._identity_query + return f"{base}?{query}" if query else base + + @property + def dataset_url(self) -> Optional[str]: + """Browsable Hub link, or ``None`` for a local dataset (which has no web page). + + Points at the dataset VIEWER on this source's config + split when a split is + configured, so the link opens on the rows this source reads rather than on the + repository's front page. + """ + if not self.path or Path(self.path).exists(): + return None + page = HF_BROWSE_PREFIX + quote(str(self.path).strip("/")) + if not self.split: + return page + config = quote(str(self.name)) if self.name else HF_DEFAULT_CONFIG + return f"{page}/viewer/{config}/{quote(str(self.split))}" + @property def resolved_metadata_features(self) -> List[str]: """``metadata_features`` resolved against the live dataset's columns (expands the ``"*"`` sentinel). diff --git a/recordstream/uri.py b/recordstream/uri.py new file mode 100644 index 0000000..62e24e4 --- /dev/null +++ b/recordstream/uri.py @@ -0,0 +1,168 @@ +"""Dataset identity — the URI/URL a source carries for the data it reads. + +A run record that says *which* data produced it needs a stable, comparable handle on +that data. A source already knows one: a Hub repo id, a directory on disk, a query +against a store. This module is the opt-in protocol for exposing it plus the walk +helpers a consumer uses, with the same shape as :mod:`recordstream.projection` — +a ``Protocol`` (never a base class), free functions that materialize a deferred +source first, and ``None`` for "this source has no such handle", which is not an error. + +Two handles, not one, because they answer different questions: + +* **``dataset_uri``** — the CANONICAL identifier: machine-parseable, stable across + machines, and the thing two runs are compared on. ``hf://datasets/ylecun/mnist?split=train``. +* **``dataset_url``** — a link a HUMAN can open, or ``None`` when the data has no web + page (a local directory has none, and inventing one would be a lie). + +Design notes +------------ +* **A wrapper propagates the URI VERBATIM.** The handle identifies the *dataset*; how + much of it a run consumed (a split, a slice, a filter) is a different fact, carried + by the wrapper's own configuration. Decorating the string would mean the same dataset + reached through two wrappers no longer compares equal, which is the one property the + handle exists to have. +* **Following happens HERE, not in every wrapper.** :func:`dataset_uri` follows a + ``.source`` attribute when the object holds no handle of its own, so every view + source in this package — and any third-party wrapper using the same attribute name — + works with no code of its own. +* **A concatenation declines.** Several datasets end to end are not one dataset, so the + singular functions answer ``None`` for a source holding ``.sources``; use + :func:`dataset_uris`, which fans out over the members. +""" + +from typing import Any, List, Optional, Protocol, runtime_checkable + +#: How many ``.source`` hops :func:`dataset_uri` will follow before giving up. Wrapping +#: is shallow in practice (a split over a range over a source is already unusual); the +#: cap is what keeps a cyclic ``source`` reference from hanging a tracking call. +MAX_WRAPPER_DEPTH = 16 + + +@runtime_checkable +class SupportsDatasetIdentity(Protocol): + """A source that can name the dataset it reads. + + Both members are properties and both may answer ``None`` — a source configured + with no dataset yet (the zero-arg construction convention) has no identity, and a + dataset with no web page has no URL. Implementations MUST be cheap and side-effect + free: this is asked of a source that may never be iterated, so it reads stored + configuration and never loads, downloads or opens anything. + """ + + @property + def dataset_uri(self) -> Optional[str]: + """Canonical, machine-parseable identifier for the data this source reads.""" + ... + + @property + def dataset_url(self) -> Optional[str]: + """Browsable link to the data, or ``None`` when it has no web page.""" + ... + + +def _identity(source: Any, attribute: str) -> Optional[str]: + """Read ``attribute`` off ``source``, following ``.source`` wrappers. + + The shared body of :func:`dataset_uri` and :func:`dataset_url`: materialize a + deferred source, read the handle if it has one, otherwise take one hop into the + wrapped source and ask again. + """ + from confluid import flow + + current = flow(source) + for _ in range(MAX_WRAPPER_DEPTH): + if current is None: + return None + value = getattr(current, attribute, None) + if value: + return str(value) + wrapped = getattr(current, "source", None) + if wrapped is None or wrapped is current: + return None + current = flow(wrapped) + return None + + +def dataset_uri(source: Any) -> Optional[str]: + """The canonical URI identifying the dataset ``source`` reads, or ``None``. + + A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized + first, exactly as :func:`~recordstream.projection.project` does, so a caller never + has to remember which entry point flows and which does not. + + A wrapper that holds no URI of its own but wraps another source via ``.source`` — + every view source in this package does — reports the wrapped source's URI + unchanged. A wrapper holding SEVERAL sources answers ``None``; ask + :func:`dataset_uris` instead. + + Args: + source: Any source, view, stream, or deferred config marker. ``None`` is accepted + and answers ``None``, so a caller needs no guard for an unwired slot. + + Example:: + + dataset_uri(HuggingFaceSource(path="ylecun/mnist")) # hf://datasets/ylecun/mnist?split=train + dataset_uri(Stream(source=split.train)) # the same string — the split is not part of it + """ + return _identity(source, "dataset_uri") + + +def dataset_url(source: Any) -> Optional[str]: + """The browsable URL for the dataset ``source`` reads, or ``None``. + + The human-facing twin of :func:`dataset_uri`, with identical flowing and wrapper + rules. ``None`` is the ordinary answer for data with no web page — a directory on + disk, a store on a mounted volume — and never an error. + + Args: + source: Any source, view, stream, or deferred config marker. + """ + return _identity(source, "dataset_url") + + +def dataset_uris(source: Any) -> List[str]: + """Every dataset URI reachable from ``source``, in order, deduplicated. + + The plural form, for a source that concatenates several datasets: it fans out over + a ``.sources`` list (recursively, so a concatenation of concatenations works) while + a single source yields its one URI. Sources with no URI contribute nothing rather + than a ``None`` entry, so the result is directly usable. + + Args: + source: Any source, view, stream, or deferred config marker. + + Example:: + + dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] + """ + from confluid import flow + + found: List[str] = [] + + def visit(node: Any, depth: int) -> None: + if node is None or depth > MAX_WRAPPER_DEPTH: + return + node = flow(node) + uri = dataset_uri(node) + if uri: + if uri not in found: + found.append(uri) + return + # No single identity: a concatenation is the case worth descending into. `.source` + # is already covered by `dataset_uri` above, so only the plural attribute is left. + members = getattr(node, "sources", None) + if isinstance(members, (list, tuple)): + for member in members: + visit(member, depth + 1) + + visit(source, 0) + return found + + +__all__ = [ + "MAX_WRAPPER_DEPTH", + "SupportsDatasetIdentity", + "dataset_uri", + "dataset_uris", + "dataset_url", +] diff --git a/tests/test_dataset_uri.py b/tests/test_dataset_uri.py new file mode 100644 index 0000000..a97e7f9 --- /dev/null +++ b/tests/test_dataset_uri.py @@ -0,0 +1,200 @@ +"""Dataset identity — ``recordstream.uri`` and ``HuggingFaceSource``'s implementation of it.""" + +from pathlib import Path +from typing import Any, List, Optional + +import pytest + +from recordstream import ( + ConcatSource, + DatasetSplit, + HuggingFaceSource, + RangeSource, + Stream, + SupportsDatasetIdentity, + dataset_uri, + dataset_uris, + dataset_url, +) +from recordstream.uri import MAX_WRAPPER_DEPTH + + +class _FakeSource: + """A minimal indexable source, so a view can wrap something with a known length.""" + + def __init__(self, size: int = 6, uri: Optional[str] = None) -> None: + self._size = size + self.dataset_uri = uri # type: ignore[assignment] + self.dataset_url = None # type: ignore[assignment] + + def __len__(self) -> int: + return self._size + + def __getitem__(self, index: int) -> Any: + return {"class": index} + + def __iter__(self) -> Any: + return iter(self[i] for i in range(self._size)) + + +# --- HuggingFaceSource's identity ------------------------------------------------------------ + + +def test_a_hub_repo_id_becomes_an_hf_uri_and_a_viewer_url() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert source.dataset_uri == "hf://datasets/ylecun/mnist?split=train" + assert source.dataset_url == "https://huggingface.co/datasets/ylecun/mnist/viewer/default/train" + + +def test_config_name_and_revision_ride_the_uri_and_the_url() -> None: + source = HuggingFaceSource(path="ylecun/mnist", name="fashion", split="test", revision="abc123") + assert source.dataset_uri == "hf://datasets/ylecun/mnist?name=fashion&revision=abc123&split=test" + assert source.dataset_url == "https://huggingface.co/datasets/ylecun/mnist/viewer/fashion/test" + + +def test_query_parameters_are_sorted_so_one_config_has_exactly_one_uri() -> None: + """Two identically-configured sources must produce the SAME string, whatever the kwarg order.""" + one = HuggingFaceSource(path="a/b", split="train", name="cfg", revision="r1") + other = HuggingFaceSource(path="a/b", revision="r1", name="cfg", split="train") + assert one.dataset_uri == other.dataset_uri + + +def test_a_local_directory_becomes_a_file_uri_with_no_browsable_url(tmp_path: Path) -> None: + source = HuggingFaceSource(path=str(tmp_path), split="train") + assert source.dataset_uri == f"{tmp_path.resolve().as_uri()}?split=train" + # Data on disk has no web page, and inventing a Hub URL for it would be a lie. + assert source.dataset_url is None + + +def test_an_unconfigured_source_has_no_identity() -> None: + source = HuggingFaceSource() + assert source.dataset_uri is None + assert source.dataset_url is None + + +def test_a_source_without_a_split_links_to_the_dataset_page() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="") + assert source.dataset_uri == "hf://datasets/ylecun/mnist" + assert source.dataset_url == "https://huggingface.co/datasets/ylecun/mnist" + + +def test_asking_for_identity_never_loads_the_dataset() -> None: + """The whole point of reading stored config: a source that is never iterated still answers.""" + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert source.dataset_uri is not None + assert source._dataset is None # nothing was materialized + + +def test_huggingface_source_satisfies_the_protocol() -> None: + assert isinstance(HuggingFaceSource(path="a/b"), SupportsDatasetIdentity) + + +# --- the free functions ---------------------------------------------------------------------- + + +def test_the_free_functions_read_a_sources_own_identity() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert dataset_uri(source) == source.dataset_uri + assert dataset_url(source) == source.dataset_url + + +def test_none_and_an_identity_less_source_answer_none() -> None: + assert dataset_uri(None) is None + assert dataset_url(None) is None + assert dataset_uri(_FakeSource()) is None + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda src: Stream(source=src), id="stream"), + pytest.param(lambda src: RangeSource(source=src, start=0, stop=2), id="range"), + pytest.param(lambda src: DatasetSplit(source=src, val_fraction=0.5, seed=0), id="split"), + pytest.param(lambda src: Stream(source=RangeSource(source=src, start=0, stop=2)), id="nested"), + ], +) +def test_a_wrapper_reports_the_wrapped_datasets_identity_verbatim(wrap: Any) -> None: + """A view identifies the same DATASET; how much of it this run reads is a separate fact.""" + source = HuggingFaceSource(path="ylecun/mnist", split="train") + wrapped = wrap(source) + assert dataset_uri(wrapped) == "hf://datasets/ylecun/mnist?split=train" + assert dataset_url(wrapped) == "https://huggingface.co/datasets/ylecun/mnist/viewer/default/train" + + +def test_a_split_view_reports_the_split_sources_identity() -> None: + """The three views are private objects only ever read off a live split — they follow too. + + Wrapping a local fake rather than a Hub source on purpose: reading ``.train`` PARTITIONS, + which calls ``len(source)``, which would make a Hub source download. + """ + inner = _FakeSource(size=8, uri="hf://datasets/a/b?split=train") + split = DatasetSplit(source=inner, val_fraction=0.25, seed=0) + assert dataset_uri(split.train) == "hf://datasets/a/b?split=train" + assert dataset_uri(split.val) == dataset_uri(split.train) + + +def test_a_wrapper_with_its_own_identity_wins_over_the_one_it_wraps() -> None: + inner = HuggingFaceSource(path="ylecun/mnist", split="train") + outer = _FakeSource(uri="store://curated/v2") + outer.source = inner # type: ignore[attr-defined] + assert dataset_uri(outer) == "store://curated/v2" + + +def test_a_cyclic_wrapper_chain_terminates() -> None: + """The depth cap exists so a tracking call can never hang on a self-referential source.""" + node = _FakeSource() + node.source = node # type: ignore[attr-defined] + assert dataset_uri(node) is None + + +def test_a_chain_deeper_than_the_cap_gives_up_rather_than_walking_forever() -> None: + deepest = HuggingFaceSource(path="a/b", split="train") + chain: Any = deepest + for _ in range(MAX_WRAPPER_DEPTH + 2): + wrapper = _FakeSource() + wrapper.source = chain # type: ignore[attr-defined] + chain = wrapper + assert dataset_uri(chain) is None + + +def test_a_deferred_config_marker_is_materialized_before_being_asked() -> None: + """A `!class:` marker straight out of a config must not need a `flow()` at the call site.""" + from confluid import load + + node = load( + "!class:recordstream.sources.huggingface.HuggingFaceSource()\n path: ylecun/mnist\n split: train\n", + flow=False, + ) + assert dataset_uri(node) == "hf://datasets/ylecun/mnist?split=train" + + +# --- concatenation --------------------------------------------------------------------------- + + +def test_a_concatenation_declines_to_name_one_dataset() -> None: + concat = ConcatSource(sources=[HuggingFaceSource(path="a/b"), HuggingFaceSource(path="c/d")]) + assert dataset_uri(concat) is None + assert dataset_url(concat) is None + + +def test_dataset_uris_fans_out_over_the_members() -> None: + concat = ConcatSource(sources=[HuggingFaceSource(path="a/b"), HuggingFaceSource(path="c/d")]) + assert dataset_uris(concat) == ["hf://datasets/a/b?split=train", "hf://datasets/c/d?split=train"] + + +def test_dataset_uris_deduplicates_and_descends_into_nested_concatenations() -> None: + same = HuggingFaceSource(path="a/b") + inner = ConcatSource(sources=[same, HuggingFaceSource(path="c/d")]) + outer = ConcatSource(sources=[inner, HuggingFaceSource(path="a/b")]) + assert dataset_uris(outer) == ["hf://datasets/a/b?split=train", "hf://datasets/c/d?split=train"] + + +def test_dataset_uris_of_a_single_source_is_its_one_uri() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert dataset_uris(source) == ["hf://datasets/ylecun/mnist?split=train"] + + +def test_dataset_uris_skips_members_that_have_no_identity() -> None: + concat = ConcatSource(sources=[_FakeSource(), HuggingFaceSource(path="a/b")]) + found: List[str] = dataset_uris(concat) + assert found == ["hf://datasets/a/b?split=train"] From 21c5e6f286dffc03a711d85a42e48a441f6fff9f Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 2 Aug 2026 14:15:20 +0200 Subject: [PATCH 071/102] =?UTF-8?q?feat(ops):=20ConvertToMask=20=E2=80=94?= =?UTF-8?q?=20an=20array=20to=20class-id=20Mask?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The counterpart of `ConvertToImage` on the target side: read an array-bearing key and write a `Mask` of integer class ids, so a segmentation target reaches the model as the typed item the collate and the storage codec already know how to carry. Registered `category="op"`, `group="image"` beside its sibling, and pinned in `tests/test_categories.py` so a dropped tag cannot silently empty the palette. --- docs/image.md | 34 +++++ recordstream/ops/__init__.py | 5 +- recordstream/ops/image.py | 91 ++++++++++++- tests/test_categories.py | 7 +- tests/test_convert_to_mask.py | 239 ++++++++++++++++++++++++++++++++++ 5 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 tests/test_convert_to_mask.py diff --git a/docs/image.md b/docs/image.md index 14bc484..e36dc2a 100644 --- a/docs/image.md +++ b/docs/image.md @@ -29,6 +29,40 @@ u8 = normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # fixed dB window across a `record_to_image(record, ...)` renders a record's first array-bearing (2-D / 3-D) value the same way — the ad-hoc whole-record preview for viewer tooling. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). +## Masks (`ConvertToMask`) + +The segmentation counterpart, and the same shape of op — read one field, write a differently-typed item under `output`. A segmentation dataset ships its target as a greyscale/paletted PNG whose pixel values *are* the class ids (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map); this turns that payload into the `int64` `[H, W]` `Mask` every per-pixel loss expects. + +```python +from recordstream.ops.image import ConvertToMask + +op = ConvertToMask( + field="segmentation_mask", # source key; blank picks the first array/PIL-bearing value + output="mask", # key the int64 Mask item is written to +) +``` + +It converts and **nothing else**, because the rest of the chain is ops that already exist: + +| you want | use | +| --- | --- | +| remap the ids (a 1-based trimap → 0-based) | `FormulaOp(field="mask", formula="a - 1")` | +| remap through a lookup table (Cityscapes id → trainId) | `EncodeTarget` | +| resize / augment it **together with the image** | a bare `albumentations` transform in the same ops list | +| drop the source column | `DropField(key="segmentation_mask")` | + +That fourth row is why `output` defaults to `"mask"`: it is albumentations' own key vocabulary, so the engine's op-family dispatch hands `image` **and** `mask` to one call — a single joint draw moves both, and the `Mask` type survives the round trip. An image-only transform (`Normalize`) still touches the image alone. + +```yaml +# the target half of a segmentation `preprocess` chain +- !class:recordstream.ops.image.ConvertToMask {field: segmentation_mask, output: mask} +- !class:recordstream.ops.formula.FormulaOp {field: mask, formula: a - 1} +- !class:recordstream.ops.structure.DropField {key: segmentation_mask} +- !class:albumentations.Resize {height: 224, width: 224} # image AND mask +``` + +`int64` is not a knob: a class-id map is integer by definition, and it is what `torch.nn.CrossEntropyLoss` requires (it rejects int32 with *"expected target dtype to be Long or Byte, but got Int"*). Libraries that cast on the way past — albumentations returns int32 — are corrected at the model boundary with `batch_tensor(batch, "mask", dtype=torch.int64)`, where the caller names the contract. An RGB-encoded mask is **refused** rather than collapsed: picking one of three channels is a decision the op must not make silently. + ## Introspection helpers Pure library functions (not ops) also live here, backing viewer tooling: `select_channel` (reduce an array/tensor to a 2-D float32 map for one channel; negative = mean across channels), `channel_count`, `array_histogram` (finite-only binning + summary stats, JSON-safe), `confusion_matrix_payload` / `confusion_matrices_payload` (render payloads for every confusion-matrix-shaped entry in a metrics result), and `draw_text` (text → `(H, W, 3)` uint8 image with word-wrap and 9-grid anchoring, plus the closed `TextPosition` Literal). diff --git a/recordstream/ops/__init__.py b/recordstream/ops/__init__.py index b005c07..d2a1980 100644 --- a/recordstream/ops/__init__.py +++ b/recordstream/ops/__init__.py @@ -5,7 +5,7 @@ - recordstream.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / connected_component_bboxes / resolve_expression helpers) - recordstream.ops.torch: ToTensor (+ to_tensor helper) - - recordstream.ops.image: ConvertToImage (+ value_to_image / normalize_to_uint8 …) + - recordstream.ops.image: ConvertToImage, ConvertToMask (+ value_to_image / normalize_to_uint8 …) - recordstream.ops.target: EncodeTarget, DecodeTarget, CocoToTorchVisionDetection, MasksToDetectionBoxes - recordstream.ops.structure: RenameField, DropField, CopyField, SelectFields @@ -28,7 +28,7 @@ from recordstream.ops.debug import PrintRecordOp from recordstream.ops.enable import Enable from recordstream.ops.formula import FormulaOp -from recordstream.ops.image import ConvertToImage +from recordstream.ops.image import ConvertToImage, ConvertToMask from recordstream.ops.numpy import ConnectedComponents, Threshold from recordstream.ops.parallel import Parallel from recordstream.ops.random_apply import RandomApply @@ -41,6 +41,7 @@ "ConfigureOp", "ConnectedComponents", "ConvertToImage", + "ConvertToMask", "CopyField", "DecodeTarget", "DropField", diff --git a/recordstream/ops/image.py b/recordstream/ops/image.py index db74971..b9407f6 100644 --- a/recordstream/ops/image.py +++ b/recordstream/ops/image.py @@ -27,7 +27,8 @@ from recordstream._compat import is_torch_tensor from recordstream.items import Image as ImageItem -from recordstream.items import NDArrayItem, Record, item_data +from recordstream.items import Mask as MaskItem +from recordstream.items import NDArrayItem, Record, item_data, item_value from recordstream.transform import Transform logger = get_logger("recordstream.ops.image") @@ -676,10 +677,98 @@ def __call__(self, record: Record) -> Record: return {**record, self.output: ImageItem(out_arr, layout="HWC")} +@configurable(category="op", group="image") +class ConvertToMask(Transform): + """A mask-bearing field → a :class:`~recordstream.Mask` item of per-pixel class ids. + + The segmentation counterpart of :class:`ConvertToImage`, and the same shape of op: it reads + ONE field and writes a differently-typed item under ``output``, leaving every other key + untouched. A segmentation dataset ships its target as a greyscale/paletted PNG whose pixel + values ARE the class ids (the Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC + segmentation map); this turns that payload into the ``int64`` ``[H, W]`` array every + per-pixel loss expects. + + **It converts and nothing else** — deliberately, because recordstream already owns the rest: + + * **remapping** the ids (1-based trimap → 0-based class ids, a Cityscapes id → trainId + table) is :class:`~recordstream.ops.formula.FormulaOp` over this op's ``output`` + (``formula: a - 1``) or :class:`~recordstream.ops.target.EncodeTarget`; + * **resizing / augmenting** it *together with the image* is a bare albumentations transform + dropped into the same ops list — the engine's op-family dispatch hands it the ``image`` + and ``mask`` keys in ONE call, so a single joint draw moves both and the ``Mask`` type + survives the round trip. That is why ``output`` defaults to ``"mask"``: it is + albumentations' own key vocabulary, so the very next op in the chain finds it. + + ``int64`` is not a knob: a class-id mask is integer by definition, and it is the dtype + ``torch.nn.CrossEntropyLoss`` requires (it rejects int32 with *"expected target dtype to be + Long or Byte, but got Int"*). A library that casts on the way past — albumentations returns + int32 — is corrected at the model boundary by ``batch_tensor(..., dtype=...)``, where the + caller names the contract. + + Args: + field: Name of the source field to read; blank (default) picks the first array/PIL-bearing item. + output: Name of the key the ``Mask`` item is written to (added if new); defaults to ``mask``. + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (MaskItem,) + + def __init__(self, field: str = "", output: str = "mask") -> None: + # Lazy / zero-arg: store config only. A missing/unusable field is reported at call time. + super().__init__() + self.field = field + self.output = output + + def _find_source(self, record: Record) -> Any: + """Resolve the payload to convert (``self.field``, else the first array/PIL-bearing item). + + The "array or PIL" rule is :class:`~recordstream.ops.torch.ToTensor`'s, not a third + spelling: a mask arrives either already decoded (an ndarray) or as the PIL image a + source handed over, and both are equally normal. + + Unwrapping goes through :func:`~recordstream.item_value` rather than + :func:`~recordstream.item_data`, and the difference is load-bearing here: a source that + does not know a column is a mask hands it over as a :class:`~recordstream.Label` (this + is what ``HuggingFaceSource`` does for every metadata column), whose payload slot is + ``value``, not ``data`` — ``item_data`` would return the ``Label`` itself and the PIL + image inside it would never be found. + """ + if self.field: + if self.field not in record: + raise ValueError(f"ConvertToMask: field {self.field!r} not in record (keys: {list(record)})") + return item_value(record[self.field]) + for _key, item in record.items(): + data = item_value(item) + if isinstance(data, np.ndarray) or is_torch_tensor(data) or hasattr(data, "convert"): + return data + raise ValueError(f"ConvertToMask: no array/PIL-bearing field in record (keys: {list(record)})") + + def __call__(self, record: Record) -> Record: + value = self._find_source(record) + if hasattr(value, "convert"): # a PIL image — an L / P mode plane is already the id map + value = np.asarray(value) + elif is_torch_tensor(value): + value = value.detach().cpu().numpy() + arr = np.asarray(value) + # Squeeze SINGLETON axes only, so a mask stored as [H, W, 1] or [1, H, W] lands as + # [H, W]. An RGB-encoded mask is deliberately NOT collapsed: picking one of three + # channels (or looking up a palette) is a decision this op must not make silently. + if arr.ndim > 2: + arr = np.squeeze(arr) + if arr.ndim != 2: + raise ValueError( + f"ConvertToMask: expected a 2-D [H, W] class-id map, got shape {tuple(np.shape(value))}. " + "An RGB-encoded mask needs a channel/palette decode first (e.g. select_channel)." + ) + return {**record, self.output: MaskItem(arr.astype(np.int64))} + + __all__ = [ "Colormap", "COLORMAPS", "ConvertToImage", + "ConvertToMask", "normalize_to_uint8", "value_to_image", "record_to_image", diff --git a/tests/test_categories.py b/tests/test_categories.py index 8b8efc6..dd11a2b 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -15,7 +15,7 @@ from recordstream.ops.debug import PrintRecordOp from recordstream.ops.enable import Enable from recordstream.ops.formula import FormulaOp -from recordstream.ops.image import ConvertToImage +from recordstream.ops.image import ConvertToImage, ConvertToMask from recordstream.ops.numpy import ConnectedComponents, Threshold from recordstream.ops.parallel import Parallel from recordstream.ops.random_apply import RandomApply @@ -54,6 +54,7 @@ def test_op_classes_tagged() -> None: ConnectedComponents, ToTensor, ConvertToImage, + ConvertToMask, Enable, Pipeline, Parallel, @@ -91,6 +92,7 @@ def test_op_group_tags() -> None: assert ConnectedComponents.__confluid_group__ == "numpy" assert ToTensor.__confluid_group__ == "torch" assert ConvertToImage.__confluid_group__ == "image" + assert ConvertToMask.__confluid_group__ == "image" assert SelectFields.__confluid_group__ == "structure" assert PrintRecordOp.__confluid_group__ == "debug" assert EncodeTarget.__confluid_group__ == "structure" @@ -119,6 +121,7 @@ def test_categories_enumerable_via_registry() -> None: "ConnectedComponents", "ToTensor", "ConvertToImage", + "ConvertToMask", "Enable", "Pipeline", "RecordSinkOp", @@ -135,7 +138,7 @@ def test_groups_enumerable_via_registry() -> None: registry = get_registry() assert {"Threshold", "ConnectedComponents"} <= registry.list_classes(group="numpy") assert {"ToTensor"} <= registry.list_classes(group="torch") - assert {"ConvertToImage"} <= registry.list_classes(group="image") + assert {"ConvertToImage", "ConvertToMask"} <= registry.list_classes(group="image") assert {"Parallel", "Enable", "Pipeline", "RandomApply", "ConfigureOp", "FormulaOp"} <= registry.list_classes( group="compose" ) diff --git a/tests/test_convert_to_mask.py b/tests/test_convert_to_mask.py new file mode 100644 index 0000000..1bf6562 --- /dev/null +++ b/tests/test_convert_to_mask.py @@ -0,0 +1,239 @@ +"""``ConvertToMask`` — a mask-bearing field to an int64 class-id ``Mask`` item. + +The segmentation counterpart of ``ConvertToImage``, and the front half of the target +pipeline a per-pixel task drives: the source hands over a greyscale/paletted PNG, this +turns it into the ``[H, W]`` int64 array a per-pixel loss consumes, and every remaining +step is an op that already existed (``FormulaOp`` remaps the ids, a bare albumentations +transform resizes it jointly with the image, ``collate_records`` stacks it). + +Also pins :func:`recordstream.item_value`, extracted here because the op needed the same +"get past a wrapper item" rule ``iter_key`` and ``batch_values`` already had. +""" + +from typing import Any, Dict, Tuple + +import numpy as np +import pytest +from PIL import Image as PILImage + +from recordstream import Image, Label, Mask, MultiLabel, collate_records, item_data, item_value +from recordstream.core import _apply_op +from recordstream.ops import ConvertToMask, DropField, FormulaOp + + +def _trimap(height: int = 30, width: int = 20) -> PILImage.Image: + """An Oxford-IIIT-Pet-style trimap: an L-mode PNG whose pixels are 1-based class ids.""" + return PILImage.fromarray(np.random.randint(1, 4, (height, width)).astype(np.uint8), mode="L") + + +# --------------------------------------------------------------------------- # +# item_value — the unwrapping rule the op shares with iter_key / batch_values +# --------------------------------------------------------------------------- # +class TestItemValue: + def test_a_label_yields_its_value_where_item_data_yields_the_label(self) -> None: + # The whole reason the function exists: a Label's payload slot is `value`, not + # `data`, so `item_data` cannot see into it. + label = Label("cat") + assert item_value(label) == "cat" + assert item_data(label) is label + + def test_a_multilabel_yields_its_values_list(self) -> None: + assert item_value(MultiLabel(["cat", "dog"])) == ["cat", "dog"] + + def test_an_array_item_yields_its_payload(self) -> None: + value = item_value(Mask(np.zeros((3, 3), dtype=np.int64))) + assert type(value) is np.ndarray + assert value.shape == (3, 3) + + def test_a_plain_value_passes_through(self) -> None: + assert item_value(30.72e6) == 30.72e6 + assert item_value(None) is None + + def test_iter_key_and_batch_values_agree_with_it(self) -> None: + """The two former copies now route through it — so they cannot drift from it.""" + from recordstream import batch_values, iter_key + + records = [{"class": Label(0)}, {"class": Label(1)}] + assert list(iter_key(records, "class")) == [0, 1] + assert list(batch_values(collate_records(records), "class")) == [0, 1] + + +# --------------------------------------------------------------------------- # +# ConvertToMask +# --------------------------------------------------------------------------- # +class TestConvertToMask: + def test_a_pil_mask_becomes_an_int64_mask_item(self) -> None: + out = ConvertToMask(field="segmentation_mask")({"segmentation_mask": _trimap()}) + mask = out["mask"] + assert isinstance(mask, Mask) + assert mask.shape == (30, 20) + assert mask.dtype == np.int64 + + def test_it_reads_a_mask_a_source_wrapped_in_a_label(self) -> None: + """The real shape: a source that does not know a column is a mask ships it as a Label. + + ``HuggingFaceSource`` does exactly this for every metadata column, so reading through + ``item_data`` (which hands the Label straight back) found nothing at all. + """ + record = {"image": Image(np.zeros((30, 20, 3), np.uint8)), "segmentation_mask": Label(_trimap())} + out = ConvertToMask(field="segmentation_mask")(record) + assert isinstance(out["mask"], Mask) + assert set(np.unique(np.asarray(out["mask"]))) <= {1, 2, 3} + + def test_a_blank_field_picks_the_first_array_or_pil_bearing_item(self) -> None: + out = ConvertToMask()({"note": "text", "seg": np.zeros((4, 5), dtype=np.uint8)}) + assert out["mask"].shape == (4, 5) + + def test_the_output_key_defaults_to_the_albumentations_vocabulary(self) -> None: + """``mask`` by default — so a bare albumentations transform finds it with no rename.""" + assert ConvertToMask().output == "mask" + assert "mask" in ConvertToMask()({"seg": np.zeros((4, 5))}) + + def test_the_output_key_is_configurable(self) -> None: + out = ConvertToMask(field="seg", output="target")({"seg": np.zeros((4, 5))}) + assert "target" in out and "mask" not in out + + def test_the_source_field_survives_so_a_later_op_can_read_it(self) -> None: + out = ConvertToMask(field="seg")({"seg": np.zeros((4, 5))}) + assert "seg" in out # dropping it is DropField's job, not this op's + + def test_other_keys_pass_through_untouched(self) -> None: + image = Image(np.zeros((4, 5, 3), np.uint8)) + out = ConvertToMask(field="seg")({"seg": np.zeros((4, 5)), "image": image, "id": 7}) + assert out["image"] is image + assert out["id"] == 7 + + @pytest.mark.parametrize("shape", [(4, 5, 1), (1, 4, 5)]) + def test_singleton_axes_are_squeezed(self, shape: Tuple[int, ...]) -> None: + assert ConvertToMask()({"seg": np.zeros(shape, dtype=np.uint8)})["mask"].shape == (4, 5) + + def test_an_rgb_mask_is_refused_rather_than_silently_collapsed(self) -> None: + """Picking one of three channels is a decision the op must not make for the user.""" + with pytest.raises(ValueError, match="2-D"): + ConvertToMask()({"seg": np.zeros((4, 5, 3), dtype=np.uint8)}) + + def test_a_torch_tensor_payload_is_accepted(self) -> None: + torch = pytest.importorskip("torch") + out = ConvertToMask()({"seg": torch.randint(0, 3, (4, 5))}) + assert isinstance(out["mask"], Mask) + assert out["mask"].dtype == np.int64 + + # ---- lazy / zero-arg construction (workspace mandate) ------------------- # + def test_zero_arg_construction_works(self) -> None: + assert ConvertToMask().field == "" + + def test_a_missing_named_field_is_reported_at_call_time(self) -> None: + with pytest.raises(ValueError, match="not in record"): + ConvertToMask(field="nope")({"seg": np.zeros((4, 5))}) + + def test_a_record_with_no_candidate_is_reported_at_call_time(self) -> None: + with pytest.raises(ValueError, match="no array/PIL-bearing field"): + ConvertToMask()({"note": "text"}) + + +# --------------------------------------------------------------------------- # +# The chain it belongs to — this is what a segmentation config actually writes +# --------------------------------------------------------------------------- # +class TestTheSegmentationTargetPipeline: + def test_convert_remap_drop_resize_collate(self) -> None: + """End to end, with every step after the conversion an op that already existed. + + This is the ``preprocess`` chain of a segmentation config, run in Python: the point + being that ``ConvertToMask`` is the ONLY piece segmentation needed added. + """ + albumentations = pytest.importorskip("albumentations") + torch = pytest.importorskip("torch") + from recordstream import batch_tensor + + record: Dict[str, Any] = { + "image": Image(np.random.randint(0, 255, (30, 20, 3), dtype=np.uint8)), + "segmentation_mask": Label(_trimap()), + } + chain = [ + ConvertToMask(field="segmentation_mask"), + FormulaOp(field="mask", formula="a - 1"), # 1-based trimap -> 0-based class ids + DropField(key="segmentation_mask"), # a PIL left in the record would break collate + albumentations.Resize(height=16, width=16), # ONE joint draw over image AND mask + ] + for op in chain: + applied = _apply_op(record, op) + assert applied is not None # no op in this chain drops a record + record = applied + + assert set(record) == {"image", "mask"} + assert record["image"].shape == (16, 16, 3) + assert record["mask"].shape == (16, 16) + assert set(np.unique(np.asarray(record["mask"]))) <= {0, 1, 2} + + batch = collate_records([record, record]) + assert batch["image"].shape == (2, 16, 16, 3) + assert batch["mask"].shape == (2, 16, 16) + # The dtype the per-pixel CrossEntropy contract needs — named by the CALLER, because + # albumentations casts the mask to int32 on the way past. + assert batch_tensor(batch, "mask", dtype=torch.int64).dtype == torch.int64 + + def test_a_normalize_leaves_the_mask_alone(self) -> None: + """Albumentations applies an image-only transform to the image only — pinned, because + a Normalize that reached the mask would turn class ids into floats silently.""" + albumentations = pytest.importorskip("albumentations") + record = { + "image": Image(np.random.randint(0, 255, (8, 8, 3), dtype=np.uint8)), + "mask": Mask(np.random.randint(0, 3, (8, 8)).astype(np.int64)), + } + out = _apply_op(record, albumentations.Normalize(mean=[0.5] * 3, std=[0.5] * 3)) + assert out is not None + assert out["image"].dtype == np.float32 + assert np.array_equal(np.asarray(out["mask"]), np.asarray(record["mask"])) + + +# --------------------------------------------------------------------------- # +# num_mask_classes — the per-pixel twin of num_classes +# --------------------------------------------------------------------------- # +class TestNumMaskClasses: + def test_it_is_the_largest_id_anywhere_plus_one(self) -> None: + from recordstream import num_mask_classes + + records = [ + {"mask": Mask(np.array([[0, 1], [1, 0]], dtype=np.int64))}, + {"mask": Mask(np.array([[0, 2], [2, 2]], dtype=np.int64))}, # 2 appears only here + ] + assert num_mask_classes(records) == 3 + + def test_a_class_in_the_last_record_still_sizes_the_head(self) -> None: + """The walk covers EVERY record — a rare class must not be missed by an early exit.""" + from recordstream import num_mask_classes + + records = [{"mask": Mask(np.zeros((2, 2), dtype=np.int64))} for _ in range(50)] + records[-1] = {"mask": Mask(np.array([[0, 7], [0, 0]], dtype=np.int64))} + assert num_mask_classes(records) == 8 + + def test_the_key_is_configurable(self) -> None: + from recordstream import num_mask_classes + + assert num_mask_classes([{"target": Mask(np.array([[0, 4]], dtype=np.int64))}], key="target") == 5 + + def test_an_empty_source_raises_rather_than_guessing(self) -> None: + from recordstream import num_mask_classes + + with pytest.raises(ValueError, match="no usable"): + num_mask_classes([]) + + def test_a_record_without_a_mask_raises(self) -> None: + """Silently skipping it would under-count the head against a partially-labelled set.""" + from recordstream import num_mask_classes + + with pytest.raises(ValueError, match="no 'mask' value"): + num_mask_classes([{"mask": Mask(np.zeros((2, 2), dtype=np.int64))}, {"mask": None}]) + with pytest.raises(ValueError, match="no 'mask' value"): + num_mask_classes([{"image": Image(np.zeros((2, 2, 3), np.uint8))}]) + + def test_num_classes_still_refuses_an_array_target(self) -> None: + """The reason this is a separate function: the scalar guard must stay strict. + + A classification run accidentally handed masks has to fail loudly rather than report + whatever the first array's `.item()` would have been. + """ + from recordstream import num_classes + + with pytest.raises((TypeError, ValueError)): + num_classes([{"class": Mask(np.array([[0, 1], [1, 0]], dtype=np.int64))}]) From cac61d9170249602a02533140ae8161fbc79d138 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sun, 2 Aug 2026 14:15:27 +0200 Subject: [PATCH 072/102] =?UTF-8?q?feat(projection):=20first=5Fvalue=20?= =?UTF-8?q?=E2=80=94=20one=20peek=20at=20a=20column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cheapest question about a column: the first non-`None` value under a key, or `None` when there is none. It is `iter_key` plus a `next()`, which is why it belongs here — a consumer re-deriving it re-derives that helper's contract (the projection skip, the deferred-source flow, and the MultiLabel-unwraps-to-a-LIST rule that makes "a sequence IS multi-label" true). Extracted from eight byte-identical private copies in one consumer's training backends, where the label peek decides both questions asked before anything can be encoded: names or ids, single- or multi-label. `iter_key` now unwraps through `item_value` rather than spelling the ladder a second time, so the two cannot disagree about what a Label yields. --- AGENTS.md | 5 +- README.md | 4 +- docs/projection.md | 18 +++++- docs/record-model.md | 11 +++- recordstream/__init__.py | 20 ++++++- recordstream/batch.py | 20 +++---- recordstream/items.py | 32 +++++++++++ recordstream/projection.py | 77 +++++++++++++++++++++----- tests/test_projection.py | 110 +++++++++++++++++++++++++++++++++++++ 9 files changed, 263 insertions(+), 34 deletions(-) create mode 100644 tests/test_projection.py diff --git a/AGENTS.md b/AGENTS.md index 7c3bd38..bfe7df9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper); `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. - **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. @@ -38,7 +38,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. - **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. -- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. **`first_value(source, key)` is the ONE-PEEK primitive beside them (2026-08-02)** — the first non-`None` value under `key`, or `None` when there is none. It answers what a column's values ARE without walking the set, and it belongs here rather than in any consumer because it is `iter_key` plus a `next()`: it inherits all three of that helper's properties (a projection-aware source never builds the values it does not ask for, a deferred source is materialized first, the walk is lazy so a normal source costs ONE record) and its unwrapping rules are what make the answer meaningful — a `MultiLabel` arrives as its `.values` LIST, so a sequence IS a multi-label column, decided by the item type rather than by guessing what a list might mean. The canonical call site pairs it with `is_class_id` to decide whether the targets need a `LabelMap` at all. It was extracted from EIGHT byte-identical private copies in one consumer's training backends (2026-08-02); a consumer re-deriving it is re-deriving `iter_key`'s contract. Pins: `tests/test_projection.py`. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). - **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. @@ -50,6 +50,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. +- **Generic MASK Conversion Lives Here Too — `ConvertToMask` (2026-08-02):** the segmentation counterpart of `ConvertToImage` and the same op SHAPE (read one field, write a differently-typed item under `output`): a mask-bearing field (an ndarray, a torch tensor, or the PIL image a source handed over) becomes an **`int64` `[H, W]` `Mask`** of per-pixel class ids — what a segmentation dataset actually ships (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map) turned into what a per-pixel loss consumes. It belongs HERE, not in a segmentation project: "a mask PNG's pixels are class ids" mentions no modality (the `Threshold` → `Mask` precedent), and a consumer owning it would be the third package to write the conversion. **It converts and NOTHING else, deliberately** — remapping the ids is `FormulaOp` over its output (`formula: a - 1` for a 1-based trimap) or `EncodeTarget` for a lookup table; resizing/augmenting it TOGETHER WITH THE IMAGE is a bare albumentations transform in the same ops list; dropping the source column is `DropField`. Do NOT grow it an `offset` / `mapping` / `dtype` knob: each one restates an op that already exists. **`output` defaults to `"mask"` and that is load-bearing, not a nicety** — it is albumentations' own key vocabulary (`_ALB_KEYS`), so the engine's op-family dispatch hands `image` AND `mask` to ONE call and a single joint draw moves both with the `Mask` type surviving the re-wrap (measured; an image-only transform like `Normalize` still touches the image alone). **`int64` is not a knob either:** a class-id map is integer by definition and it is what `torch.nn.CrossEntropyLoss` requires (*"expected target dtype to be Long or Byte, but got Int"*); a library that casts on the way past — albumentations returns int32 — is corrected at the MODEL boundary by `batch_tensor(..., dtype=...)`, where the caller names the contract (the `dtype`-is-a-parameter rule). It reads through **`item_value`, never `item_data`**, because a source that does not know a column is a mask ships it as a `Label` (`HuggingFaceSource` does this for every metadata column) — see the record-model mandate. Singleton axes are squeezed (`[H,W,1]` / `[1,H,W]` → `[H,W]`); an **RGB-encoded mask RAISES** rather than being collapsed, because picking one of three channels or decoding a palette is a decision the op must not make silently. Pins: `tests/test_convert_to_mask.py` (incl. the whole `preprocess` chain end to end, and that a `Normalize` leaves the mask untouched). Usage: `docs/image.md` → "Masks". - **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). - **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) diff --git a/README.md b/README.md index 58bfe0c..59c974f 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,9 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | | [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | -| [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), `num_classes`, the fittable `LabelMap`, class-balance weights | +| [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), one-peek `first_value`, `num_classes`, the fittable `LabelMap`, class-balance weights | | [docs/predictions.md](docs/predictions.md) | The model boundary: prediction-output contracts (`ClassificationOutput` & co), `ensure_record_dataset`, the `PredictionsSink` protocol + the classification sink | -| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), array introspection helpers | +| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), mask→class-id conversion (`ConvertToMask`), array introspection helpers | | [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | | [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers + `run_entrypoint` dispatch with a worked example, `TorchRunner` / `ProgressReporting` | | [docs/workflow.md](docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | diff --git a/docs/projection.md b/docs/projection.md index 8c78c6f..44cfa1c 100644 --- a/docs/projection.md +++ b/docs/projection.md @@ -5,7 +5,7 @@ Walking a source for a single record key (the classic case: counting classes from the label key) shouldn't pay to build the values you don't need. `recordstream.projection` adds an opt-in protocol plus lazy helpers, all **key-addressed** — any subset of record keys: ```python -from recordstream import project, iter_key, num_classes +from recordstream import first_value, project, iter_key, num_classes # A source MAY implement SupportsProjection (`project(keys)`) to skip building # unrequested values — e.g. an image dataset reads only the label column for a @@ -16,11 +16,27 @@ for record in project(my_source, ("class",)): labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, a MultiLabel to # its .values LIST, other items to their payload, # plain values verbatim +first = first_value(my_source, "class") # ONE peek — the cheapest question about a column n = num_classes(my_source, key="class") # max(class_id) + 1 — always walks ``` A **deferred** source — a `!class:` marker straight out of a config — is materialized first, so a caller never has to know which entry point flows and which doesn't (flowing a live object is a no-op). Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Stream.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Stream` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. +`first_value` answers what the values in a column *are* without walking the set — one peek is enough to learn a kind. On a label column it decides both questions a consumer has before it can encode anything: + +```python +from recordstream import first_value, is_class_id + +target = first_value(train_source, "class") +multilabel = isinstance(target, (list, tuple, set)) # a MultiLabel arrives as its .values LIST, + # so a sequence IS multi-label — the item + # type, never a guess about what a list means +probe = next(iter(target), None) if multilabel else target +needs_a_label_map = probe is not None and not is_class_id(probe) +``` + +It stops at the first hit, so it is one record on a normal source; a missing or all-`None` column costs one full pass and answers `None` rather than raising. + ## `LabelMap` — fittable name↔id encoding When a dataset's label is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTarget` / `DecodeTarget` ops need — the *fittable* companion to those ops. Fit it once (deterministic `sorted(set(...))` ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: diff --git a/docs/record-model.md b/docs/record-model.md index 1a7e4b8..cbbf827 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -2,7 +2,7 @@ A record is a **plain `dict`** of **typed values**. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, -as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). +as_transform, item_data, item_value, with_data, register_item, register_kernel, register_io, collate_records, ...`). The design rationale is recorded in [architecture.md](architecture.md#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25). @@ -64,11 +64,18 @@ are dataclass wrappers (a bounding-box set is not an array). A uniform payload a kernels: ```python -from recordstream import item_data, with_data +from recordstream import item_data, item_value, with_data item_data(Image(arr)) # -> the plain ndarray with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved ``` +`item_value` is the same question one step further out, and the difference is the label items: a +`Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` hands the `Label` back +while `item_value(Label("cat"))` gives you `"cat"` (and a `MultiLabel` its `.values` list). Use +`item_data` in a kernel, where the item type is already known; use `item_value` when you want *the +value* whatever wrapper carried it — which is what `iter_key`, `batch_values` and `ConvertToMask` all +want, and why the rule is one function rather than three copies of it. + `register_item` / `is_item` / `item_types` / `get_item_type` / `item_type_names` are the open item registry — the extensibility surface a domain package or user type plugs into (one class + one decorator, no core edit). diff --git a/recordstream/__init__.py b/recordstream/__init__.py index aaaaeaf..aa9003d 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -49,6 +49,7 @@ item_data, item_type_names, item_types, + item_value, register_item, with_data, ) @@ -63,7 +64,15 @@ ) from recordstream.predictions import ClassificationPredictionsSink, PredictionsSink from recordstream.processing import DatasetProcessor -from recordstream.projection import SupportsProjection, class_names, iter_key, num_classes, project +from recordstream.projection import ( + SupportsProjection, + class_names, + first_value, + iter_key, + num_classes, + num_mask_classes, + project, +) from recordstream.runnable import ( ProgressCallback, ProgressReporting, @@ -75,6 +84,7 @@ ) from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName from recordstream.transform import FunctionTransform, Pipeline, Transform, as_transform +from recordstream.uri import SupportsDatasetIdentity, dataset_uri, dataset_uris, dataset_url from recordstream.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch __all__ = [ @@ -93,6 +103,7 @@ "get_item_type", "is_item", "item_data", + "item_value", "with_data", "Transform", "Pipeline", @@ -145,11 +156,18 @@ "RangeSource", "ConcatSource", "SplitName", + # ---- dataset identity ---- + "SupportsDatasetIdentity", + "dataset_uri", + "dataset_uris", + "dataset_url", # ---- projection ---- "SupportsProjection", + "first_value", "iter_key", "class_names", "num_classes", + "num_mask_classes", "project", # ---- runnable protocol + orchestration ---- "TorchRunner", diff --git a/recordstream/batch.py b/recordstream/batch.py index 13ece8d..a84db2c 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -33,7 +33,7 @@ import numpy as np -from recordstream.items import Label, MultiLabel, Record, item_data +from recordstream.items import Record, item_value if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only from torch import Tensor @@ -44,11 +44,12 @@ def batch_values(batch: Record, key: str) -> Any: """The raw batched values under ``key``, unwrapped from their item type. - The one place that knows how to get *past* a wrapper item: a :class:`~recordstream.Label` - yields its ``.value`` (the collate leaves it a per-record LIST — a wrapper item's payload - is not stacked), a :class:`~recordstream.MultiLabel` its ``.values`` (a list OF lists), - and anything else goes through :func:`~recordstream.item_data` (an array item yields its - stacked payload, a plain value the per-record list the collate gathered). + Getting *past* a wrapper item is :func:`~recordstream.item_value`'s rule: a + :class:`~recordstream.Label` yields its ``.value`` (the collate leaves it a per-record LIST + — a wrapper item's payload is not stacked), a :class:`~recordstream.MultiLabel` its + ``.values`` (a list OF lists), and anything else its payload (an array item yields its + stacked payload, a plain value the per-record list the collate gathered). What is THIS + function's own is the BATCH reading — that the values arrive already collated. No torch, no stacking, no dtype opinion — just the values. Use :func:`batch_tensor` when a tensor is what you need. @@ -58,12 +59,7 @@ def batch_values(batch: Record, key: str) -> Any: batch_values(collate_records([{"class": Label(0)}, {"class": Label(1)}]), "class") # [0, 1] """ - item = batch[key] - if isinstance(item, MultiLabel): - return item.values - if isinstance(item, Label): - return item.value - return item_data(item) + return item_value(batch[key]) def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") -> np.ndarray: diff --git a/recordstream/items.py b/recordstream/items.py index 760443c..87461ff 100644 --- a/recordstream/items.py +++ b/recordstream/items.py @@ -65,6 +65,7 @@ "get_item_type", "is_item", "item_data", + "item_value", "with_data", ] @@ -284,6 +285,37 @@ def item_data(item: Any) -> Any: return item +def item_value(item: Any) -> Any: + """The semantic VALUE of a record entry — one step further past a wrapper than :func:`item_data`. + + The difference is the label items, and it is the whole reason this exists beside + :func:`item_data`: a :class:`Label`'s payload slot is ``value``, not ``data``, so + ``item_data`` hands the ``Label`` itself back. A consumer that wants *the class id* — or + *the mask array*, without caring which wrapper carried it — wants this instead. + + The rule: a :class:`MultiLabel` yields its ``.values`` list, a :class:`Label` its + ``.value``, any other registered item its payload via :func:`item_data`, and a plain value + passes through verbatim. + + It is ONE function because the rule was written out three times — + :func:`~recordstream.projection.iter_key` (per record), + :func:`~recordstream.batch.batch_values` (per batch) and, the copy that prompted the + extraction, an op reading a mask that a source had wrapped in a ``Label``. Those state + WHERE they read; the unwrapping itself never differed. + + Example:: + + item_value(Label("cat")) # 'cat' (item_data returns the Label) + item_value(Mask(np.zeros((4, 4)))) # the ndarray + item_value(30.72e6) # 30720000.0 + """ + if isinstance(item, MultiLabel): + return item.values + if isinstance(item, Label): + return item.value + return item_data(item) + + def with_data(item: _ItemT, new_data: Any) -> _ItemT: """A copy of ``item`` carrying ``new_data`` as its payload, metadata preserved (same type). diff --git a/recordstream/projection.py b/recordstream/projection.py index a49d0a3..c9fb479 100644 --- a/recordstream/projection.py +++ b/recordstream/projection.py @@ -23,7 +23,7 @@ from typing import Any, Collection, Iterator, List, Optional, Protocol, runtime_checkable -from recordstream.items import Label, MultiLabel, Record, is_item, item_data +from recordstream.items import Record, item_value @runtime_checkable @@ -64,21 +64,36 @@ def project(source: Any, keys: Collection[str]) -> Iterator[Record]: def iter_key(source: Any, key: str) -> Iterator[Any]: """Lazily yield each record's ``key`` VALUE (skipping other-key construction when supported). - A :class:`~recordstream.items.Label` unwraps to its ``.value`` (the class id / name), a - :class:`~recordstream.items.MultiLabel` to its ``.values`` list; any - other registered item unwraps to its payload via :func:`~recordstream.items.item_data`; a - plain value passes through verbatim. A record without ``key`` yields ``None``. + Unwrapping is :func:`~recordstream.items.item_value`'s rule, not a second spelling of it: a + :class:`~recordstream.items.Label` yields its ``.value`` (the class id / name), a + :class:`~recordstream.items.MultiLabel` its ``.values`` list, any other registered item its + payload, and a plain value passes through verbatim. What is THIS function's own is only the + projection — a record without ``key`` yields ``None``. """ for record in project(source, (key,)): - value = record.get(key) - if isinstance(value, MultiLabel): - yield value.values - elif isinstance(value, Label): - yield value.value - elif is_item(value): - yield item_data(value) - else: - yield value + yield item_value(record.get(key)) + + +def first_value(source: Any, key: str) -> Any: + """The first non-``None`` value under ``key`` in ``source`` — ``None`` when there is none. + + The cheapest possible question about a column: ONE peek, which is all a consumer needs to + learn the KIND of the values without walking the set. The canonical use is a label column, + where the peek decides whether the targets are class NAMES needing a + :class:`~recordstream.labels.LabelMap` or ids that pass straight through (ask + :func:`~recordstream.items.is_class_id` of the answer), and whether the column is + multi-label (a :class:`~recordstream.items.MultiLabel` arrives here as its ``values`` + LIST, so a sequence IS multi-label — the item type, not a guess about what a list means). + + Built on :func:`iter_key`, so all three of its properties carry over: a + projection-aware source never builds the values this does not ask for, a deferred source + is materialized first, and the walk stops at the first hit — a missing or all-``None`` + column costs one full pass and answers ``None`` rather than raising. + """ + for value in iter_key(source, key): + if value is not None: + return value + return None def _to_int(value: Any) -> int: @@ -171,9 +186,43 @@ def num_classes(source: Any, key: str = "class") -> int: return highest + 1 +def num_mask_classes(source: Any, key: str = "mask") -> int: + """The per-pixel twin of :func:`num_classes` — walk a MASK column and return ``max + 1``. + + A per-pixel target is an ``[H, W]`` array of class ids rather than one id, so the count is + the largest id anywhere in the column plus one. Same contract as :func:`num_classes` + otherwise: the walk is key-restricted (a projection-aware source never decodes the image + beside the mask), it covers EVERY record so a class appearing only in the last one still + sizes the head, and an empty column raises rather than returning a plausible number. + + It is a separate function rather than a widening of :func:`num_classes`, and the reason is + that function's strictness: ``_to_int`` REJECTS a non-scalar target on purpose, so a + classification run that is accidentally handed arrays fails loudly instead of miscounting. + Teaching it to take the max of whatever it gets would trade that guard away for both tasks. + + Args: + source: The source to walk (any iterable of records; projection-aware when supported). + key: The record key holding the per-pixel mask. Defaults to ``"mask"``. + """ + import numpy as np + + highest = -1 + for mask in iter_key(source, key): + if mask is None: + raise ValueError(f"num_mask_classes: a record has no {key!r} value — cannot derive a class count.") + arr = np.asarray(mask) + if arr.size == 0: + continue # an empty mask constrains nothing; a column of them raises below + highest = max(highest, int(arr.max())) + if highest < 0: + raise ValueError(f"num_mask_classes: source yielded no usable {key!r} masks — cannot derive a class count.") + return highest + 1 + + __all__ = [ "SupportsProjection", "project", "iter_key", "num_classes", + "num_mask_classes", ] diff --git a/tests/test_projection.py b/tests/test_projection.py new file mode 100644 index 0000000..e7337c4 --- /dev/null +++ b/tests/test_projection.py @@ -0,0 +1,110 @@ +"""Tests for :mod:`recordstream.projection` — the key-addressed walk helpers. + +Focused on :func:`recordstream.first_value`, the one-peek primitive: what it unwraps, +what it skips, what it costs, and that it inherits ``iter_key``'s three properties +(projection-aware sources, deferred sources, laziness). +""" + +from typing import Any, Collection, Dict, Iterator, List + +from confluid import LazyClass + +from recordstream import Label, MultiLabel, first_value, is_class_id +from recordstream.items import Record + + +class _Source: + """A plain iterable source — no projection protocol, no laziness tricks.""" + + def __init__(self, records: List[Record]) -> None: + self.records = records + + def __iter__(self) -> Iterator[Record]: + return iter(self.records) + + +class _CountingProjectionSource: + """A projection-aware source that records what it was asked for, and counts reads.""" + + def __init__(self, records: List[Record]) -> None: + self.records = records + self.requested: List[Collection[str]] = [] + self.reads = 0 + + def project(self, keys: Collection[str]) -> Iterator[Record]: + self.requested.append(set(keys)) + for record in self.records: + self.reads += 1 + yield {k: v for k, v in record.items() if k in keys} + + def __iter__(self) -> Iterator[Record]: # pragma: no cover - project() is what runs + return iter(self.records) + + +def test_first_value_returns_the_first_present_value() -> None: + source = _Source([{"class": "cat"}, {"class": "dog"}]) + assert first_value(source, "class") == "cat" + + +def test_first_value_skips_leading_nones_and_missing_keys() -> None: + """``None`` means "no value here", which is not an answer about the column's kind.""" + source = _Source([{"class": None}, {"other": 1}, {"class": "dog"}]) + assert first_value(source, "dog_key_absent_everywhere") is None + assert first_value(source, "class") == "dog" + + +def test_an_all_none_column_answers_none_rather_than_raising() -> None: + source = _Source([{"class": None}, {"class": None}]) + assert first_value(source, "class") is None + + +def test_an_empty_source_answers_none() -> None: + assert first_value(_Source([]), "class") is None + + +def test_a_label_item_unwraps_to_its_value() -> None: + source = _Source([{"class": Label(value="cat", classes=["cat", "dog"])}]) + assert first_value(source, "class") == "cat" + + +def test_a_multilabel_item_unwraps_to_its_values_LIST() -> None: + """The peek is how a consumer learns the column is multi-label — by the ITEM type. + + ``iter_key`` unwraps a ``MultiLabel`` to its list, so a sequence here IS multi-label + rather than a guess about what a list might mean. + """ + source = _Source([{"class": MultiLabel(values=["cat", "dog"])}]) + peeked = first_value(source, "class") + assert peeked == ["cat", "dog"] + assert isinstance(peeked, (list, tuple, set)) + + +def test_the_peek_answers_the_names_versus_ids_question() -> None: + """The canonical call site: one peek + ``is_class_id`` decides whether a LabelMap is needed.""" + assert not is_class_id(first_value(_Source([{"class": "cat"}]), "class")) + assert is_class_id(first_value(_Source([{"class": 3}]), "class")) + + +def test_first_value_stops_at_the_first_hit() -> None: + """One peek, not a walk — the whole reason to call this instead of ``list(iter_key(...))``.""" + source = _CountingProjectionSource([{"class": i} for i in range(100)]) + assert first_value(source, "class") == 0 + assert source.reads == 1 + + +def test_first_value_asks_only_for_the_requested_key() -> None: + """A projection-aware source never builds the values this does not ask for.""" + source = _CountingProjectionSource([{"class": 0, "image": "expensive"}]) + assert first_value(source, "class") == 0 + assert source.requested == [{"class"}] + + +def test_first_value_materializes_a_deferred_source() -> None: + """``project()`` flows a ``!class:`` marker, so a caller writes no ``flow()`` here.""" + marker = LazyClass(_Source, records=[{"class": "cat"}]) + assert first_value(marker, "class") == "cat" + + +def test_plain_values_pass_through_verbatim() -> None: + payload: Dict[str, Any] = {"samplerate": 30.72e6} + assert first_value(_Source([payload]), "samplerate") == 30.72e6 From 91ab642ca35978231d97822fe813fd3555fc2b1b Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 3 Aug 2026 10:09:01 +0200 Subject: [PATCH 073/102] fix(sources): close the two independent fork hazards on the DataLoader path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DataLoader worker may be a FORKED child — torch's default on Linux, and on macOS whenever anything in the process has set `fork` (fastai does, at import). A forked child inherits memory but NOT threads, so anything holding a thread pool or a system handle across the fork dies there — with a SIGSEGV and no Python traceback, surfacing only as `DataLoader worker exited unexpectedly`. Two hazards sat on this path and neither fixed the other: * `ensure_materialized` — a LAZY source first built inside the child runs its download there, and the download stack is not fork-safe. Materializing in the parent means the child inherits a built source and downloads nothing. * `_disable_cv2_threading` — OpenCV's internal thread pool, inherited dead by a forked child the moment the parent has executed any cv2 op. Spawning instead is not the fix: it has its own cost (a model on Apple's MPS cannot be shared across a spawn), so a consumer that forks needs both guards. Pinned in tests/test_fork_safety.py, which records the diagnosis so the next SIGSEGV-with-no-traceback is not re-derived from scratch. --- AGENTS.md | 3 + README.md | 4 +- docs/architecture.md | 81 ++++++++++++++ docs/augmentation.md | 30 ++++++ docs/sources.md | 78 ++++++++++++++ recordstream/__init__.py | 2 + recordstream/core/__init__.py | 2 + recordstream/core/families.py | 43 ++++++++ recordstream/core/stream.py | 39 +++++++ recordstream/sources/huggingface.py | 45 ++++++-- recordstream/uri.py | 23 ++-- tests/test_dataset_uri.py | 36 +++++++ tests/test_fork_safety.py | 158 ++++++++++++++++++++++++++++ tests/test_record_source.py | 77 +++++++++++++- 14 files changed, 603 insertions(+), 18 deletions(-) create mode 100644 tests/test_fork_safety.py diff --git a/AGENTS.md b/AGENTS.md index bfe7df9..1dd5176 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. - **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. - **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **OpenCV's Thread Pool Is The SECOND Fork Hazard, And The Guard Fires On USE (`core.families._disable_cv2_threading`, 2026-08-02):** the companion to the `ensure_materialized` mandate below, and **independent of it — neither fixes the other**. albumentations runs on OpenCV, whose thread pool is not fork-safe: once the PARENT has executed a cv2 op, a forked `DataLoader` worker inheriting that pool dies with a **SIGSEGV and no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` — the same symptom as the lazy-source hazard, which is exactly why they get confused for each other. Measured 2026-08-02 with `DataLoader(num_workers=2)` over a `Stream` whose ops list held an `A.Resize`: **with the source warmed and the pool ON the worker still SIGSEGVs, and with the pool off a cold source is still built in the child.** A consumer that forks needs BOTH guards. `cv2.setNumThreads(0)` is fired ONCE at the top of `_invoke_albumentations` — the one place this package INVOKES albumentations — and never at import: a process that never uses albumentations must not have its OpenCV settings changed by importing a data library, and cv2 must not become an import-time dependency of the engine. It costs nothing where it matters, because inside a worker the WORKER is the parallelism and cv2's own threads oversubscribe rather than help (albumentations' own docs recommend exactly this for multiprocessing loaders). Two cv2 quirks a test must not get wrong, both measured: `setNumThreads(0)` makes `getNumThreads()` report **1**, not 0; and `setNumThreads(4)` does not change what it reports at all. **Why the classification consumer never hit this and the segmentation one did:** classification resizes with `ConvertToImage` (PIL), while a per-pixel task must resize the image and its mask in ONE JOINT DRAW — which only a bare albumentations transform does. Segmentation is the first thing in the workspace to put cv2 on the worker path. Pins: `tests/test_fork_safety.py` (the forked-worker subprocess reproduction is the one that matters — removing the guard makes it FAIL, not pass differently). - **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. - **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.execute.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). @@ -45,6 +46,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. +- **A Source NAMES The Data It Reads — `dataset_uri` + `dataset_url`, And A View Propagates Them VERBATIM (`recordstream.uri`, 2026-08-02):** `SupportsDatasetIdentity` is a `@runtime_checkable` Protocol with TWO properties, and the pair is deliberate rather than one field doing double duty: **`dataset_uri`** is the CANONICAL handle (machine-parseable, stable across machines, the string two runs are COMPARED on — `hf://datasets/ylecun/mnist?split=train`, matching the convention hosted tracking services already use for a dataset source) while **`dataset_url`** is a link a PERSON opens and is `None` whenever the data has no web page. Collapsing them forces a choice between a browsable string that lies about local data and a canonical one nobody can click. `HuggingFaceSource` implements both: a Hub repo id -> `hf://datasets/?…`, a local directory -> its `file://` URI, decided by `Path(path).exists()` (the same question `load_dataset` answers); the query params are SORTED so one configuration has exactly ONE string, and both read STORED CONFIG ONLY — asking never loads, so a source that is never iterated still names itself (`tests/test_dataset_uri.py::test_asking_for_identity_never_loads_the_dataset`). **`revision` is a DECLARED ctor param BECAUSE identity reads it** (it was in the removed `**kwargs`), and `load_options` merges it over `load_kwargs` as a read-only property rather than in `__init__` — a declared key may be set post-construction, and a dict assembled in the constructor would keep the value the object was born with. **Following happens in the FREE FUNCTIONS, not in every wrapper:** `dataset_uri(x)` / `dataset_url(x)` flow a deferred `!class:` marker (as `project` does) and then follow a `.source` attribute when the object holds no handle — depth-capped (`MAX_WRAPPER_DEPTH`) and cycle-safe — so `Stream` / `DatasetSplit` / `_SplitView` / `RangeSource` / `MetadataFilterSource` all work with ZERO code of their own, as does any third-party wrapper using that attribute name. **A wrapper must NOT decorate the URI it passes up** (no `#train`, no `#0:1000`): the handle identifies the DATASET, how much of it a run consumed is already recorded by the wrapper's own configuration, and decorating would mean one dataset reached two ways stops comparing equal — the single property the handle exists to have, and what makes a consumer's dedup exact. **`ConcatSource` answers `None` ON PURPOSE** — several datasets end to end are not one dataset, and picking a member would be a lie; `dataset_uris(source)` is the plural form that fans out over `.sources` (recursive, deduplicated). `None` is an ordinary answer everywhere (unconfigured source, in-memory stream, data with no page), never an error. Adding identity to a new source is TWO properties and no registration. Package-root exports + `__all__`; no entry point (the module holds no `@configurable`, same as `projection.py`). Rationale: `docs/architecture.md` §13. Usage: `docs/sources.md` → "Identifying a dataset". Pins: `tests/test_dataset_uri.py`. +- **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset`; pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise). - **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). diff --git a/README.md b/README.md index 59c974f..19e34af 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ generated tool schema set the toggle too (see [docs/architecture.md](docs/archit | [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | | [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), the Keras `RecordSequence` adapter, 1→N expanding ops | | [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | -| [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing | +| [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing, dataset identity (`dataset_uri` / `dataset_url`) | | [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | | [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), one-peek `first_value`, `num_classes`, the fittable `LabelMap`, class-balance weights | | [docs/predictions.md](docs/predictions.md) | The model boundary: prediction-output contracts (`ClassificationOutput` & co), `ensure_record_dataset`, the `PredictionsSink` protocol + the classification sink | @@ -121,7 +121,7 @@ RecordStream deliberately contains **no domain-specific code** — every op, sou RecordStream is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: -- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability (see [docs/sources.md](docs/sources.md)). +- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability, and [names the dataset it reads](docs/sources.md#identifying-a-dataset) so a run record can point at it (see [docs/sources.md](docs/sources.md)). - **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node — including bare library transforms — every run reproducible. - **PyTorch**: `Stream` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#batching--collate_records--the-collate-registry-recordstreamcollate) (`collate_records` is the default). - **Keras 3**: no `DataLoader` exists to do the batching, so [`RecordSequence`](docs/kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you) is the `keras.utils.PyDataset` half — row order, slicing, per-epoch reshuffle, `collate_records` — and a `transform` callable supplies the batch shape, exactly as `collate_fn` does for torch. diff --git a/docs/architecture.md b/docs/architecture.md index a13a24b..b87f962 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1196,3 +1196,84 @@ monkeypatch.setattr(recordstream.flow, "_result_readers", counting) # built-ins, and they register through the same public API (§1). - **Split `flow/execute.py` further** if a third route appears — but `is_linear` must stay the single gate, and the routes must keep agreeing record-for-record (§3). + +## 13. Dataset identity is a protocol, and a view propagates it verbatim (`recordstream.uri`, 2026-08-02) + +### Context + +A source knew what it read; nothing else could ask. `HuggingFaceSource` logged +`Loading mnist (train)...` and stamped `hf_path` / `hf_split` onto every record, but both are +per-record payload — a consumer wanting to answer "which data produced this?" for the RUN had to +reach into a record, or reach into the source's constructor arguments and reassemble an +identifier by hand. Every such consumer reassembled it differently, so two answers to the same +question could not be compared. + +What the question needs is one string per dataset that is stable across machines and across the +wrappers a config puts in front of a source — a trainer's `train_set` is typically a stream over +a split view over the real source, and all three read the same dataset. + +### Decision + +**A source may expose `dataset_uri` and `dataset_url`; free functions read them, and following +wrappers happens in the functions rather than in every wrapper.** + +Two properties, because they answer different questions. `dataset_uri` is the CANONICAL handle +— machine-parseable, stable, the string two runs are compared on. `dataset_url` is a link a +person can open, and `None` whenever the data has no web page. Collapsing them would force a +choice between a browsable string that lies about local data and a canonical one nobody can +click. + +`dataset_uri(source)` materializes a deferred source (as `project` does), reads the property, +and — when the object has none — follows a `.source` attribute and asks again, depth-capped and +cycle-safe. Every view source in this package uses that attribute name, so all of them work with +no code of their own, and so does any third-party wrapper that follows the convention. + +**A wrapper propagates the URI unchanged.** Decorating it with the slice or the split the wrapper +applies was considered and rejected: the handle identifies the *dataset*, and how much of it a +run consumed is already recorded by the wrapper's own configuration. Decorating would mean one +dataset reached two ways no longer compares equal — the single property the handle exists to +have. + +A source holding SEVERAL datasets (`ConcatSource`) answers `None` rather than picking a member; +`dataset_uris` is the plural form that fans out over them. + +### Consequences + +- **Asking is free and never loads.** The properties read stored configuration, so a tracking + layer can identify a dataset that the run has not touched yet — and a source that is never + iterated still names itself. +- **A new source needs one property, not a registration.** `SupportsDatasetIdentity` is a + `Protocol`, so a domain package opts in by defining the members. +- **A consumer derives the classification from the URI, not from the source.** The scheme is the + kind, the path is the name, a `split` parameter is the split — so a consumer recording datasets + needs no knowledge of any particular source type, and a new source type needs no change there. +- **The query string is sorted.** Two identically-configured sources must produce one string; + parameter order that depended on insertion would silently break comparison. +- **`None` is an ordinary answer**, not an error: an unconfigured source, a stream over an + in-memory list, and data with no web page all legitimately have nothing to say. + +### Example + +```python +from recordstream import HuggingFaceSource, Stream, dataset_uri, dataset_uris, dataset_url + +source = HuggingFaceSource(path="ylecun/mnist", split="train") +source.dataset_uri # 'hf://datasets/ylecun/mnist?split=train' +source.dataset_url # 'https://huggingface.co/datasets/ylecun/mnist/viewer/default/train' + +# a view reports the same DATASET — the slice it takes is its own configuration, not identity +dataset_uri(Stream(source=RangeSource(source=source, stop=100))) +# 'hf://datasets/ylecun/mnist?split=train' + +# several datasets end to end are not one dataset +dataset_uri(ConcatSource(sources=[a, b])) # None +dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] +``` + +### What you may change (and where it's documented) + +- **Give a new source an identity**: add the two properties. Keep them cheap and side-effect free + — they are asked of sources that may never be read. +- **Recognise a new wrapper shape**: `recordstream/uri.py` follows `.source`; a wrapper using a + different attribute name implements the properties itself instead. +- **Usage** is [docs/sources.md](sources.md#identifying-a-dataset). diff --git a/docs/augmentation.md b/docs/augmentation.md index f6e8c0f..7901150 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -112,6 +112,36 @@ Stochasticity lives where each library puts it — the engine adds no seed plumb - per-record gating of any op (native or library): `RandomApply(op=..., probability=..., random_state=N)`. +## Multiprocessing: two fork hazards, both handled + +A `DataLoader` worker is often a **forked** child (torch's default on Linux, and on macOS whenever +something in the process has set `fork` — `import fastai` does). A fork inherits memory but not +threads, so two things on this path would otherwise crash the child with a **SIGSEGV and no Python +traceback**, surfacing only as `DataLoader worker exited unexpectedly`: + +| hazard | guard | who calls it | +|---|---|---| +| OpenCV's thread pool, inherited across the fork (albumentations runs on cv2) | `cv2.setNumThreads(0)` | the engine, automatically, the first time it invokes an albumentations op | +| a lazy source first built in the child — the download reaches `_scproxy`, which is not fork-safe | `ensure_materialized(source)` | **you**, in the parent, before building the loader | + +The first needs nothing from you. The second does: + +```python +from recordstream import ensure_materialized, ensure_record_dataset + +dataset = ensure_materialized(ensure_record_dataset(train_set)) # reads ONE record, in the parent +loader = DataLoader(dataset, num_workers=4, collate_fn=collate_records) +``` + +They are **independent** — neither fixes the other. With the source warmed but the cv2 pool on, the +worker still dies; with the pool off but the source cold, it is still built in the child. Setting +`num_workers=0` avoids both by not forking at all, which is why the crash looks intermittent and +gets misread as a flaky test rather than an ordering bug. + +Turning cv2's pool off costs nothing where it matters: inside a worker the *worker* is the +parallelism, so cv2's own threads oversubscribe rather than help. The engine does it at the point +of use, never at import — a process that never touches albumentations keeps its OpenCV settings. + ## Other libraries — register an op family albumentations and torchvision v2 are the built-in families, registered through the same OPEN diff --git a/docs/sources.md b/docs/sources.md index b5203a8..1d71f65 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -97,4 +97,82 @@ val_set: !class:recordstream.sources.split.DatasetSplit() **HuggingFace native slicing** (alternative, no RecordStream split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. +## Identifying a dataset + +A source can name the data it reads, so a run record, a report, or a log line can point at it. +Two handles, because they answer different questions: + +| property | what it is | when it is `None` | +| --- | --- | --- | +| `dataset_uri` | the **canonical** identifier — machine-parseable and stable, the string two runs are compared on | the source has no dataset configured | +| `dataset_url` | a link a **person** can open | the data has no web page (anything local) | + +```python +from recordstream import HuggingFaceSource, dataset_uri, dataset_url + +source = HuggingFaceSource(path="ylecun/mnist", split="train") +source.dataset_uri # 'hf://datasets/ylecun/mnist?split=train' +source.dataset_url # 'https://huggingface.co/datasets/ylecun/mnist/viewer/default/train' +``` + +What `HuggingFaceSource` produces, for each way it can be configured: + +| configuration | `dataset_uri` | `dataset_url` | +| --- | --- | --- | +| `path="ylecun/mnist", split="train"` | `hf://datasets/ylecun/mnist?split=train` | `…/ylecun/mnist/viewer/default/train` | +| `+ name="fashion", revision="abc123"` | `hf://datasets/ylecun/mnist?name=fashion&revision=abc123&split=train` | `…/ylecun/mnist/viewer/fashion/train` | +| `path="/data/imagefolder"` | `file:///data/imagefolder?split=train` | `None` | +| `path=""` | `None` | `None` | + +Both read **stored configuration only** — asking never loads, downloads or opens anything, so a +source that is never iterated still names itself. Whether a `path` is a Hub repo id or a local +directory is decided by whether it exists on disk, the same question `load_dataset` answers. The +query parameters are sorted, so one configuration has exactly one URI. + +**Use the free functions rather than the attributes when the source may be wrapped or deferred.** +`dataset_uri(x)` / `dataset_url(x)` materialize a `!class:` marker straight out of a config, and +follow a view's `.source` to the dataset underneath: + +```python +from recordstream import RangeSource, Stream, dataset_uri + +# a view reports the same DATASET — the slice it takes is its own configuration, not identity +dataset_uri(Stream(source=RangeSource(source=source, stop=100))) +# 'hf://datasets/ylecun/mnist?split=train' +``` + +That verbatim propagation is deliberate: one dataset reached two ways must compare equal. How +much of it a run consumed is recorded by the wrapper's own settings (`start` / `stop`, the split +fractions), not by mangling the handle. + +A source holding **several** datasets declines to be one of them — `ConcatSource` answers `None`, +and `dataset_uris` is the plural form: + +```python +from recordstream import ConcatSource, dataset_uri, dataset_uris + +dataset_uri(ConcatSource(sources=[a, b])) # None — a concatenation is not one dataset +dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] +``` + +**Adding identity to your own source** is defining the two properties — `SupportsDatasetIdentity` +is a `Protocol`, so there is nothing to register: + +```python +@configurable(category="source") +class MyStoreSource: + def __init__(self, bucket: str = "", prefix: str = "") -> None: + self.bucket, self.prefix = bucket, prefix + + @property + def dataset_uri(self) -> Optional[str]: + return f"s3://{self.bucket}/{self.prefix}" if self.bucket else None + + @property + def dataset_url(self) -> Optional[str]: + return None # no web page: honest, and never an error +``` + +Rationale: [docs/architecture.md §13](architecture.md#13-dataset-identity-is-a-protocol-and-a-view-propagates-it-verbatim-recordstreamuri-2026-08-02). + > **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. diff --git a/recordstream/__init__.py b/recordstream/__init__.py index aa9003d..e6364d5 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -18,6 +18,7 @@ RecordSource, Stream, WrappedOp, + ensure_materialized, ensure_record_dataset, register_op_family, registered_op_families, @@ -123,6 +124,7 @@ "Stream", "JointStream", "RecordSource", + "ensure_materialized", "ensure_record_dataset", "FilterOp", "WrappedOp", diff --git a/recordstream/core/__init__.py b/recordstream/core/__init__.py index 86aa594..6f06ac6 100644 --- a/recordstream/core/__init__.py +++ b/recordstream/core/__init__.py @@ -48,6 +48,7 @@ Stream, _check_ops_materialized, _worker_task, + ensure_materialized, ensure_record_dataset, linear_steps, ) @@ -62,6 +63,7 @@ "RecordSource", "Stream", "WrappedOp", + "ensure_materialized", "ensure_record_dataset", "linear_steps", "register_op_family", diff --git a/recordstream/core/families.py b/recordstream/core/families.py index 8c03b1a..edb3298 100644 --- a/recordstream/core/families.py +++ b/recordstream/core/families.py @@ -106,6 +106,48 @@ def _is_albumentations(op: Any) -> bool: return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(op).__mro__) +#: Whether :func:`_disable_cv2_threading` has already run. Module-level, so the cost of the +#: guarantee is one bool check per op application. +_CV2_THREADING_DISABLED = False + + +def _disable_cv2_threading() -> None: + """Turn OpenCV's internal thread pool off, ONCE, the first time albumentations is used. + + **This prevents a SIGSEGV, not a slowdown.** albumentations runs on OpenCV, whose thread pool + is not fork-safe: once a parent process has executed a cv2 op, a FORKED child inheriting that + pool crashes with *"DataLoader worker ... is killed by signal: Segmentation fault: 11"* — no + Python traceback, because the child never gets to raise. Measured on macOS, 2026-08-02, with a + ``DataLoader(num_workers=2)`` over a ``Stream`` whose ops list contained ``A.Resize``: it + segfaults reliably with the pool on and passes reliably with it off. + + It is the SECOND fork hazard on this path and is independent of the first + (:func:`~recordstream.ensure_materialized`, which warms a lazy source so the child does not + run a download through the non-fork-safe ``_scproxy``). Neither fixes the other: with the + source warmed and the pool ON the worker still dies, and with the pool off a cold source is + still built in the child. A consumer that forks needs both. + + Turning the pool off costs nothing where it matters. Inside a ``DataLoader`` worker the + WORKER is the parallelism — cv2's own threads oversubscribe the machine rather than help — + which is why albumentations' own documentation recommends exactly this for multiprocessing + loaders. + + Done HERE, at the one place this package invokes albumentations, rather than at import: a + process that never uses albumentations must not have its OpenCV settings changed by importing + a data library, and cv2 must not become an import-time dependency of the engine. + """ + global _CV2_THREADING_DISABLED + if _CV2_THREADING_DISABLED: + return + _CV2_THREADING_DISABLED = True # set FIRST: a cv2-less install must not retry on every record + try: + import cv2 + + cv2.setNumThreads(0) + except Exception as exc: # pragma: no cover - albumentations without cv2 is not a real install + logger.debug(f"could not disable OpenCV threading ({exc}); a forked DataLoader worker may crash.") + + def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: """albumentations dispatches by KWARG NAME: hand the op exactly its own target keys present in the record (one call = one joint draw across them); array outputs are @@ -113,6 +155,7 @@ def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: keep their type and metadata. Box-carrying augmentation belongs in albumentations' own ``A.Compose(..., bbox_params=...)`` — format handling is Compose's job in that library. """ + _disable_cv2_threading() kwargs = {k: record[k] for k in _ALB_KEYS if k in record} if not kwargs: logger.debug( diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py index e699804..f34f43e 100644 --- a/recordstream/core/stream.py +++ b/recordstream/core/stream.py @@ -423,6 +423,45 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: yield {k: v for k, v in record.items() if k in want} +def ensure_materialized(source: RecordSource) -> RecordSource: + """Read ONE whole record, so everything this source builds lazily is built in THIS process. + + The companion to :func:`ensure_record_dataset`: that one normalizes a source's TYPE, this one + normalizes its STATE. Returns the source, so it composes. + + A source in this package is lazy on purpose — a constructor does no work, and the download / + file open / client construction happens on first read. That is right until the first read + happens somewhere it must not, and there is one such place: **a forked child process.** + + Measured, on macOS, 2026-08-02. A ``DataLoader`` worker was the first to touch a + ``HuggingFaceSource``, so ``load_dataset`` ran in the child, called ``hf_hub_download`` -> + ``httpx.Client()`` -> ``urllib.request.getproxies`` -> ``_scproxy`` -> CoreFoundation, which + is not fork-safe: **SIGSEGV**, with no Python traceback, surfacing only as + ``DataLoader worker exited unexpectedly``. Calling this in the parent first makes the child + inherit a source that needs nothing, and the crash does not happen. The consumer cannot + always choose spawn instead — a framework may set the start method globally (fastai sets + ``fork`` at import), and spawning has its own cost (a model on Apple's MPS cannot be shared + to a spawned worker at all). + + It reads a whole RECORD rather than asking a cheaper question, and that is the point: + ``len(source)`` was measured NOT to be enough, because loading the dataset object is not the + same as building everything a read needs. :func:`~recordstream.first_value` is no substitute + either — it is projection-aware, so it deliberately avoids building the values it was not + asked for. + + An empty source is not an error: there is nothing to build, and a caller should not need a + guard for a split that happens to have no rows. + """ + try: + if hasattr(source, "__getitem__"): + source[0] # type: ignore[index] + else: + next(iter(source), None) # type: ignore[call-overload] + except (IndexError, StopIteration): + pass # empty source — nothing to warm, and not an error + return source + + def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> "Stream": """Normalize any wired source into a map-style ``Dataset`` that yields record dicts. diff --git a/recordstream/sources/huggingface.py b/recordstream/sources/huggingface.py index b85bfc3..954213d 100644 --- a/recordstream/sources/huggingface.py +++ b/recordstream/sources/huggingface.py @@ -89,6 +89,13 @@ class HuggingFaceSource: metadata_features: Columns -> per-column ``Label`` entries; ``None``=none, ``"*"``=all-but-i/o, else a list. count: Optional cap on the number of records yielded (useful for fast smoke runs). name: Optional HF subset/config name (e.g. for multi-config datasets). + revision: Optional dataset git revision (branch / tag / commit). Declared rather than + left to ``load_kwargs`` because it is part of the source's IDENTITY — ``dataset_uri`` + reads it. + load_kwargs: The remaining ``datasets.load_dataset`` options (``token``, ``cache_dir``, + ``trust_remote_code``, …) as an explicit mapping. A dict rather than ``**kwargs``: + see the note in ``__init__`` — a ``**kwargs`` constructor accepts every broadcast key + there is, and ``load_dataset`` turns an unknown keyword into a builder config name. """ def __init__( @@ -100,7 +107,8 @@ def __init__( metadata_features: Optional[List[str] | str] = "*", count: Optional[int] = None, name: Optional[str] = None, - **kwargs: Any, + revision: Optional[str] = None, + load_kwargs: Optional[Dict[str, Any]] = None, ) -> None: # Lazy constructor: store config only — never load here. Real work (the network/disk # download) is deferred to the ``dataset`` property so the object is cheap to build and @@ -114,12 +122,34 @@ def __init__( self.metadata_features = metadata_features self.count = count self.name = name - # Extra kwargs forwarded verbatim to ``datasets.load_dataset`` at load time (e.g. ``token``, - # ``trust_remote_code``). Captured now, applied lazily in the ``dataset`` property. - self._load_kwargs = dict(kwargs) + self.revision = revision + # The long tail of `datasets.load_dataset` options (``token``, ``cache_dir``, + # ``trust_remote_code``), as an EXPLICIT dict rather than the `**kwargs` this took until + # 2026-08-02. That `**kwargs` was a live hazard, not merely untidy: confluid/liquifai + # broadcast a key into any node whose constructor ACCEPTS it, and a `**kwargs` constructor + # accepts every key there is — so unrelated run identity landed in `load_dataset`, which + # turns unknown keyword arguments into a builder CONFIG NAME. A single + # `sonair train … --run_name my_run` therefore looked for `mnist` under a config named + # `default-`, missed the cache, and went to the Hub. Measured: the same + # command passes with no name override and fails with one. + self._load_kwargs = dict(load_kwargs or {}) # Lazy cache for the materialized dataset (see the ``dataset`` property). self._dataset: Any = None + @property + def load_options(self) -> Dict[str, Any]: + """``load_kwargs`` with the declared ``revision`` folded in — what reaches ``load_dataset``. + + Recomputed on every read rather than merged in ``__init__``, per the workspace + derived-state rule: ``revision`` is a DECLARED parameter, so the config layer may set it + post-construction (that is how a broadcast key arrives), and a dict assembled once in the + constructor would silently keep the value the object was born with. + """ + options = dict(self._load_kwargs) + if self.revision is not None: + options["revision"] = self.revision + return options + @property def dataset(self) -> Any: """The HF dataset, loaded on first access and cached. Resetting ``_dataset`` to None reloads. @@ -139,7 +169,7 @@ def dataset(self) -> Any: # can tie the run to a specific dataset, and the browsable URL is what makes that # tie followable rather than merely recorded. logger.info(f"HuggingFaceSource: Loading {self.dataset_url or self.dataset_uri}...") - self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self._load_kwargs) + self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self.load_options) return self._dataset # -- identity (see recordstream.uri) ------------------------------------------------------ @@ -154,9 +184,8 @@ def _identity_query(self) -> str: parts: Dict[str, str] = {} if self.name: parts["name"] = str(self.name) - revision = self._load_kwargs.get("revision") - if revision: - parts["revision"] = str(revision) + if self.revision: + parts["revision"] = str(self.revision) if self.split: parts["split"] = str(self.split) return urlencode(sorted(parts.items())) diff --git a/recordstream/uri.py b/recordstream/uri.py index 62e24e4..12b38b4 100644 --- a/recordstream/uri.py +++ b/recordstream/uri.py @@ -67,9 +67,7 @@ def _identity(source: Any, attribute: str) -> Optional[str]: deferred source, read the handle if it has one, otherwise take one hop into the wrapped source and ask again. """ - from confluid import flow - - current = flow(source) + current = _materialize(source) for _ in range(MAX_WRAPPER_DEPTH): if current is None: return None @@ -79,10 +77,23 @@ def _identity(source: Any, attribute: str) -> Optional[str]: wrapped = getattr(current, "source", None) if wrapped is None or wrapped is current: return None - current = flow(wrapped) + current = _materialize(wrapped) return None +def _materialize(node: Any) -> Any: + """Build ``node`` only if it is a DEFERRED config marker; hand a live object back untouched. + + Narrower than a bare ``flow()``, and the difference is the "asking never loads" property. + ``flow()`` on a LIVE object still runs confluid's post-construction ``solidify()`` hook, so a + source that grows one would be materialized merely by being asked its name. A marker has + nothing to read until it is built, and building one is cheap by the lazy-construction rule. + """ + from confluid import Fluid, flow + + return flow(node) if isinstance(node, Fluid) else node + + def dataset_uri(source: Any) -> Optional[str]: """The canonical URI identifying the dataset ``source`` reads, or ``None``. @@ -135,14 +146,12 @@ def dataset_uris(source: Any) -> List[str]: dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] """ - from confluid import flow - found: List[str] = [] def visit(node: Any, depth: int) -> None: if node is None or depth > MAX_WRAPPER_DEPTH: return - node = flow(node) + node = _materialize(node) uri = dataset_uri(node) if uri: if uri not in found: diff --git a/tests/test_dataset_uri.py b/tests/test_dataset_uri.py index a97e7f9..bd605c0 100644 --- a/tests/test_dataset_uri.py +++ b/tests/test_dataset_uri.py @@ -89,6 +89,42 @@ def test_huggingface_source_satisfies_the_protocol() -> None: assert isinstance(HuggingFaceSource(path="a/b"), SupportsDatasetIdentity) +# --- the load-option surface `revision` is shared with ---------------------------------------- + + +def test_revision_reaches_both_the_uri_and_the_load_options() -> None: + source = HuggingFaceSource(path="a/b", revision="v2") + assert "revision=v2" in (source.dataset_uri or "") + assert source.load_options == {"revision": "v2"} + + +def test_load_options_merges_the_explicit_dict_under_the_declared_revision() -> None: + source = HuggingFaceSource(path="a/b", revision="v2", load_kwargs={"token": "t", "cache_dir": "/c"}) + assert source.load_options == {"token": "t", "cache_dir": "/c", "revision": "v2"} + + +def test_a_revision_set_after_construction_is_not_stale() -> None: + """`revision` is a DECLARED param, so the config layer may set it post-construction.""" + source = HuggingFaceSource(path="a/b") + assert source.load_options == {} + source.revision = "v3" + assert source.load_options == {"revision": "v3"} + assert "revision=v3" in (source.dataset_uri or "") + + +def test_the_constructor_does_not_accept_arbitrary_keywords() -> None: + """A `**kwargs` constructor accepts every broadcast key there is; this one must not. + + The rejection is what keeps an unrelated broadcast key (a run name) out of + ``load_dataset``, which reads an unknown keyword as a builder CONFIG NAME and so misses + the cache entirely. The exception type is confluid's validation wrap, not a bare + ``TypeError``, so the assertion is on the rejected KEY being named. + """ + with pytest.raises(Exception) as raised: + HuggingFaceSource(path="a/b", run_name="my_run") # type: ignore[call-arg] + assert "run_name" in str(raised.value) + + # --- the free functions ---------------------------------------------------------------------- diff --git a/tests/test_fork_safety.py b/tests/test_fork_safety.py new file mode 100644 index 0000000..249cdb7 --- /dev/null +++ b/tests/test_fork_safety.py @@ -0,0 +1,158 @@ +"""The two fork hazards on the DataLoader path, and the guards that remove them. + +A ``DataLoader`` worker may be a FORKED child — torch's default start method on Linux, and what +you get on macOS whenever something in the process has set ``fork`` (fastai does, at import). A +forked child inherits the parent's memory but NOT its threads, so anything holding a thread pool +or a system handle across the fork can die there. When it does it dies with a **SIGSEGV and no +Python traceback**, surfacing only as ``DataLoader worker exited unexpectedly`` — which is why +both guards are pinned here rather than left to be re-diagnosed. + +The two are INDEPENDENT and neither fixes the other: + +* :func:`~recordstream.ensure_materialized` — a lazy source built in the child runs its download + through ``_scproxy`` / CoreFoundation, which is not fork-safe. +* ``cv2.setNumThreads(0)`` — OpenCV's thread pool, inherited across a fork, crashes the child. + albumentations runs on OpenCV, so any ops list containing one is exposed. + +The subprocess tests are the ones that matter: they reproduce the real crash, so removing a guard +makes them fail rather than making them pass differently. They are subprocesses because a SIGSEGV +takes the whole interpreter down — an in-process test could not report it. +""" + +import subprocess +import sys +import textwrap +from typing import Any + +import numpy as np +import pytest + +from recordstream import Image, Mask, ensure_materialized + +albumentations = pytest.importorskip("albumentations") +pytest.importorskip("torch") + + +def _run(body: str) -> subprocess.CompletedProcess: + """Run ``body`` in a fresh interpreter that forks its DataLoader workers.""" + script = ( + textwrap.dedent( + """ + import multiprocessing as mp + mp.set_start_method("fork", force=True) # what fastai does at import + import numpy as np, torch + from torch.utils.data import DataLoader + from recordstream import Image, Mask, collate_records + from recordstream.core import _apply_op + import albumentations as A + """ + ) + + textwrap.dedent(body) + ) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=300) + + +# --------------------------------------------------------------------------- # +# cv2's thread pool +# --------------------------------------------------------------------------- # +class TestOpenCVThreadingIsDisabled: + def test_applying_an_albumentations_op_turns_the_pool_off(self, monkeypatch: pytest.MonkeyPatch) -> None: + """The guard fires on USE, not on import — a process that never touches albumentations + must not have its OpenCV settings changed by importing a data library. + + The module flag is reset first because the guard is deliberately ONE-TIME (its cost is + then a single bool check per op); by the time this runs, another test has usually already + tripped it. + """ + import cv2 + + from recordstream.core import _apply_op, families + + monkeypatch.setattr(families, "_CV2_THREADING_DISABLED", False) + record = { + "image": Image(np.zeros((8, 8, 3), np.uint8)), + "mask": Mask(np.zeros((8, 8), np.int64)), + } + + _apply_op(record, albumentations.Resize(height=4, width=4)) + # `setNumThreads(0)` means "no pool — run on the calling thread", and cv2 then REPORTS 1 + # rather than 0 (measured). Asserting 0 would look right and always fail. + assert cv2.getNumThreads() == 1 + + def test_the_guard_runs_only_once(self, monkeypatch: pytest.MonkeyPatch) -> None: + """So the cost of the guarantee is one bool check per op, not a cv2 call per record.""" + from recordstream.core import _apply_op, families + + monkeypatch.setattr(families, "_CV2_THREADING_DISABLED", False) + calls: list = [] + monkeypatch.setattr(families, "_disable_cv2_threading", lambda: calls.append(1)) + + record = {"image": Image(np.zeros((8, 8, 3), np.uint8))} + resize = albumentations.Resize(height=4, width=4) + for _ in range(3): + _apply_op(record, resize) + # The invoker calls it every time; the FUNCTION is what short-circuits, so the guarantee + # holds for a record that arrives after something else re-enabled the pool. + assert len(calls) == 3 + + def test_a_forked_worker_survives_an_albumentations_op(self) -> None: + """The real crash, reproduced. Without the guard this dies with + *"DataLoader worker ... is killed by signal: Segmentation fault: 11"*. + """ + result = _run( + """ + records = [{"image": Image(np.zeros((32, 32, 3), np.uint8)), + "mask": Mask(np.zeros((32, 32), np.int64))} for _ in range(8)] + resize = A.Resize(height=16, width=16) + # The PARENT applies one first — which is what creates the thread pool that the + # forked child would inherit. Without that this passes vacuously. + _apply_op(records[0], resize) + + class _DS(torch.utils.data.Dataset): + def __len__(self): return len(records) + def __getitem__(self, i): return _apply_op(records[i], resize) + + loader = DataLoader(_DS(), batch_size=2, num_workers=2, collate_fn=collate_records) + n = sum(1 for _ in loader) + print("BATCHES", n) + """ + ) + assert result.returncode == 0, f"forked worker died:\n{result.stderr[-2000:]}" + assert "BATCHES 4" in result.stdout + + +# --------------------------------------------------------------------------- # +# a lazily-built source +# --------------------------------------------------------------------------- # +class TestEnsureMaterialized: + def test_it_reads_one_whole_record(self) -> None: + """A whole RECORD, not a cheaper question: loading a dataset object is not the same as + building everything a read needs, so `len()` was measured NOT to be enough.""" + reads: list = [] + + class _Lazy: + def __getitem__(self, index: int) -> dict: + reads.append(index) + return {"image": Image(np.zeros((4, 4, 3), np.uint8))} + + def __len__(self) -> int: + return 4 + + source = _Lazy() + assert ensure_materialized(source) is source, "it must return the source, so it composes" + assert reads == [0] + + def test_an_empty_source_is_not_an_error(self) -> None: + """A split that happens to have no rows has nothing to warm, and a caller should not + need a guard for it.""" + assert ensure_materialized([]) == [] + + def test_an_iterable_only_source_is_warmed_too(self) -> None: + consumed = [] + + def _gen() -> Any: + consumed.append(1) + yield {"image": Image(np.zeros((4, 4, 3), np.uint8))} + + ensure_materialized(_gen()) + assert consumed == [1] diff --git a/tests/test_record_source.py b/tests/test_record_source.py index 373f758..afa1a46 100644 --- a/tests/test_record_source.py +++ b/tests/test_record_source.py @@ -10,7 +10,7 @@ import numpy as np import torch -from recordstream import Label, Record, Stream, ensure_record_dataset +from recordstream import Label, Record, Stream, ensure_materialized, ensure_record_dataset class _RowDataset(torch.utils.data.Dataset): @@ -79,3 +79,78 @@ def test_exported_from_the_package_root() -> None: assert "ensure_record_dataset" in recordstream.__all__ assert "RecordSource" in recordstream.__all__ + + +# --------------------------------------------------------------------------- +# ensure_materialized — normalize a source's STATE, not its type +# --------------------------------------------------------------------------- +class _LazySource: + """A source that does its real work on first read, like every source in this package.""" + + def __init__(self, rows: int = 3) -> None: + self.rows = rows + self.reads = 0 + self.built = False + + def __len__(self) -> int: + return self.rows # deliberately does NOT build: len() was measured not to be enough + + def __getitem__(self, index: int) -> Dict[str, Any]: + if index >= self.rows: + raise IndexError(index) + self.built = True + self.reads += 1 + return {"image": index} + + +def test_ensure_materialized_builds_what_a_read_needs() -> None: + """The point: after this, a forked child inherits a source that needs nothing.""" + source = _LazySource() + + returned = ensure_materialized(source) + + assert source.built, "the source was never actually read" + assert returned is source, "it returns the source so it composes" + + +def test_len_is_not_enough_which_is_why_this_reads_a_record() -> None: + """Pins the measurement the docstring cites, so the implementation cannot be 'simplified'. + + Loading a dataset object is not the same as building everything a read needs — a real + HuggingFaceSource still went to the Hub from inside a worker after `len()` in the parent. + """ + source = _LazySource() + + len(source) + + assert not source.built, "if len() built it, this test's premise is wrong, not the code" + + +def test_it_reads_exactly_one_record() -> None: + """One is enough, and more would make warming a large split expensive.""" + source = _LazySource(rows=100) + + ensure_materialized(source) + + assert source.reads == 1 + + +def test_an_empty_source_is_not_an_error() -> None: + """A split with no rows has nothing to build, and a caller should need no guard.""" + assert ensure_materialized(_LazySource(rows=0)) is not None + + +def test_an_iterable_only_source_is_warmed_too() -> None: + """Not every source is map-style; the iterable path must build the same way.""" + + class _IterableOnly: + def __init__(self) -> None: + self.built = False + + def __iter__(self) -> Iterator[Dict[str, Any]]: + self.built = True + yield {"image": 0} + + source = _IterableOnly() + ensure_materialized(source) + assert source.built From c388e028fea5531b1a7b6ab77b8d406e986e412f Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 3 Aug 2026 16:05:26 +0200 Subject: [PATCH 074/102] chore(tasks): drop two duplicate backlog entries "GPU-aware batch processing engine" and "S3 storage backend support" each restated a detailed entry further down the file. Kept the detailed halves; 21 open -> 19. --- TASKS.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/TASKS.md b/TASKS.md index d0fd364..afd1c79 100644 --- a/TASKS.md +++ b/TASKS.md @@ -10,8 +10,6 @@ workspace root `TASKS.md`. Completed items are not archived here — git history - [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, marainer, …) were not. @low - [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in recordstream, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor - [ ] **Decide whether `PredictionsSink` collapses into `DataSink`** @medium @refactor — since 2026-07-29 this package carries TWO sink protocols: `storage.base.DataSink.write(record)` (adapted into op chains by `RecordSinkOp`) and `predictions.PredictionsSink.write(prediction, metadata)`. The split is real — a model emits a BATCH while the sink contract is per-record, so the prediction and its record's metadata arrive separately — and it is load-bearing downstream, where a visual editor's node palette keys off the signature difference. The alternative: have the consuming runnable build the record (it already holds both halves — it slices the batch per record before calling `write`) and write through the ordinary `DataSink`, deleting `PredictionsSink` and letting prediction sinks become ordinary `category="sink"` canvas nodes. That touches the predict path of every consuming runnable, which is why it was NOT bundled into the move. Decide deliberately; do not let the two protocols blur by drift. Rationale for the current state: `docs/architecture.md` §8. -- [ ] GPU-aware batch processing engine @performance -- [ ] S3 storage backend support @feature - [ ] **RecordStream Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance - [ ] **Typed-bag PoC → torch-`Tensor`-subclass item base:** array-backed items are `np.ndarray` subclasses only; add a `TensorItem` base (torch `__torch_function__` attr-preservation) so torch payloads can be array items instead of riding wrapper `Signal`s. @low @ml - [ ] **Typed-bag PoC → confluid-native item-type discovery:** item types register in a local `register_item` registry (an `np.ndarray` subclass fights confluid's `__init__` validation wrap); make item types `@configurable(category="itemtype")` + entry-pointed so navigaitor/StreamStudio enumerate them as socket types. @low @tooling From 42ff07963e2b42fc7c00fbf17ba96731eef17fa5 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 3 Aug 2026 18:33:26 +0200 Subject: [PATCH 075/102] refactor: follow the marainer -> matrainer rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tracking package is now matrainer — a portmanteau of MARINER (the one who keeps the log) and TRAINER — so every import, class path, YAML tag, extra name and doc reference here moves with it. There is no back-compat alias upstream, so a missed reference fails loudly rather than silently resolving. Mechanical apart from that: every changed line is the token swap. --- AGENTS.md | 8 ++++---- TASKS.md | 2 +- docs/architecture.md | 2 +- recordstream/batch.py | 2 +- recordstream/labels.py | 8 ++++---- tests/test_labels.py | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1dd5176..af52c0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,9 +16,9 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → ONE step-graph engine behind two facades (`Stream`/`JointStream` for the dataset surface, `FlowGraph` for a `flow:` document) → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config; the context ops + the flow⇄ops lowering pass were DELETED 2026-07-30 (one step-graph engine, see the mandate below). Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of marainer when marainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the marainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of matrainer when matrainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the matrainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). -- **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and marainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. +- **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and matrainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. - **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. @@ -41,8 +41,8 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. **`first_value(source, key)` is the ONE-PEEK primitive beside them (2026-08-02)** — the first non-`None` value under `key`, or `None` when there is none. It answers what a column's values ARE without walking the set, and it belongs here rather than in any consumer because it is `iter_key` plus a `next()`: it inherits all three of that helper's properties (a projection-aware source never builds the values it does not ask for, a deferred source is materialized first, the walk is lazy so a normal source costs ONE record) and its unwrapping rules are what make the answer meaningful — a `MultiLabel` arrives as its `.values` LIST, so a sequence IS a multi-label column, decided by the item type rather than by guessing what a list might mean. The canonical call site pairs it with `is_class_id` to decide whether the targets need a `LabelMap` at all. It was extracted from EIGHT byte-identical private copies in one consumer's training backends (2026-08-02); a consumer re-deriving it is re-deriving `iter_key`'s contract. Pins: `tests/test_projection.py`. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. -- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in marainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file marainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from marainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in matrainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file matrainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from matrainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. diff --git a/TASKS.md b/TASKS.md index afd1c79..89a459d 100644 --- a/TASKS.md +++ b/TASKS.md @@ -7,7 +7,7 @@ workspace root `TASKS.md`. Completed items are not archived here — git history - [ ] **`to_tensor`'s `normalize` heuristic silently corrupts already-standardized floats** @bug — `recordstream/ops/torch.py::to_tensor` does `elif normalize and tensor.max() > 1.0: tensor = tensor / 255.0`, i.e. it infers "a float whose max exceeds 1 must be 0-255 pixels". An ImageNet-standardized array (range ~[-2.12, 2.64]) satisfies that test, so chaining a `Normalize` op before `ToTensor` divides the standardized values by 255 and squashes them to ~[-0.01, 0.01] — no error, no warning, just a model that learns nothing. Hit for real 2026-07-29 while wiring a consumer's example config; the caller's fix is `ToTensor(normalize=false)`, which is correct but only discoverable by inspecting the tensor. Options: gate the rescale on an INTEGER dtype only (what the docstring already claims — "scale integer pixel inputs"), or keep the heuristic and warn when it fires on a float input. Changing it is a behaviour change for anyone relying on the 0-255-float path, so it needs a decision rather than a quiet edit. - [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor - [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `recordstream.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in recordstream; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low -- [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, marainer, …) were not. @low +- [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, matrainer, …) were not. @low - [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in recordstream, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor - [ ] **Decide whether `PredictionsSink` collapses into `DataSink`** @medium @refactor — since 2026-07-29 this package carries TWO sink protocols: `storage.base.DataSink.write(record)` (adapted into op chains by `RecordSinkOp`) and `predictions.PredictionsSink.write(prediction, metadata)`. The split is real — a model emits a BATCH while the sink contract is per-record, so the prediction and its record's metadata arrive separately — and it is load-bearing downstream, where a visual editor's node palette keys off the signature difference. The alternative: have the consuming runnable build the record (it already holds both halves — it slices the batch per record before calling `write`) and write through the ordinary `DataSink`, deleting `PredictionsSink` and letting prediction sinks become ordinary `category="sink"` canvas nodes. That touches the predict path of every consuming runnable, which is why it was NOT bundled into the move. Decide deliberately; do not let the two protocols blur by drift. Rationale for the current state: `docs/architecture.md` §8. - [ ] **RecordStream Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance diff --git a/docs/architecture.md b/docs/architecture.md index b87f962..bdd9731 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -841,7 +841,7 @@ def predict_step(self, batch, batch_idx): ### Context `recordstream` declared `torch` as a hard dependency, so `import recordstream` imported ~2GB of -PyTorch — and marainer inherited it transitively, declaring no torch of its own. That was fine +PyTorch — and matrainer inherited it transitively, declaring no torch of its own. That was fine while every consumer was a Lightning trainer. It stopped being fine when a second training engine landed: a Keras-on-TensorFlow install, or a plain-numpy dataset-conversion job, paid for a framework it never called. diff --git a/recordstream/batch.py b/recordstream/batch.py index a84db2c..3b466c8 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -79,7 +79,7 @@ def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") key: The record key holding the multi-label target. num_classes: Matrix width. Ids outside ``[0, num_classes)`` are IGNORED rather than raising — a stray label must not abort a training run (the same rule - ``marainer.torch.inverse_frequency_weights`` applies to class counting). + ``matrainer.torch.inverse_frequency_weights`` applies to class counting). dtype: Result dtype, default ``"float32"`` — the multi-label losses (``BCEWithLogitsLoss`` and friends) want float targets shaped like the logits, not integer class ids. diff --git a/recordstream/labels.py b/recordstream/labels.py index 932e607..425acb2 100644 --- a/recordstream/labels.py +++ b/recordstream/labels.py @@ -7,7 +7,7 @@ * :meth:`LabelMap.fit` derives a deterministic name→id mapping from a stream of raw targets (backed by scikit-learn's ``LabelEncoder``) — the one-time fit that happens at **train** time. -* :meth:`LabelMap.save` / :meth:`LabelMap.load` persist it (in marainer's ``class_names.json`` +* :meth:`LabelMap.save` / :meth:`LabelMap.load` persist it (in matrainer's ``class_names.json`` format) so **eval / predict** reload the *same* mapping rather than refitting on a subset. * :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the recordstream ops that apply it. @@ -62,7 +62,7 @@ class LabelMap: Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`class_names`, builds the :class:`~recordstream.ops.target.EncodeTarget` / :class:`~recordstream.ops.target.DecodeTarget` - that apply it, and round-trips to disk in marainer's ``class_names.json`` format. + that apply it, and round-trips to disk in matrainer's ``class_names.json`` format. Args: mapping: Explicit name→id lookup, e.g. ``{"cat": 0, "dog": 1}``. ``None`` (default) builds an @@ -219,7 +219,7 @@ def to_ids(self, target: Any) -> List[int]: return ids def save(self, path: Union[str, Path]) -> None: - """Persist as ``{"class_names": [...], "num_classes": N}`` — marainer's ``class_names.json`` format. + """Persist as ``{"class_names": [...], "num_classes": N}`` — matrainer's ``class_names.json`` format. Args: path: Destination file. Parent directories are created as needed. @@ -231,7 +231,7 @@ def save(self, path: Union[str, Path]) -> None: @classmethod def load(cls, path: Union[str, Path]) -> "LabelMap": - """Restore from a ``class_names.json``-shaped file written by :meth:`save` or marainer. + """Restore from a ``class_names.json``-shaped file written by :meth:`save` or matrainer. Args: path: Source file shaped ``{"class_names": [...]}`` (the ``num_classes`` key is optional; diff --git a/tests/test_labels.py b/tests/test_labels.py index 583830e..84b7e4b 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -108,7 +108,7 @@ def test_encode_op_ignore_unknown() -> None: # --------------------------------------------------------------------------- -# Persistence — same format as marainer's class_names.json +# Persistence — same format as matrainer's class_names.json # --------------------------------------------------------------------------- @@ -129,8 +129,8 @@ def test_save_writes_class_names_payload(tmp_path: object) -> None: assert data == {"class_names": ["a", "b", "c"], "num_classes": 3} -def test_load_reads_marainer_written_file(tmp_path: object) -> None: - # A class_names.json written by marainer's _write_class_names is byte-compatible. +def test_load_reads_matrainer_written_file(tmp_path: object) -> None: + # A class_names.json written by matrainer's _write_class_names is byte-compatible. path = tmp_path / "class_names.json" # type: ignore[operator] path.write_text(json.dumps({"class_names": ["x", "y"], "num_classes": 2})) # type: ignore[attr-defined] lm = LabelMap.load(path) From 013239f9fe460024a5d5349b72e2c1c96cdbe672 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 4 Aug 2026 09:25:43 +0200 Subject: [PATCH 076/102] test(docs): check that documentation links resolve, and fix the one dead anchor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapted from confluid: the absolute-URL rule is dropped because this project's README links to docs RELATIVELY, which is correct for a project not published to PyPI — that rule exists because a PyPI landing page cannot resolve a relative link. Those relative links are still covered, since the README is in the scanned set. It found a real one on the first run. `docs/kinds.md` pointed at `architecture.md#10-the-frameworks-batching-half-lives-beside-the-collate-2026-07-30`, but the heading gained a parenthetical since that link was written, so the real anchor ends `...-recordstreamkeras-2026-07-30`. That is the failure mode this check exists for: a wrong anchor does not 404, it silently lands the reader at the top of the page. Nothing would ever have reported it. 34 cross-doc links, all resolving now. --- docs/kinds.md | 2 +- tests/test_docs_links.py | 155 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tests/test_docs_links.py diff --git a/docs/kinds.md b/docs/kinds.md index 198cedc..21dcd7d 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -97,7 +97,7 @@ model.fit(train, epochs=3) # reshuffles betwee `transform` is the whole task-facing surface: it maps one collated record to what the model consumes, so the shape decision stays in your code exactly as it does with a `DataLoader`. Omit it and `__getitem__` hands over the batched record itself — which is also what `seq.batches()` yields, the pairing half of prediction (a model emits `[N, ...]` while a [`PredictionsSink`](predictions.md) writes per record, so you need the batch its output came from to read that batch's [`batch_metadata`](#reading-a-batch-back-recordstreambatch)). -Needs the extra — `pip install "recordstream[keras]"`. Importing `recordstream.keras` is also what sets `KERAS_BACKEND` (Keras reads it at import time and would otherwise default to TensorFlow, which this extra does not install), so it must be the first keras-touching import in a process; never `import keras` ahead of it. Why the adapter lives here rather than in a training project is recorded in [architecture.md](architecture.md#10-the-frameworks-batching-half-lives-beside-the-collate-2026-07-30). +Needs the extra — `pip install "recordstream[keras]"`. Importing `recordstream.keras` is also what sets `KERAS_BACKEND` (Keras reads it at import time and would otherwise default to TensorFlow, which this extra does not install), so it must be the first keras-touching import in a process; never `import keras` ahead of it. Why the adapter lives here rather than in a training project is recorded in [architecture.md](architecture.md#10-the-frameworks-batching-half-lives-beside-the-collate-recordstreamkeras-2026-07-30). ## 1→N expanding ops (iterable-only pipelines) diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py new file mode 100644 index 0000000..b063093 --- /dev/null +++ b/tests/test_docs_links.py @@ -0,0 +1,155 @@ +"""The documentation's internal links resolve. + +A dead docs link is the kind of rot nobody notices until a reader hits one, and +it is created by exactly the ordinary edits this project does constantly: renaming +a heading, splitting a page, moving a section between guides. Nothing else checks +it — CI runs the test suite and the examples, and a broken `[text](other.md#x)` +breaks neither. + +Three rules, one per failure mode: + +* a relative link names a file that exists, +* an `#anchor` names a heading that exists in that file, +* every docs page is reachable from the README index. + +This project's README links to docs RELATIVELY, which is correct for a project not +published to PyPI — the absolute-URL rule exists because a PyPI landing page cannot +resolve a relative link. Those relative links are still covered by the two rules +above, since the README is in the scanned set. + +Fenced code blocks are excluded from the scan — see :func:`_strip_code`. + +The CI workflow is generated and must not be hand-edited, so these live as tests +— which is also where they belong: they run locally on the same command. +""" + +import re +from pathlib import Path +from typing import List, Set + +import pytest + +_REPO = Path(__file__).resolve().parent.parent +_DOCS = _REPO / "docs" +_README = _REPO / "README.md" + +#: `[text](target)` where the target is a relative path — absolute URLs are excluded +#: by requiring the target not to contain `://`. +_RELATIVE_LINK = re.compile(r"\[[^\]]*\]\((?!\w+://)([^)\s]+)\)") + + +def _slug(heading: str) -> str: + """GitHub's anchor slug for a heading: lowercase, punctuation dropped, spaces hyphenated. + + Punctuation removal is what makes `## Registering a class you don't own` reachable + as `#registering-a-class-you-dont-own` and a backticked `## \\`flow()\\` finishes...` + reachable without the backticks or parens. + """ + return re.sub(r"[^\w\s-]", "", heading.lower()).replace(" ", "-") + + +def _headings(path: Path) -> Set[str]: + return {_slug(line.lstrip("#").strip()) for line in path.read_text().splitlines() if line.startswith("#")} + + +def _markdown_files() -> List[Path]: + return sorted(_DOCS.glob("*.md")) + [_README] + + +def _strip_code(text: str) -> str: + """Blank out fenced code blocks before scanning for links. + + Python subscript-then-call — `LazyClass[Metric](SomeClass)` — is + indistinguishable from a markdown link to a regex, so a code sample containing + one gets reported as a link to a file named `SomeClass`. Found exactly that way: + the first run of this check against another project's docs produced a false + positive, which in a link checker is worse than a miss, because it trains the + reader to ignore it. + """ + out, fenced = [], False + for line in text.splitlines(): + if line.lstrip().startswith("```"): + fenced = not fenced + out.append("") + continue + out.append("" if fenced else line) + return "\n".join(out) + + +def _links(path: Path) -> List[str]: + return _RELATIVE_LINK.findall(_strip_code(path.read_text())) + + +@pytest.mark.parametrize("path", _markdown_files(), ids=lambda p: p.name) +def test_relative_links_name_a_file_that_exists(path: Path) -> None: + """A `[text](other.md)` must point at a real file. + + Catches a page that was renamed or deleted while something still pointed at it. + """ + missing = [] + for target in _links(path): + if target.startswith("#"): + continue # same-page anchor, checked below + file_part = target.split("#", 1)[0] + if not (path.parent / file_part).resolve().exists(): + missing.append(target) + + assert not missing, f"{path.name} links to files that do not exist: {missing}" + + +@pytest.mark.parametrize("path", _markdown_files(), ids=lambda p: p.name) +def test_anchors_name_a_heading_that_exists(path: Path) -> None: + """A `#anchor` must match a heading in the file it points at. + + This is the half that rots silently: renaming a heading leaves every link to it + pointing at the top of the page instead of failing, so the reader lands + somewhere plausible and never reports it. + """ + dead = [] + for target in _links(path): + if "#" not in target: + continue + file_part, _, anchor = target.partition("#") + if not anchor: + continue + target_file = path if not file_part else (path.parent / file_part).resolve() + if not target_file.exists(): + continue # reported by the test above; don't double-fail + if anchor not in _headings(target_file): + dead.append(target) + + assert not dead, f"{path.name} links to headings that do not exist: {dead}" + + +def test_every_docs_page_is_listed_in_the_readme_index() -> None: + """The README is the docs index; a page missing from it is a page nobody finds. + + `architecture.md` is exempt: it is the rationale record for maintainers, and the + README index is the user-facing table. + """ + exempt = {"architecture.md"} + listed = set(re.findall(r"docs/([a-z0-9_-]+\.md)", _README.read_text())) + present = {p.name for p in _DOCS.glob("*.md")} - exempt + + assert not (present - listed), f"docs pages missing from the README index: {sorted(present - listed)}" + + +def test_the_slug_rule_matches_githubs() -> None: + """Guard the slug helper itself — a wrong rule makes every test above vacuous. + + If `_slug` stopped stripping punctuation, every anchor would fail to match and + the suite would report dead links everywhere; if it over-stripped, nothing would + match and everything would pass. Either way the failure is in the helper the + other tests trust, so it needs assertions that do not depend on them. + + Deliberately literal rather than read from this repo's headings: the same file + is used across projects, and a fixture that names a particular page would make + it non-portable for no gain. + + The three cases are the punctuation classes that actually occur in headings — + an apostrophe, code backticks with parens, and a comma plus an em dash (which + leaves a double hyphen, matching GitHub). + """ + assert _slug("Registering a class you don't own") == "registering-a-class-you-dont-own" + assert _slug("`flow()` finishes the object") == "flow-finishes-the-object" + assert _slug("Bare, addressed, glob — the scoping model") == "bare-addressed-glob--the-scoping-model" From b9321b23845aa7c6f2f9087522dc24c3aa7863b0 Mon Sep 17 00:00:00 2001 From: gearlux Date: Thu, 6 Aug 2026 15:34:21 +0200 Subject: [PATCH 077/102] feat: selectable batch shape, the geometry-desync warning, and the restoration contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collate becomes a CHOICE rather than an accident: `collate_list` ("list") stacks nothing and keeps items as items, `RecordSequence` takes `collate=`, and every read-back helper (`batch_values` / `batch_regions` / `batch_metadata`) accepts both shapes. A stack failure now names the key, the differing shapes and the way out, where numpy's raw ValueError named none of the three. A bare albumentations transform receives only albumentations' own key vocabulary, so a `Regions` detection target sits out the call and does not move — measured, a Resize takes a 200x200 image to 64x64 and leaves the boxes where they were, and a HorizontalFlip changes no shape at all. `_apply_op` now warns, matching on the library's OWN taxonomy (a DualTransform is by definition one that applies to boxes) so there is no name list to drift, and once per transform TYPE. Every op that makes or re-frames a Regions now fills in its `canvas`, including for an empty target — the frame was previously known only where it was least needed. `image_frame` is the shared read; the lookup stays narrow so a Regions' own [N, 4] box array is never mistaken for an N x 4 raster. Plus `RestorationOutput` + `restoration_output()` — one key (`image`), because an image-to-image model emits the answer rather than something to interpret. --- AGENTS.md | 12 +- docs/augmentation.md | 69 ++++++++ docs/kinds.md | 30 +++- docs/predictions.md | 10 ++ recordstream/__init__.py | 17 +- recordstream/batch.py | 89 +++++++++- recordstream/collate.py | 98 +++++++++-- recordstream/core/families.py | 127 +++++++++++++- recordstream/items.py | 14 +- recordstream/keras.py | 24 ++- recordstream/ops/image.py | 62 ++++++- recordstream/ops/target.py | 146 +++++++++++++++- recordstream/ops/torch.py | 17 +- recordstream/outputs.py | 41 +++++ recordstream/sources/huggingface.py | 8 +- tests/test_batch.py | 150 ++++++++++++++++ tests/test_convert_to_mask.py | 112 +++++++++++- tests/test_huggingface_source.py | 27 +++ tests/test_keras_sequence.py | 43 +++++ tests/test_op_families.py | 210 ++++++++++++++++++++++- tests/test_outputs.py | 37 ++++ tests/test_typed_detection_target_ops.py | 56 ++++++ tests/test_typed_target_ops.py | 34 ++++ 23 files changed, 1386 insertions(+), 47 deletions(-) create mode 100644 tests/test_huggingface_source.py diff --git a/AGENTS.md b/AGENTS.md index af52c0a..a1ea188 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,13 +19,13 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of matrainer when matrainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the matrainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and matrainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. -- **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). +- **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — and since 2026-08-05 the COLLATE ITSELF is selectable here too (`collate=` takes a registered key or a function, resolved per batch so a key registered later still works), because a `PyDataset` otherwise had no way to say "don't stack" and the batch-shape choice was torch-only — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. - **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). **A geometry-changing transform WARNS when a `Regions` sat out the call (2026-08-06):** the vocabulary rule above is what makes a bare library transform work unmodified, but a detection target rides as a `Regions` item under a key of the pipeline's choosing — so it is not in that vocabulary, is not passed, and does not move. Measured: a bare `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes on `[10, 10, 100, 100]`, and a bare `A.HorizontalFlip` mirrors the pixels while changing NO shape at all. Nothing errors either way — the shapes stay valid and only the coordinates become wrong. **The condition is the LIBRARY'S OWN taxonomy, not a raster comparison**: a `DualTransform` is by definition one that applies to boxes, an `ImageOnlyTransform` cannot touch geometry, so `_has_spatial_transform` matches `DualTransform` by MRO class NAME (import-free, like `_is_albumentations`) and recurses a `Compose` through `.transforms` — the flip case proves why a "did the size change" test is not enough, and reading the taxonomy means no list of transform names to drift. WARNING not error (a record may legitimately carry regions describing something else), once per transform TYPE (`_WARNED_SPATIAL` — the message is about the configuration), and silent when `bboxes` WAS passed. The message names both ways out: `bbox_params` on a `Compose`, or `ops.target.ResizeDetection` for a plain coupled resize. Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (incl. `TestGeometryLeavingRegionsBehind`, which asserts the desync PREMISE before the guard). **The v2 family has the SAME gap by the other route and the same guard (2026-08-06):** v2 walks the record natively but transforms only its own `tv_tensors` TYPES, so a `Regions` is passed through untouched — measured, `v2.Resize((64,64))` takes a 200x200 image to 64x64 with the boxes still on `[10,10,100,100]`, while the same transform over a `tv_tensors.BoundingBoxes` gives `[3.2, 3.2, 32, 32]`. `_is_v2_geometry` matches v2's private `torchvision.transforms.v2._geometry` module in the MRO (the library offers no public marker) and shares `_WARNED_SPATIAL` with the albumentations guard. **The behavioural test — apply the transform to a throwaway `BoundingBoxes` and see if they move — is deliberately NOT used: it would draw from the RNG and change the augmentation stream of the run being diagnosed.** The private path can go stale on a torchvision upgrade; it FAILS OPEN (no warning, nothing else changes), so `test_the_geometry_signal_still_matches_this_torchvision` asserts the CLASSIFICATION directly rather than only through a warning that would silently stop appearing (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **OpenCV's Thread Pool Is The SECOND Fork Hazard, And The Guard Fires On USE (`core.families._disable_cv2_threading`, 2026-08-02):** the companion to the `ensure_materialized` mandate below, and **independent of it — neither fixes the other**. albumentations runs on OpenCV, whose thread pool is not fork-safe: once the PARENT has executed a cv2 op, a forked `DataLoader` worker inheriting that pool dies with a **SIGSEGV and no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` — the same symptom as the lazy-source hazard, which is exactly why they get confused for each other. Measured 2026-08-02 with `DataLoader(num_workers=2)` over a `Stream` whose ops list held an `A.Resize`: **with the source warmed and the pool ON the worker still SIGSEGVs, and with the pool off a cold source is still built in the child.** A consumer that forks needs BOTH guards. `cv2.setNumThreads(0)` is fired ONCE at the top of `_invoke_albumentations` — the one place this package INVOKES albumentations — and never at import: a process that never uses albumentations must not have its OpenCV settings changed by importing a data library, and cv2 must not become an import-time dependency of the engine. It costs nothing where it matters, because inside a worker the WORKER is the parallelism and cv2's own threads oversubscribe rather than help (albumentations' own docs recommend exactly this for multiprocessing loaders). Two cv2 quirks a test must not get wrong, both measured: `setNumThreads(0)` makes `getNumThreads()` report **1**, not 0; and `setNumThreads(4)` does not change what it reports at all. **Why the classification consumer never hit this and the segmentation one did:** classification resizes with `ConvertToImage` (PIL), while a per-pixel task must resize the image and its mask in ONE JOINT DRAW — which only a bare albumentations transform does. Segmentation is the first thing in the workspace to put cv2 on the worker path. Pins: `tests/test_fork_safety.py` (the forked-worker subprocess reproduction is the one that matters — removing the guard makes it FAIL, not pass differently). -- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. **A `Regions`' `canvas` is the load-bearing case of that rule, and EVERY op that makes or re-frames one fills it in (2026-08-06):** a box means nothing without the raster it is stated in, so `CocoToTorchVisionDetection` records the frame of the image its annotation describes, `MasksToDetectionBoxes` the shape of the mask its boxes were derived from (exact and free), and `ResizeDetection` the size it resized to — **including for an EMPTY target**, because a check that silently skips exactly the records with nothing to check reports a clean bill for the wrong reason. Previously ONLY the resize set it, so the frame was known exactly where it was least needed and unknown in the chain where boxes and pixels actually drift apart. The lookup (`ops.target._source_frame`) is deliberately NARROW — the `"image"` key, then the first `Image` item, then `None` — because a generic "first 2-D array" search reads a `Regions`' own `[N, 4]` box array as an `N x 4` raster and records a confident lie (pinned). `None` stays an ordinary answer: nothing depends on the lookup succeeding. **The desync this makes detectable is otherwise SILENT:** an image-only resize (`ConvertToImage` and friends) moves pixels without moving boxes, every downstream SHAPE stays valid, only the coordinates are wrong, and a model trains against misplaced targets reporting nothing — so `ConvertToImage` WARNS once per op instance when it resizes a record carrying a `Regions`, naming `ResizeDetection`. Once per INSTANCE, not per record: the message is about the configuration, and a copy per record only buries it. **`ops.image.image_frame(value)` is the shared "what raster is this" read** (PIL `.size` transposed, an `Image` item's DECLARED `layout`, else HWC) — it does NOT sniff a channel axis, because this package already carries three deliberately divergent channels-first heuristics and a guessing fourth would mislabel the very frame box coordinates are validated against. Consumer note: a downstream frame check that skipped on `canvas is None` now fires in chains it used to pass (verified on a real COCO-style set — the recorded frame matches the dataset's own `width`/`height` columns). Pins: `tests/test_convert_to_mask.py::TestBoxesKnowTheirFrame` (incl. the box-array-is-not-a-raster case and the once-per-op warning). NOTE for any test asserting on that warning: loggair is loguru, so `caplog` stays EMPTY, and its sink is ENQUEUED so `capfd` alone races it — add a sink and call `logger.complete()` (the mandated flush, never a sleep). Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. - **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.execute.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). - **1→N Expanding Steps Fork the REMAINING Subgraph (2026-07-30, supersedes the flat-engine pending-queue rule):** An op carrying `EXPANDS = True` yields N children from one record; the remaining steps then run ONCE PER CHILD over that child's own shallow copy of the step environment (independent name→result maps, shared values), DEPTH-FIRST so sibling order matches the nested-loop intuition. An empty expansion or a `None` child drops that branch. This works in EVERY route — serial, spawn-parallel (the worker returns a LIST), and inside a `flow:` graph (the old `FlowGraph` raised `NotImplementedError` on an expanding step; that limit is gone). CONSEQUENCES: (1) `__len__`/`__getitem__` RAISE on `Stream` AND `FlowGraph` when any step op expands — the expanded index map is unknowable up front, so the pipeline is ITERABLE-ONLY (iterate, wrap in a torch IterableDataset, window at the SOURCE for random access, or `list(...)`); (2) `run_steps` (the strict 1→1 twin used for indexing) raises rather than silently dropping siblings. Pins: `tests/test_typed_flow.py::TestExpandingSteps`. @@ -33,7 +33,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core.families._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry (`recordstream.collate`):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)` — whose default key is **`"record"`** = `collate_records`: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. +- **Collation Is a Pluggable Registry, And The BATCH SHAPE Is A CHOICE (`recordstream.collate`; the choice landed 2026-08-05):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`. **TWO collates ship, differing in ONE decision — whether array payloads are STACKED — because that decision belongs to the MODEL, not to the data:** `"record"` (the default, `collate_records`) stacks what can stack; `"list"` (`collate_list` = `collate_records(items, stack=False)`) stacks nothing and leaves every key a per-record list with ITEMS KEPT AS ITEMS (so an `Image`'s `layout` survives per record). Left implicit the shape is decided by ACCIDENT — a column stacks if it holds array items and stays a list if it holds PLAIN values, so `ToTensor` running or not silently decides whether a detector gets its `List[Tensor]`; the key lets a consumer DECLARE it (raidar's torch loaders pass `collate_fn=collate_list`, `RecordSequence` takes `collate=`). **Every read-back helper accepts BOTH shapes** (`batch_values` unwraps a list-of-items element-wise, `batch_regions` reads a batched `Regions` or a list of them, `batch_metadata` transposes either) — that is what makes the collate a free choice rather than a fork in every consumer, and it is pinned by a parametrized test over both keys. **A stack failure now EXPLAINS itself** (`_stack_or_explain`): it names the key, the differing shapes and the `"list"` way out, where the raw `ValueError: all input arrays must have the same shape` from inside numpy named none of the three — this closed the old TASKS item about `_stack`'s unreachable "else a list" promise, by making the fallback a DECLARED mode rather than a silent type change. **`register_collate` is signature-PRESERVING** (`TypeVar` bound to `CollateFn`, not a flat `-> CollateFn`), so registering a collate no longer erases its own parameters — that is what lets `collate_list` call `collate_records(items, stack=False)` and type-check. The default `collate_records` behaviour is unchanged: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_regions(batch, key)` (a collated `Regions` column transposed into per-record `{boxes, labels}` dicts — see the detection note below), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. **A DETECTION batch needs NO consumer collate, and that was measured (2026-08-05):** `collate_records` already batches one correctly — a variable-size image column stays a per-image LIST (`ToTensor` emits a live tensor as a PLAIN value, and plain values are gathered, never stacked) and a variable-N `Regions` keeps per-record COLUMNS (its `boxes`/`labels` are declared attrs, which the collate lists rather than stacking). So the only piece that was missing is the inverse, `batch_regions`, which is why it lives here: a detection consumer was carrying ~90 lines of its own collate (a `DetectionBatchInput` container, a `Regions`->`{boxes,labels}` unwrap, a metadata transpose) that re-derived exactly these rules, and it was deleted in favour of `collate_records` + `batch_regions` + `batch_metadata`. `batch_regions` is FRAMEWORK-FREE like its neighbours (torch stays torch, numpy stays numpy — a target's dtype/device is the caller's contract), omits a field the item left `None` (so a training target is exactly `{boxes, labels}`), and leaves `canvas`/`extras` on the batched item (per-image frame metadata and an open dict are not per-box columns). **A variable-size `Image` ITEM column is batched by the `"list"` collate** — the default still raises for it, deliberately and with an explanatory message, because a caller who asked for stacking should hear that it could not happen rather than silently receive a different type. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. @@ -42,7 +42,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. **`first_value(source, key)` is the ONE-PEEK primitive beside them (2026-08-02)** — the first non-`None` value under `key`, or `None` when there is none. It answers what a column's values ARE without walking the set, and it belongs here rather than in any consumer because it is `iter_key` plus a `next()`: it inherits all three of that helper's properties (a projection-aware source never builds the values it does not ask for, a deferred source is materialized first, the walk is lazy so a normal source costs ONE record) and its unwrapping rules are what make the answer meaningful — a `MultiLabel` arrives as its `.values` LIST, so a sequence IS a multi-label column, decided by the item type rather than by guessing what a list might mean. The canonical call site pairs it with `is_class_id` to decide whether the targets need a `LabelMap` at all. It was extracted from EIGHT byte-identical private copies in one consumer's training backends (2026-08-02); a consumer re-deriving it is re-deriving `iter_key`'s contract. Pins: `tests/test_projection.py`. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. - **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in matrainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file matrainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). -- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from matrainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` / `RestorationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` / `restoration_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. **`RestorationOutput` carries ONE key (`image`) and that is the contract working, not a stub (2026-08-05):** the other three tasks emit something that needs interpreting, so their keys separate `logits` from `probs` from `class_idx`; an image-to-image model emits the answer, so what a consumer must be told is precisely that the array is a picture in the INPUT's value range — not logits to softmax, not a residual to add back. Its builder therefore transforms NOTHING and exists only so every backend spells the wrapping identically. Do NOT add a `residual` key (it is `input - image`, i.e. a second place for the two to disagree) and do NOT clamp inside the builder: a residual denoiser can legitimately overshoot `[0, 1]`, and a metric computed on clamped values is a different number, so that is the run's decision. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from matrainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. @@ -52,7 +52,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the two detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` (`recordstream.ops.target` — both emit a `Regions` detection target, lazy-importing torch: the first from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the second from a segmentation MASK)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` / `ResizeDetection` (`recordstream.ops.target` — the first two emit a `Regions` detection target, lazy-importing torch: one from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the other from a segmentation MASK; `ResizeDetection` is the COUPLED image+boxes resize for fixed-input-size detectors — PIL/uint8 image to `(height, width)` + the `Regions` boxes scaled by the same factors, torch staying torch, `canvas` updated — run it BEFORE any float conversion such as `ToTensor`, and omit it for detectors that resize internally)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. - **Generic MASK Conversion Lives Here Too — `ConvertToMask` (2026-08-02):** the segmentation counterpart of `ConvertToImage` and the same op SHAPE (read one field, write a differently-typed item under `output`): a mask-bearing field (an ndarray, a torch tensor, or the PIL image a source handed over) becomes an **`int64` `[H, W]` `Mask`** of per-pixel class ids — what a segmentation dataset actually ships (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map) turned into what a per-pixel loss consumes. It belongs HERE, not in a segmentation project: "a mask PNG's pixels are class ids" mentions no modality (the `Threshold` → `Mask` precedent), and a consumer owning it would be the third package to write the conversion. **It converts and NOTHING else, deliberately** — remapping the ids is `FormulaOp` over its output (`formula: a - 1` for a 1-based trimap) or `EncodeTarget` for a lookup table; resizing/augmenting it TOGETHER WITH THE IMAGE is a bare albumentations transform in the same ops list; dropping the source column is `DropField`. Do NOT grow it an `offset` / `mapping` / `dtype` knob: each one restates an op that already exists. **`output` defaults to `"mask"` and that is load-bearing, not a nicety** — it is albumentations' own key vocabulary (`_ALB_KEYS`), so the engine's op-family dispatch hands `image` AND `mask` to ONE call and a single joint draw moves both with the `Mask` type surviving the re-wrap (measured; an image-only transform like `Normalize` still touches the image alone). **`int64` is not a knob either:** a class-id map is integer by definition and it is what `torch.nn.CrossEntropyLoss` requires (*"expected target dtype to be Long or Byte, but got Int"*); a library that casts on the way past — albumentations returns int32 — is corrected at the MODEL boundary by `batch_tensor(..., dtype=...)`, where the caller names the contract (the `dtype`-is-a-parameter rule). It reads through **`item_value`, never `item_data`**, because a source that does not know a column is a mask ships it as a `Label` (`HuggingFaceSource` does this for every metadata column) — see the record-model mandate. Singleton axes are squeezed (`[H,W,1]` / `[1,H,W]` → `[H,W]`); an **RGB-encoded mask RAISES** rather than being collapsed, because picking one of three channels or decoding a palette is a decision the op must not make silently. Pins: `tests/test_convert_to_mask.py` (incl. the whole `preprocess` chain end to end, and that a `Normalize` leaves the mask untouched). Usage: `docs/image.md` → "Masks". - **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). diff --git a/docs/augmentation.md b/docs/augmentation.md index 7901150..82a42e1 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -74,6 +74,75 @@ engine adds nothing on top. The detection-target ops (`CocoToTorchVisionDetectio `MasksToDetectionBoxes`) produce a `Regions` item for the training boundary; the plain `bboxes`/`labels` list keys are the augmentation-time form the library consumes. +For a plain deterministic resize of the `Regions` form there is `ResizeDetection` +(`recordstream.ops.target`) — the detection twin of the joint image+mask draw: it resizes the +image (PIL or uint8 array) to a fixed `(height, width)` AND scales the `Regions` boxes by the +same factors in one coupled step, recording the new frame in `canvas`. Fixed-input-size +detectors need it; detectors that resize internally simply omit it. Run it BEFORE any float +conversion (e.g. before `ToTensor`): + +```yaml +ops: + - !class:recordstream.ops.target.CocoToTorchVisionDetection { bbox_format: xywh, label_offset: 1 } + - !class:recordstream.ops.target.ResizeDetection { width: 256, height: 256 } + - !class:recordstream.ops.torch.ToTensor { mode: RGB, normalize: true } +``` + +### Boxes carry the frame they are stated in + +A box is only meaningful against a raster, so a `Regions` records that raster in `canvas` — +`(H, W)` — and every op that makes or re-frames one fills it in: `CocoToTorchVisionDetection` +from the image the annotation describes, `MasksToDetectionBoxes` from the mask the boxes were +derived from, `ResizeDetection` from the size it resized to (including for an empty target, so a +negative example is not the one record whose frame is unknown). It stays `None` only when no +image is in the record to read. + +That makes a desync *detectable*, which matters because it is otherwise silent. An **image-only** +resize moves pixels without moving boxes: + +```yaml +ops: + - !class:recordstream.ops.target.CocoToTorchVisionDetection { bbox_format: xywh } + - !class:recordstream.ops.image.ConvertToImage { width: 256, height: 256 } # ← boxes left behind +``` + +Every shape downstream stays valid — only the coordinates are wrong — so a model trains happily +against misplaced targets. `ConvertToImage` warns once per op when it resizes a record carrying a +`Regions`, and a consumer that must be certain compares `canvas` against the image itself +(`recordstream.ops.image.image_frame` reads the `(H, W)` of either). Use `ResizeDetection`, which +moves both. + +**A bare library transform has the same gap**, for the reason that makes the dispatch work: an +albumentations op receives exactly its own key vocabulary, and a `Regions` is not in it. So a bare +`A.Resize` resizes the image and leaves the boxes; a bare `A.HorizontalFlip` mirrors the pixels +and leaves them — *without changing any shape at all*. The engine warns once per transform type +when a geometry-changing transform runs while a `Regions` sat out the call, deciding "geometry- +changing" by the library's own `DualTransform` / `ImageOnlyTransform` split (so `Normalize` and +friends stay silent). Speak the library's vocabulary and it moves them for you, in the same draw: + +```yaml +ops: + - !class:recordstream.ops.structure.RenameField { src: my_boxes, dst: bboxes } + - !class:albumentations.Compose + transforms: [!class:albumentations.HorizontalFlip { p: 0.5 }] + bbox_params: !class:albumentations.BboxParams { format: pascal_voc, label_fields: [labels] } +``` + +**torchvision v2 has it too, by the other route.** v2 walks the record natively but transforms +only its OWN `tv_tensors` types, and a `Regions` is not one — so `v2.Resize` moves the pixels and +leaves the boxes, while the same transform over a `tv_tensors.BoundingBoxes` rescales them +correctly. The engine warns once per transform type here as well, using v2's geometric-transform +grouping so `ColorJitter` and `Normalize` stay silent. Carry boxes in v2's own type when you want +v2 to move them: + +```python +from torchvision import tv_tensors + +record["boxes"] = tv_tensors.BoundingBoxes(boxes, format="XYXY", canvas_size=(h, w)) +``` + +Either way, `ResizeDetection` remains the plain coupled resize over the `Regions` form. + ## YAML — bare library transforms are ordinary `!class:` nodes No library-specific serialization format — a transform is a Confluid `!class:` node like any op, diff --git a/docs/kinds.md b/docs/kinds.md index 21dcd7d..4d3d793 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -51,10 +51,36 @@ batch = collate_records(list(stream)) # ONE batched record: payloads st loader = DataLoader(stream, collate_fn=collate_records) ``` -Collation is a pluggable registry keyed by name, so a task can register its own convention additively: +### The batch SHAPE is a choice — `"record"` vs `"list"` + +Whether a column is STACKED is a requirement of the **model**, not a property of the data: a +torchvision detector takes `List[Tensor]` (its images differ in size), a classifier takes one +`[N, C, H, W]` tensor. Two collates ship, differing in exactly that: + +```python +from recordstream import collate_list, collate_records + +collate_records(records)["image"] # Image (2, 3, 8, 8) — stacked +collate_list(records)["image"] # [Image (3, 8, 8), Image (3, 12, 12)] — per record, items kept +``` + +Declare it where the batch is built — `DataLoader(stream, collate_fn=collate_list)` on torch, +`RecordSequence(source, collate="list")` on Keras. Left implicit, the shape is decided by +accident: under the default collate a column stacks if it holds array *items* and stays a list if +it holds *plain* values, so whether an op like `ToTensor` ran ends up choosing for you. + +Two things make the choice free: + +* **every read-back helper accepts both shapes** — `batch_values` unwraps a list-of-items + element-wise, `batch_regions` reads a batched `Regions` or a list of them, `batch_metadata` + transposes either — so a consumer never branches on which collate ran; +* **a stack failure explains itself**, naming the key, the differing shapes and the `"list"` way + out, rather than surfacing numpy's bare *"all input arrays must have the same shape"*. + +The registry is additive, so a task can register its own convention too: ```python -from recordstream import register_collate, get_collate +from recordstream import get_collate, register_collate @register_collate("yolo") # task aliases are additive def yolo_collate(items): ... diff --git a/docs/predictions.md b/docs/predictions.md index 38c229b..fd3ea26 100644 --- a/docs/predictions.md +++ b/docs/predictions.md @@ -33,10 +33,20 @@ argmax'd class ids: | `ClassificationOutput` | `logits` `[B, C]`, `probs` `[B, C]`, `class_idx` `[B]` | | `DetectionOutput` | `boxes` `[N, 4]` xyxy absolute pixels, `scores` `[N]`, `labels` `[N]` | | `SegmentationOutput` | `logits` `[B, C, H, W]`, `probs` `[B, C, H, W]`, `mask` `[B, H, W]` | +| `RestorationOutput` | `image` `[B, C, H, W]` — the restored image, in the input's value range | That guess is not hypothetical: two independently-written detector wrappers agree that `boxes` is xyxy in absolute pixels only because `DetectionOutput` says so. +`RestorationOutput` carries **one** key while the others carry three, and that asymmetry is the +contract doing its job rather than an unfinished row. The other tasks emit something that needs +interpreting — `logits` are not `probs` are not `class_idx` — whereas an image-to-image model emits +the answer directly, so the thing a consumer needs told is exactly that: the array under `image` is +a picture in the input's units, not logits to be softmaxed and not a residual to be added back. +`restoration_output()` therefore transforms nothing; it exists so every backend spells the wrapping +identically. The residual is deliberately absent — it is `input - image`, and a key a consumer can +compute is a second place for the two to disagree. + Each contract is **generic in the array type**, so the same declaration describes a torch run and a numpy/TF/JAX one: diff --git a/recordstream/__init__.py b/recordstream/__init__.py index e6364d5..6a2d94c 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -10,8 +10,15 @@ """ # --- shared infrastructure ----------------------------------------------------------------- -from recordstream.batch import batch_metadata, batch_tensor, batch_values, multi_hot -from recordstream.collate import collate, collate_records, get_collate, register_collate, registered_collates +from recordstream.batch import batch_metadata, batch_regions, batch_tensor, batch_values, multi_hot +from recordstream.collate import ( + collate, + collate_list, + collate_records, + get_collate, + register_collate, + registered_collates, +) from recordstream.core import ( FilterOp, JointStream, @@ -59,8 +66,10 @@ ClassificationOutput, DetectionOutput, DetectionPredictions, + RestorationOutput, SegmentationOutput, classification_output, + restoration_output, segmentation_output, ) from recordstream.predictions import ClassificationPredictionsSink, PredictionsSink @@ -133,9 +142,11 @@ "FlowGraph", "collate", "batch_metadata", + "batch_regions", "batch_tensor", "batch_values", "multi_hot", + "collate_list", "collate_records", "get_collate", "register_collate", @@ -147,8 +158,10 @@ "ClassificationOutput", "DetectionOutput", "DetectionPredictions", + "RestorationOutput", "SegmentationOutput", "classification_output", + "restoration_output", "segmentation_output", "PredictionsSink", "ClassificationPredictionsSink", diff --git a/recordstream/batch.py b/recordstream/batch.py index 3b466c8..6b26eca 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -12,6 +12,8 @@ * :func:`batch_values` — the raw values, past the wrapper item. Framework-free. * :func:`multi_hot` — a :class:`~recordstream.MultiLabel` column as an ``[N, C]`` matrix. Framework-free (numpy). +* :func:`batch_regions` — a :class:`~recordstream.Regions` column as per-record + ``{boxes, labels}`` dicts. Framework-free. * :func:`batch_tensor` — the torch adapter: stack, optional dtype, optional device. * :func:`batch_metadata` — the collate's transpose, for prediction sinks. Framework-free. @@ -33,12 +35,18 @@ import numpy as np -from recordstream.items import Record, item_value +from recordstream.items import Record, Regions, is_item, item_value if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only from torch import Tensor -__all__ = ["batch_metadata", "batch_tensor", "batch_values", "multi_hot"] +__all__ = ["batch_metadata", "batch_regions", "batch_tensor", "batch_values", "multi_hot"] + +#: The per-box PARALLEL ARRAY fields of a :class:`~recordstream.Regions`, in the order a +#: per-record dict presents them. ``canvas`` and ``extras`` are deliberately absent: the first is +#: per-IMAGE frame metadata and the second an open dict, neither of which is a per-box column — +#: read them off the batched item itself (``batch[key].canvas`` is the per-record list). +_REGION_FIELDS = ("boxes", "labels", "scores") def batch_values(batch: Record, key: str) -> Any: @@ -54,12 +62,21 @@ def batch_values(batch: Record, key: str) -> Any: No torch, no stacking, no dtype opinion — just the values. Use :func:`batch_tensor` when a tensor is what you need. + **Both collates read the same here.** Under ``"record"`` a wrapper item's column arrives as + ONE batched item; under ``"list"`` (:func:`~recordstream.collate_list`) it arrives as a LIST + of per-record items. The second shape is unwrapped ELEMENT-WISE, so a caller gets + ``[0, 1]`` either way and never branches on which collate ran — the property that makes the + collate a free choice rather than a fork in every consumer. + Example:: batch_values(collate_records([{"class": Label(0)}, {"class": Label(1)}]), "class") # [0, 1] """ - return item_value(batch[key]) + value = batch[key] + if isinstance(value, list) and any(is_item(entry) for entry in value): + return [item_value(entry) for entry in value] + return item_value(value) def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") -> np.ndarray: @@ -105,6 +122,72 @@ def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") return out +def batch_regions(batch: Record, key: str) -> List[Dict[str, Any]]: + """A collated :class:`~recordstream.Regions` column back into PER-RECORD dicts. + + The collate cannot stack a region set — every record has its own N — so it leaves each + declared attr as a per-record LIST (``boxes`` = ``[[N0, 4], [N1, 4], …]``). That is the + right batch, and it is also not what a model takes: every detection interface in use wants + ONE dict per image. This is that transpose, and it belongs beside :func:`batch_metadata` + (which transposes the same way for the remaining columns) rather than in whichever consumer + needed it first — a consumer re-deriving it is re-deriving the collate. + + **Framework-free, deliberately.** The values are handed back EXACTLY as the record carried + them — torch stays torch, numpy stays numpy — because a detection target's dtype and device + are the caller's contract, not this module's (the same rule that keeps + :func:`batch_values` framework-free and confines torch to :func:`batch_tensor`). A torch + backend moves the dicts to its device in one comprehension; a numpy one uses them as they + are. + + Args: + batch: A batched record — from EITHER collate (``"record"`` leaves one batched + :class:`~recordstream.Regions` with per-record columns; ``"list"`` leaves a list of + per-record ``Regions``; both are read here). + key: The record key holding the collated :class:`~recordstream.Regions`. + + Returns: + One dict per record, carrying whichever of ``boxes`` / ``labels`` / ``scores`` that + record actually has — a field left ``None`` on the item is OMITTED rather than handed + over as ``None``, so a prediction-free training target is exactly ``{boxes, labels}``. + ``canvas`` and ``extras`` stay on the batched item (per-image frame metadata and an + open dict are not per-box columns); read them off ``batch[key]``. + + Raises: + TypeError: when ``key`` does not hold a :class:`~recordstream.Regions`. + ValueError: when the item is not COLLATED (its ``boxes`` is not a per-record list) — + passing a single record's ``Regions`` here is the mistake the message names. + + Example:: + + targets = batch_regions(batch, "target") # [{"boxes": [N0, 4], "labels": [N0]}, …] + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] # a torch caller + """ + item = batch[key] + # The "list" collate leaves a LIST of per-record Regions; the default leaves ONE batched + # Regions whose attrs are per-record lists. Both mean the same thing, so both read the same + # — a consumer never branches on which collate ran. + if isinstance(item, list): + if not all(isinstance(entry, Regions) for entry in item): + raise TypeError(f"batch_regions: {key!r} holds a list whose entries are not all Regions.") + return [ + {name: getattr(entry, name) for name in _REGION_FIELDS if getattr(entry, name, None) is not None} + for entry in item + ] + if not isinstance(item, Regions): + raise TypeError(f"batch_regions: {key!r} holds {type(item).__name__}, not a Regions.") + if not isinstance(item.boxes, list): + raise ValueError( + f"batch_regions: {key!r} is not a COLLATED Regions — its `boxes` is " + f"{type(item.boxes).__name__}, not the per-record list collate_records leaves. " + "Pass the batched record, not a single record's Regions." + ) + columns = {name: getattr(item, name) for name in _REGION_FIELDS if isinstance(getattr(item, name, None), list)} + return [ + {name: values[index] for name, values in columns.items() if values[index] is not None} + for index in range(len(item.boxes)) + ] + + def batch_tensor(batch: Record, key: str, device: Any = None, dtype: Any = None) -> "Tensor": """The batched values under ``key`` as ONE torch tensor. diff --git a/recordstream/collate.py b/recordstream/collate.py index e8c2697..72b0682 100644 --- a/recordstream/collate.py +++ b/recordstream/collate.py @@ -11,13 +11,18 @@ :func:`registered_collates`; in Python (and in YAML via a dotted ``!ref:`` to the function), passing a collate function directly remains the normal path. -The default registered here is ``"record"`` — N plain record dicts collated into ONE -batched record (array payloads stacked per key, per-record item attrs as lists, plain -values gathered into lists). Consumer conventions are deliberately NOT unified here; the -registry is additive. +TWO collates are registered here, and they differ in ONE decision — whether array payloads +are STACKED — because that decision belongs to the model, not to the data: + +* ``"record"`` (the default) — array payloads stacked per key, per-record item attrs as + lists, plain values gathered into lists. What a classifier or a dense-target detector wants. +* ``"list"`` — nothing stacked; every key becomes a per-record list, items kept as items. + What a model taking variable-size inputs wants (a torchvision detector's ``List[Tensor]``). + +Consumer conventions beyond that are deliberately NOT unified here; the registry is additive. """ -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar from loggair import get_logger @@ -28,12 +33,26 @@ CollateFn = Callable[[Sequence[Any]], Any] +#: Bound to `CollateFn` but PRESERVED through the decorator, so registering a collate does +#: not erase its own signature — `collate_records`' `stack=` stays visible to callers and +#: to a type checker (a plain `-> CollateFn` return flattened every registered collate to +#: the loose one-argument protocol). +F = TypeVar("F", bound=CollateFn) + _REGISTRY: Dict[str, CollateFn] = {} -__all__ = ["CollateFn", "collate", "collate_records", "get_collate", "register_collate", "registered_collates"] +__all__ = [ + "CollateFn", + "collate", + "collate_list", + "collate_records", + "get_collate", + "register_collate", + "registered_collates", +] -def register_collate(key: str) -> Callable[[CollateFn], CollateFn]: +def register_collate(key: str) -> Callable[[F], F]: """Register a collate function under ``key`` (a task alias). Usable as a decorator:: @@ -45,7 +64,7 @@ def yolo_collate(items): ... replace a default). """ - def _register(fn: CollateFn) -> CollateFn: + def _register(fn: F) -> F: if key in _REGISTRY: logger.debug(f"collate registry: overwriting existing collate for key {key!r}") _REGISTRY[key] = fn @@ -102,7 +121,7 @@ def _stack(values: List[Any]) -> Any: @register_collate("record") -def collate_records(items: Sequence[Record]) -> Record: +def collate_records(items: Sequence[Record], stack: bool = True) -> Record: """The record collate: N record dicts → ONE batched record dict. Per key (union of keys is NOT taken — every record must carry the same keys, a @@ -111,6 +130,12 @@ def collate_records(items: Sequence[Record]) -> Record: array, else a list) and each declared item attr becomes a LIST of per-record values, decoding back into ONE batched item of the same type. A ``"plain"``-tagged value (a scalar / string / bare value) batches as the plain LIST of per-record values. + + Args: + items: The records to batch. + stack: Whether array payloads are STACKED into one array/tensor. ``False`` is the + registered ``"list"`` collate (:func:`collate_list`) — see it for why the choice + belongs to the caller rather than to the data. """ if not items: raise ValueError("collate_records: cannot collate an empty batch") @@ -126,12 +151,65 @@ def collate_records(items: Sequence[Record]) -> Record: ) batched: Record = {} for key in keys: + if not stack: + # The values VERBATIM, one per record — items stay items, so per-record metadata + # (an `Image`'s layout) survives instead of being flattened into one batched item. + batched[key] = [record[key] for record in items] + continue encoded = [encode_item(record[key]) for record in items] type_name = encoded[0].type_name if type_name == PLAIN_TYPE: batched[key] = [e.payload for e in encoded] continue - stacked_payload = _stack([e.payload for e in encoded]) if encoded[0].payload is not None else None + payloads = [e.payload for e in encoded] + stacked_payload = _stack_or_explain(payloads, key) if encoded[0].payload is not None else None batched_attrs = {name: [e.attrs.get(name) for e in encoded] for name in encoded[0].attrs} batched[key] = decode_item(EncodedItem(type_name=type_name, payload=stacked_payload, attrs=batched_attrs)) return batched + + +@register_collate("list") +def collate_list(items: Sequence[Record]) -> Record: + """The UNSTACKED collate: N record dicts → one record whose every key is a per-record LIST. + + The sibling of :func:`collate_records`, and the reason the registry exists: **the batch + SHAPE is a model requirement, not a property of the data.** A torchvision detector takes + ``List[Tensor]`` because its images differ in size; a classifier takes one ``[N, C, H, W]`` + tensor. Left implicit, that choice is made by accident — under the default collate a column + stacks if it holds array ITEMS and stays a list if it holds PLAIN values, so the shape ends + up decided by whether an op like ``ToTensor`` happened to run. This key lets the consumer + DECLARE it instead. + + Two consequences worth knowing: + + * a variable-size column is fine here (nothing is stacked), where the default collate + raises — and the type does not depend on the data, because you asked for lists; + * items stay ITEMS (a list of :class:`~recordstream.Image`, not a list of bare arrays), so + per-record metadata survives. The read-back helpers (:func:`~recordstream.batch_values`, + :func:`~recordstream.batch_regions`, :func:`~recordstream.batch_metadata`) accept BOTH + shapes, so a consumer reads the batch the same way under either collate. + + Example:: + + DataLoader(stream, collate_fn=collate_list) # or: collate(items, key="list") + """ + return collate_records(items, stack=False) + + +def _stack_or_explain(payloads: List[Any], key: str) -> Any: + """:func:`_stack`, but a stacking failure names the KEY, the shapes and the way out. + + Raw, the failure is ``ValueError: all input arrays must have the same shape`` from inside + numpy — which says nothing about which column, what shapes, or what to do. A variable-size + column is a legitimate batch; it just is not a STACKED one. + """ + try: + return _stack(payloads) + except (ValueError, RuntimeError) as exc: + shapes = [getattr(p, "shape", None) for p in payloads] + raise ValueError( + f"collate_records: cannot stack the {key!r} column — its payloads have differing " + f"shapes {shapes}. Either make them uniform upstream (a resize op), or batch with " + f'the "list" collate (`collate_list` / `collate(items, key="list")`), which keeps ' + f"every column as a per-record list." + ) from exc diff --git a/recordstream/core/families.py b/recordstream/core/families.py index edb3298..51e6481 100644 --- a/recordstream/core/families.py +++ b/recordstream/core/families.py @@ -7,11 +7,11 @@ is the same question as "how do I apply this op" — the graph kernel asks all three together. """ -from typing import Any, Callable, List, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, cast from loggair import get_logger -from recordstream.items import NDArrayItem, Record, with_data +from recordstream.items import NDArrayItem, Record, Regions, with_data logger = get_logger(__name__) @@ -148,6 +148,73 @@ def _disable_cv2_threading() -> None: logger.debug(f"could not disable OpenCV threading ({exc}); a forked DataLoader worker may crash.") +#: Transform classes already warned about (see :func:`_warn_if_regions_are_left_behind`). Keyed +#: by CLASS, not instance: the message describes a configuration pattern, and two `Resize`s in +#: one chain have the same thing wrong with them. +_WARNED_SPATIAL: Set[type] = set() + + +def _has_spatial_transform(op: Any, depth: int = 0) -> bool: + """True when ``op`` contains a transform that would MOVE boxes, per albumentations' own taxonomy. + + The library already draws this line: a ``DualTransform`` is defined as one that applies to + boxes and masks as well as the image (``Resize``, ``HorizontalFlip``, ``RandomCrop``), while + an ``ImageOnlyTransform`` cannot touch geometry (``Normalize``, ``ColorJitter``). Reading THAT + distinction is why this needs no list of transform names to drift out of date, and no guess + about what a given transform does. + + Matched by MRO class NAME rather than `isinstance`, for the same reason + :func:`_is_albumentations` matches by module name: recognising a library must never import + one. A ``Compose`` is recursed through its ``transforms``, depth-capped against a cycle. + """ + if any(cls.__name__ == "DualTransform" for cls in type(op).__mro__): + return True + if depth >= 4: + return False + children = getattr(op, "transforms", None) + if not children: + return False + return any(_has_spatial_transform(child, depth + 1) for child in children) + + +def _warn_if_regions_are_left_behind(record: Record, op: Any, passed: Dict[str, Any]) -> None: + """Warn once when a geometry-changing transform ran while a ``Regions`` sat out the call. + + This family passes the op EXACTLY the keys of albumentations' own vocabulary, which is what + lets a bare library transform work unmodified — but a detection target rides as a ``Regions`` + item under a key of the pipeline's choosing, so it is not in that vocabulary and does not get + passed. Measured: a bare ``A.Resize`` moves a 200x200 image to 64x64 and leaves the boxes on + ``[10, 10, 100, 100]``; a bare ``A.HorizontalFlip`` mirrors the pixels and leaves the boxes + where they were WITHOUT changing the raster at all — which is why the condition here is the + library's spatial/photometric taxonomy and not "did the image size change". + + Nothing errors either way: the shapes stay valid and only the coordinates become wrong, so a + model trains against misplaced targets and reports nothing. It stays a WARNING rather than an + error because a record may legitimately carry regions describing something other than the + image being augmented — this family cannot know, and refusing the call would break a pipeline + that is right. + + The fix is to speak the library's vocabulary: put boxes under ``bboxes`` with their + ``labels`` and declare ``bbox_params`` on the ``Compose``, and the library moves them in the + same joint draw. For a plain deterministic resize, ``ops.target.ResizeDetection`` does the + coupled step over the ``Regions`` form directly. + """ + if "bboxes" in passed or type(op) in _WARNED_SPATIAL: + return + keys = [key for key, value in record.items() if isinstance(value, Regions)] + if not keys or not _has_spatial_transform(op): + return + _WARNED_SPATIAL.add(type(op)) + logger.warning( + f"albumentations {type(op).__name__} changes GEOMETRY, but this record's detection " + f"boxes ({keys}) ride as a Regions item, which is not in the library's key vocabulary " + f"({', '.join(_ALB_KEYS)}) — so the pixels moved and the boxes did not. Put boxes under " + f"'bboxes' + 'labels' with A.Compose(..., bbox_params=A.BboxParams(...)) so the library " + f"moves them in the same draw, or use recordstream.ops.target.ResizeDetection for a " + f"plain coupled resize. (Warned once per transform type.)" + ) + + def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: """albumentations dispatches by KWARG NAME: hand the op exactly its own target keys present in the record (one call = one joint draw across them); array outputs are @@ -163,6 +230,7 @@ def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." ) return record + _warn_if_regions_are_left_behind(record, op, kwargs) out = op(**kwargs) merged = dict(record) for key, value in out.items(): @@ -178,12 +246,65 @@ def _is_torchvision_v2(op: Any) -> bool: return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(op).__mro__) +#: torchvision v2's geometric transforms all live in ONE private module — the closest thing the +#: library has to albumentations' `DualTransform` marker. Private, so this can go stale across a +#: torchvision release; it FAILS OPEN (no warning, nothing else changes), which is the right +#: direction for a diagnostic. +_V2_GEOMETRY_MODULE = "torchvision.transforms.v2._geometry" + + +def _is_v2_geometry(op: Any, depth: int = 0) -> bool: + """True when a v2 transform (or one nested in a ``Compose``) changes GEOMETRY. + + The behavioural test — apply it to a throwaway ``BoundingBoxes`` and see whether they move — + would be authoritative and is deliberately NOT used: running a transform speculatively draws + from the RNG, which would change the augmentation stream of the run being diagnosed. A + diagnostic must not alter what it observes. + """ + if any(getattr(cls, "__module__", "") == _V2_GEOMETRY_MODULE for cls in type(op).__mro__): + return True + if depth >= 4: + return False + children = getattr(op, "transforms", None) + if not children: + return False + return any(_is_v2_geometry(child, depth + 1) for child in children) + + def _invoke_torchvision_v2(record: Record, op: Any) -> Optional[Record]: """torchvision v2 natively walks a dict: params sampled once, tensor/tv_tensor/PIL - leaves transformed, everything else passed through — called as-is.""" + leaves transformed, everything else passed through — called as-is. + + "Everything else passed through" is where detection boxes fall: v2 recognises its OWN + ``tv_tensors`` types, and a :class:`~recordstream.Regions` is not one, so a geometric + transform moves the pixels and leaves the boxes — the same silent desync the albumentations + family has, reached by a different route (there the boxes are not in the key vocabulary; here + they are not in the TYPE vocabulary). Measured: ``v2.Resize((64, 64))`` takes a 200x200 image + to 64x64 with the boxes still on ``[10, 10, 100, 100]``, while the same transform over a + ``tv_tensors.BoundingBoxes`` correctly rescales them to ``[3.2, 3.2, 32, 32]``. + """ + _warn_if_v2_leaves_regions_behind(record, op) return cast(Record, op(record)) +def _warn_if_v2_leaves_regions_behind(record: Record, op: Any) -> None: + """The v2 twin of :func:`_warn_if_regions_are_left_behind` — once per transform type.""" + if type(op) in _WARNED_SPATIAL: + return + keys = [key for key, value in record.items() if isinstance(value, Regions)] + if not keys or not _is_v2_geometry(op): + return + _WARNED_SPATIAL.add(type(op)) + logger.warning( + f"torchvision v2 {type(op).__name__} changes GEOMETRY, but this record's detection boxes " + f"({keys}) ride as a Regions item, which is not one of v2's tv_tensors types — so v2 " + f"passes them through untouched while the pixels move. Carry boxes as " + f"torchvision.tv_tensors.BoundingBoxes(..., format=…, canvas_size=…) so v2 transforms " + f"them in the same call, or use recordstream.ops.target.ResizeDetection for a plain " + f"coupled resize. (Warned once per transform type.)" + ) + + # The built-in families register through the SAME open registry third parties use — # one mechanism, no privileged code path. Registered at import, so spawn workers # rebuild them by importing this module. diff --git a/recordstream/items.py b/recordstream/items.py index 87461ff..be8aa32 100644 --- a/recordstream/items.py +++ b/recordstream/items.py @@ -173,9 +173,11 @@ class Regions: """A set of rectangular regions / bounding boxes with optional labels and scores. Attributes: - boxes: A list of boxes — pixel ``[x0, y0, x1, y1]`` or signal ``[f0, f1, t0, t1]``. - labels: Optional per-box class labels. - scores: Optional per-box confidence scores. + boxes: The boxes — pixel ``[x0, y0, x1, y1]`` or signal ``[f0, f1, t0, t1]`` rows, as a + list OR an ``[N, 4]`` array/tensor (a detection pipeline keeps its framework's type; + annotated ``Any`` because list, ndarray and tensor share no useful protocol). + labels: Optional per-box class labels (list or ``[N]`` array/tensor, like ``boxes``). + scores: Optional per-box confidence scores (list or ``[N]`` array/tensor). canvas: Optional ``(H, W)`` reference frame — the coordinate system boxes live in, so a geometric transform (flip / resize) has a self-contained frame. extras: Auxiliary PER-BOX parallel arrays and region-set measurements keyed by name @@ -183,9 +185,9 @@ class Regions: travels WITH the boxes it describes. """ - boxes: List[Any] = field(default_factory=list) - labels: Optional[List[Any]] = None - scores: Optional[List[Any]] = None + boxes: Any = field(default_factory=list) + labels: Optional[Any] = None + scores: Optional[Any] = None canvas: Optional[Tuple[int, int]] = None extras: Dict[str, Any] = field(default_factory=dict) diff --git a/recordstream/keras.py b/recordstream/keras.py index 89f8d30..b5497a5 100644 --- a/recordstream/keras.py +++ b/recordstream/keras.py @@ -42,11 +42,11 @@ import importlib.util import os -from typing import Any, Callable, Iterator, Optional, cast +from typing import Any, Callable, Iterator, Optional, Union, cast import numpy as np -from recordstream.collate import collate_records +from recordstream.collate import CollateFn, get_collate from recordstream.core import MapStyle from recordstream.items import Record @@ -96,8 +96,8 @@ class RecordSequence(keras.utils.PyDataset): """A map-style record source as a ``keras.utils.PyDataset`` of collated batches. The DataLoader half of Keras batching: row order, batch slicing, per-epoch reshuffle, and - :func:`~recordstream.collate.collate_records`. It is deliberately task-blind — ``transform`` - is the caller's ``collate_fn``-equivalent and decides what the model actually receives. + the collate. It is deliberately task-blind — ``transform`` is the caller's + ``collate_fn``-equivalent and decides what the model actually receives. Args: source: Any map-style record source (a ``Stream`` is one). @@ -106,6 +106,11 @@ class RecordSequence(keras.utils.PyDataset): seed: Shuffle seed, so a shuffled run is reproducible. transform: Maps one collated record batch to what the model consumes. ``None`` hands over the batched record itself. + collate: Which batch shape to build — a registered KEY (``"record"`` stacks what can + stack; ``"list"`` keeps every column as a per-record list) or a collate function. + The torch half takes this choice as ``DataLoader(collate_fn=…)``; this is the same + choice on the Keras side, so an engine that needs ``List[Tensor]`` inputs is not + forced to the stacking default. See :mod:`recordstream.collate`. workers: ``PyDataset`` prefetch workers. ``1`` (Keras's own default) loads batches on the calling thread; higher values overlap the record walk with the training step, which is what a slow source (decode, resize, remote read) needs. @@ -125,6 +130,7 @@ def __init__( shuffle: bool = False, seed: int = 0, transform: Optional[Callable[[Record], Any]] = None, + collate: Union[str, CollateFn] = "record", workers: int = 1, use_multiprocessing: bool = False, max_queue_size: int = 10, @@ -139,6 +145,9 @@ def __init__( self.shuffle = bool(shuffle) self.seed = int(seed) self.transform = transform + # Stored VERBATIM (a key or a function); resolved per batch by `_collate`, so a key + # registered AFTER this object was built still resolves. + self.collate: Union[str, CollateFn] = collate self._rng = np.random.default_rng(seed) self._indices: Optional[np.ndarray] = None @@ -160,6 +169,11 @@ def indices(self) -> np.ndarray: self._indices = order return self._indices + @property + def _collate(self) -> CollateFn: + """The chosen collate — a registered key resolved on use, or the function as given.""" + return get_collate(self.collate) if isinstance(self.collate, str) else self.collate + def __len__(self) -> int: """Number of batches — Keras asks once per epoch.""" return int(np.ceil(len(self.indices) / self.batch_size)) @@ -172,7 +186,7 @@ def batch(self, index: int) -> Record: # as `CollateFn = Callable[[Sequence[Any]], Any]` — deliberately loose, because the # registry holds task collates with divergent conventions — which erases # `collate_records`' own `-> Record` at the call site. - collated: Record = collate_records([source[int(i)] for i in rows]) + collated: Record = self._collate([source[int(i)] for i in rows]) return collated def batches(self) -> Iterator[Record]: diff --git a/recordstream/ops/image.py b/recordstream/ops/image.py index b9407f6..571a588 100644 --- a/recordstream/ops/image.py +++ b/recordstream/ops/image.py @@ -28,7 +28,7 @@ from recordstream._compat import is_torch_tensor from recordstream.items import Image as ImageItem from recordstream.items import Mask as MaskItem -from recordstream.items import NDArrayItem, Record, item_data, item_value +from recordstream.items import NDArrayItem, Record, Regions, is_item, item_data, item_value from recordstream.transform import Transform logger = get_logger("recordstream.ops.image") @@ -111,6 +111,35 @@ def _text_to_image(text: str, width: int = 512, height: int = 160) -> np.ndarray return np.array(img) +def image_frame(value: Any) -> Optional[Tuple[int, int]]: + """The ``(H, W)`` raster of an image-bearing value, or ``None`` when it is not one. + + The reference frame a :class:`~recordstream.Regions`' boxes are stated in is a raster, so + "what raster is this?" is asked wherever boxes and pixels have to agree — the coupled + image+boxes resize reads it to derive its scale factors, the ops that CREATE a target read + it to record the frame on the item, and any consumer comparing the two reads it to notice a + desync. It was written out per call site before it was extracted. + + It reads the DECLARED layout and does not guess: an :class:`~recordstream.Image` item is + trusted for its ``layout``, a PIL image for its ``size`` (which is ``(W, H)`` — the one + transposed convention here), and a bare array is read as ``HWC``, the layout ``Image`` + documents and the one every pre-tensor path in this package produces. It deliberately does + NOT sniff a channel axis: this package already carries three separate, deliberately + divergent channels-first heuristics, and a fourth guessing one HERE would silently mislabel + the frame that box coordinates are validated against. + """ + payload = item_data(value) if is_item(value) else value + if hasattr(payload, "size") and hasattr(payload, "convert"): # PIL: size is (W, H) + width, height = payload.size + return int(height), int(width) + shape = getattr(payload, "shape", None) + if shape is None or len(shape) not in (2, 3): + return None + if isinstance(value, ImageItem) and getattr(value, "layout", "HWC") == "CHW" and len(shape) == 3: + return int(shape[1]), int(shape[2]) + return int(shape[0]), int(shape[1]) + + def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: """Render an arbitrary value to an ``(H, W, 3)`` uint8 RGB image WITHOUT resizing. @@ -651,6 +680,9 @@ def __init__( self.flip_vertical = bool(flip_vertical) self.field = field self.output = output + # Private, so it stays out of the config surface (it is not a knob) — see + # `_warn_if_it_desyncs_regions`, which reports the configuration once, not per record. + self._warned_about_regions = False def _find_source(self, record: Record) -> Any: """Resolve the payload to render (``self.field`` or the first array-bearing item).""" @@ -664,6 +696,33 @@ def _find_source(self, record: Record) -> Any: return item_data(item) raise ValueError(f"ConvertToImage: no array-bearing field in record (keys: {list(record)})") + def _warn_if_it_desyncs_regions(self, record: Record, before: Tuple[int, int], after: Tuple[int, int]) -> None: + """Warn ONCE when this op resized the pixels of a record whose boxes describe them. + + This op resizes the IMAGE and nothing else, which is correct for what it is — but a + record carrying a :class:`~recordstream.Regions` states its boxes in a raster, and moving + the pixels out from under them leaves the two disagreeing with no error of its own: every + shape stays valid and only the coordinates become wrong. Downstream that surfaces as a + model quietly training against misplaced targets, which is the expensive way to find out. + + It is a WARNING and not an error because this op cannot know what the boxes describe — a + record may legitimately carry regions belonging to a different key than the field being + rendered — so the condition is likely, not certain. It fires once per op instance: the + message is about the CONFIGURATION, so a second copy per record only buries it. + """ + if self._warned_about_regions or before == after: + return + keys = [key for key, value in record.items() if isinstance(value, Regions)] + if not keys: + return + self._warned_about_regions = True + logger.warning( + f"ConvertToImage resized {before} -> {after} (H, W) on a record whose {keys} " + f"carries detection boxes — this op moves PIXELS ONLY, so those boxes now describe " + f"a raster that no longer exists. Use recordstream.ops.target.ResizeDetection, which " + f"moves the image and its boxes in one coupled step. (Warned once per op.)" + ) + def __call__(self, record: Record) -> Record: rgb = _render_rgb(self._find_source(record), self.colormap) if self.flip_vertical: @@ -674,6 +733,7 @@ def __call__(self, record: Record) -> Record: ) else: out_arr = _bound_longest_side(rgb, self.max_size) + self._warn_if_it_desyncs_regions(record, rgb.shape[:2], out_arr.shape[:2]) return {**record, self.output: ImageItem(out_arr, layout="HWC")} diff --git a/recordstream/ops/target.py b/recordstream/ops/target.py index 57d70c0..3517b03 100644 --- a/recordstream/ops/target.py +++ b/recordstream/ops/target.py @@ -14,7 +14,7 @@ it into a framework tensor downstream (e.g. a collate function) when a loss needs one. """ -from typing import Any, Dict, Literal, Optional +from typing import Any, Dict, Literal, Optional, Tuple import numpy as np from confluid import configurable @@ -27,6 +27,37 @@ BBoxFormat = Literal["xywh", "xyxy", "cxcywh"] +def _source_frame(record: Record) -> Optional[Tuple[int, int]]: + """The ``(H, W)`` raster a record's boxes are stated in, or ``None`` when it cannot be read. + + An annotation gives coordinates in the pixel space of the image it annotates, so the frame + is the IMAGE's — which the record carries but the annotation does not. It is looked up under + the ``"image"`` key first (this engine's declared key vocabulary — what the op-family + dispatch hands to a library, what the coupled resize defaults to, what the dataset sources + yield), then as the first :class:`~recordstream.Image` item. + + It is deliberately narrow: only a declared image is trusted. A generic "first array with two + dimensions" search would happily read a ``Regions``' own ``[N, 4]`` box array as an ``N x 4`` + raster and record a confident lie. ``None`` is an ordinary answer — it leaves ``canvas`` + exactly as it was before this was recorded at all, so nothing depends on the lookup + succeeding. + """ + from recordstream.items import Image as ImageItem + from recordstream.ops.image import image_frame + + candidate = record.get("image") + if candidate is not None: + frame = image_frame(candidate) + if frame is not None: + return frame + for value in record.values(): + if isinstance(value, ImageItem): + frame = image_frame(value) + if frame is not None: + return frame + return None + + def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: Any, op_name: str) -> Any: """Return ``mapping[value]``, or ``default`` when missing and ``ignore_unknown``. @@ -335,7 +366,13 @@ def __call__(self, record: Record) -> Record: item = record[key] objects = item.value if isinstance(item, Label) else item_data(item) target = coco_to_detection(objects, self.bbox_key, self.category_key, self.bbox_format, self.label_offset) - return {**record, self.output: Regions(boxes=target["boxes"], labels=target["labels"])} + # `canvas` IS the frame the boxes are stated in — a COCO box is in the source image's + # pixel space, so it is knowable here and recording it costs one lookup. Left None when + # no image is in the record, which is what every Regions carried before this. + return { + **record, + self.output: Regions(boxes=target["boxes"], labels=target["labels"], canvas=_source_frame(record)), + } @configurable(category="op", group="structure") @@ -405,7 +442,109 @@ def _find_mask(self, record: Record) -> np.ndarray: def __call__(self, record: Record) -> Record: mask = self._find_mask(record) target = masks_to_detection(mask, self.label, self.connected, self.min_area, self.connectivity) - return {**record, self.output: Regions(boxes=target["boxes"], labels=target["labels"])} + # The boxes were derived FROM this mask, so its shape is the frame exactly — no lookup, + # no fallback, nothing to be wrong about. + height, width = int(mask.shape[0]), int(mask.shape[1]) + return { + **record, + self.output: Regions(boxes=target["boxes"], labels=target["labels"], canvas=(height, width)), + } + + +@configurable(category="op", group="structure") +class ResizeDetection(Transform): + """Resize an image AND scale its detection-target boxes in ONE coupled step. + + The detection twin of the joint image+mask draw: a fixed-input-size detector needs the image + resized, and a resize that moved the pixels without moving the boxes would silently train on + misplaced targets. Reads the image under ``input_key`` (PIL or a uint8 HWC/2-D array), + resizes it to ``(height, width)`` (bilinear, PIL), and scales the + :class:`~recordstream.Regions` boxes under ``target_key`` by the same factors — torch boxes + stay torch, numpy stays numpy. The resized ``Regions`` records the new frame in ``canvas``. + + Ops that need no fixed size (torchvision detectors resize internally) simply omit this op — + it exists for the detectors that require pre-sized square inputs. + + Args: + width: Target width in pixels; required at use (validated lazily, ``0`` = unset). + height: Target height in pixels; required at use (validated lazily, ``0`` = unset). + input_key: Record key carrying the image (default ``"image"``). + target_key: Record key carrying the target ``Regions``; a record without it resizes the image alone. + """ + + consumes = (Regions,) + produces = (Regions,) + + def __init__( + self, + width: int = 0, + height: int = 0, + input_key: str = "image", + target_key: str = "target", + ) -> None: + super().__init__() + self.width = int(width) + self.height = int(height) + self.input_key = str(input_key) + self.target_key = str(target_key) + + def _resize_image(self, payload: Any) -> Any: + from PIL import Image as PILImage + + if hasattr(payload, "convert"): # PIL + return payload.resize((self.width, self.height), PILImage.Resampling.BILINEAR) + array = np.asarray(payload) + if array.dtype != np.uint8: + raise TypeError( + f"ResizeDetection: expected a PIL image or a uint8 array under {self.input_key!r}; " + f"got dtype {array.dtype}. Run it BEFORE any float conversion (e.g. before ToTensor)." + ) + resized = PILImage.fromarray(array).resize((self.width, self.height), PILImage.Resampling.BILINEAR) + return np.array(resized) + + @staticmethod + def _scale_boxes(boxes: Any, sx: float, sy: float) -> Any: + """Scale ``[N, 4]`` xyxy boxes by per-axis factors, preserving the array framework.""" + from recordstream._compat import is_torch_tensor + + if is_torch_tensor(boxes): + import torch + + return boxes * torch.tensor([sx, sy, sx, sy], dtype=boxes.dtype) + return np.asarray(boxes, dtype=np.float64).reshape(-1, 4) * np.array([sx, sy, sx, sy]) + + def __call__(self, record: Record) -> Record: + if self.width < 1 or self.height < 1: + raise ValueError(f"ResizeDetection needs positive width/height (got {self.width}x{self.height}).") + if self.input_key not in record: + raise ValueError(f"ResizeDetection: field {self.input_key!r} not in record (keys: {list(record)})") + item = record[self.input_key] + payload = item_data(item) + frame = _source_frame({"image": item}) + if frame is None: + raise ValueError( + f"ResizeDetection: the value under {self.input_key!r} is not an image " + f"(got {type(payload).__name__}) — it has no raster to resize." + ) + orig_h, orig_w = frame + resized = self._resize_image(payload) + merged = dict(record) + from recordstream.items import NDArrayItem, with_data + + merged[self.input_key] = with_data(item, resized) if isinstance(item, NDArrayItem) else resized + + target = record.get(self.target_key) + if isinstance(target, Regions): + import dataclasses + + # An EMPTY target is re-framed too. Scaling no boxes is a no-op, but leaving the + # canvas behind would make a negative example the ONE record in a set whose frame is + # unknown — and a frame check that silently skips exactly the records with nothing + # to check is a check that reports a clean bill for the wrong reason. + sx, sy = float(self.width) / float(orig_w), float(self.height) / float(orig_h) + boxes = self._scale_boxes(target.boxes, sx, sy) if len(target.boxes) else target.boxes + merged[self.target_key] = dataclasses.replace(target, boxes=boxes, canvas=(self.height, self.width)) + return merged __all__ = [ @@ -413,6 +552,7 @@ def __call__(self, record: Record) -> Record: "DecodeTarget", "CocoToTorchVisionDetection", "MasksToDetectionBoxes", + "ResizeDetection", "coco_to_detection", "masks_to_detection", ] diff --git a/recordstream/ops/torch.py b/recordstream/ops/torch.py index 9dc3c78..791567f 100644 --- a/recordstream/ops/torch.py +++ b/recordstream/ops/torch.py @@ -11,14 +11,27 @@ def to_tensor(img: Any, normalize: bool = True, mode: Optional[str] = None) -> torch.Tensor: """Convert a PIL image / NumPy array to a CHW ``torch.Tensor``. - A PIL image is optionally mode-coerced (``mode="RGB"`` forces 3 channels) then arrayed; - an ``[H, W, C]`` array is transposed to ``[C, H, W]`` (a 2-D array gets a leading channel + ``mode`` (e.g. ``"RGB"``, forcing 3 channels) coerces a PIL payload directly — and a + ``uint8`` ARRAY payload through a PIL round-trip, because a decoding source may hand over + the already-arrayed pixels: a mixed-mode dataset (RGB + RGBA + grayscale rows) then reaches + the model with ragged channel counts unless the coercion applies to arrays too. (Found the + hard way: cppe-5 ships 3-, 4- and 1-channel images, and a ``mode="RGB"`` that silently + skipped arrays crashed torchvision's normalize mid-epoch with a channel mismatch.) A + non-``uint8`` array with ``mode`` set is left as-is — PIL cannot represent it faithfully. + + An ``[H, W, C]`` array is transposed to ``[C, H, W]`` (a 2-D array gets a leading channel axis). With ``normalize`` an integer / 0-255-float payload is scaled into ``[0, 1]``. """ if hasattr(img, "convert"): if mode is not None: img = img.convert(mode) img = np.array(img) + elif mode is not None and isinstance(img, np.ndarray) and img.dtype == np.uint8: + from PIL import Image as PILImage + + # A trailing singleton channel axis defeats `fromarray` — squeeze it to the 2-D form. + arr = img[..., 0] if (img.ndim == 3 and img.shape[2] == 1) else img + img = np.array(PILImage.fromarray(arr).convert(mode)) if isinstance(img, np.ndarray): if img.ndim == 3: diff --git a/recordstream/outputs.py b/recordstream/outputs.py index b557363..572b8f8 100644 --- a/recordstream/outputs.py +++ b/recordstream/outputs.py @@ -39,8 +39,10 @@ class ids. That guess is not hypothetical: two independently-written detector wr "ClassificationOutput", "DetectionOutput", "DetectionPredictions", + "RestorationOutput", "SegmentationOutput", "classification_output", + "restoration_output", "segmentation_output", ] @@ -88,6 +90,27 @@ class SegmentationOutput(TypedDict, Generic[ArrayT]): mask: ArrayT +class RestorationOutput(TypedDict, Generic[ArrayT]): + """Per-record image-to-image restoration prediction. + + Keys: + image: ``[B, C, H, W]`` float — the restored image, in the SAME value range as the + model's input (``[0, 1]`` for a pipeline that ends in ``ToTensor(normalize=True)``). + + **One key, and that is the whole point of the contract.** The other tasks here carry three + because their raw output needs interpreting — ``logits`` are not ``probs`` are not + ``class_idx``. A restoration model emits the answer directly, so what a consumer needs told + is precisely that: the array under ``image`` is a picture in the input's units, NOT logits + to be softmaxed and NOT a residual to be added back. A sink that writes a PNG, a PSNR metric + and a viewer all read it the same way because this says so. + + The residual (what the model removed) is deliberately absent: it is ``input - image``, and a + contract key that a consumer can compute is a second place for the two to disagree. + """ + + image: ArrayT + + #: A detector's per-image results — one :class:`DetectionOutput` per image in the batch. DetectionPredictions = List[DetectionOutput] @@ -118,5 +141,23 @@ def segmentation_output(logits: "Tensor") -> "SegmentationOutput[Tensor]": return SegmentationOutput(logits=logits, probs=probs, mask=mask) +def restoration_output(image: "Tensor") -> "RestorationOutput[Tensor]": + """Build a :class:`RestorationOutput` from a restored ``[B, C, H, W]`` image. + + It transforms nothing — unlike its two siblings, which softmax and argmax — because a + restoration model's output already IS the contract's one key. The builder exists anyway so + every backend spells the wrapping identically and a reader of ``predict_step`` sees the same + shape of line in all four tasks:: + + def predict_step(self, batch, batch_idx): + return restoration_output(self(x)) # what a predictions sink reads + + Clamping is deliberately NOT done here: whether a restored image may leave ``[0, 1]`` is the + model's business (a residual denoiser can legitimately overshoot, and a metric computed on + clamped values is a different number), so a run that wants it wires it where it decides that. + """ + return RestorationOutput(image=image) + + # Detection deliberately has NO builder: boxes come from the detector's own interface, so the # dict is built inline at the call site rather than invented from nothing here. diff --git a/recordstream/sources/huggingface.py b/recordstream/sources/huggingface.py index 954213d..8a64d64 100644 --- a/recordstream/sources/huggingface.py +++ b/recordstream/sources/huggingface.py @@ -300,4 +300,10 @@ def __len__(self) -> int: # would report 0 for the common "0 == unlimited" case, making the source # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) # even though iteration yields every record. - return self.count or len(self.dataset) + # + # A ``count`` LARGER than the split is clamped, never reported verbatim: a map-style + # consumer trusts ``len()`` for its index space, so a lying length surfaces as an + # ``IndexError`` deep inside a DataLoader worker, one epoch in. (Found the hard way: + # ``count: 32`` over cppe-5's 29-row test split killed the first validation pass.) + n = len(self.dataset) + return min(self.count, n) if self.count else n diff --git a/tests/test_batch.py b/tests/test_batch.py index f5621bf..c3964ca 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -5,6 +5,8 @@ the caller's job, so there is deliberately no dtype promotion or multi-hot test here. """ +from typing import Any + import numpy as np import pytest import torch @@ -202,3 +204,151 @@ def test_a_label_column_contributes_its_values() -> None: if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +# --------------------------------------------------------------------------- # +# batch_regions — the collate's transpose for a region-set column +# --------------------------------------------------------------------------- # +class TestBatchRegions: + """A collated `Regions` back into the per-record dicts every detection interface takes.""" + + def _batch(self, counts: tuple = (1, 3), scores: bool = False) -> Any: + import torch + + from recordstream import Regions, collate_records + + records = [ + { + "target": Regions( + boxes=torch.rand(n, 4), + labels=torch.zeros(n, dtype=torch.int64), + scores=torch.ones(n) if scores else None, + ) + } + for n in counts + ] + return collate_records(records) + + def test_it_transposes_a_variable_n_column_into_per_record_dicts(self) -> None: + from recordstream import batch_regions + + targets = batch_regions(self._batch(counts=(1, 3)), "target") + assert len(targets) == 2 + assert [tuple(t["boxes"].shape) for t in targets] == [(1, 4), (3, 4)] + assert [tuple(t["labels"].shape) for t in targets] == [(1,), (3,)] + + def test_an_absent_field_is_OMITTED_not_handed_over_as_none(self) -> None: + """A training target is exactly {boxes, labels} — a `None` scores key would reach a model.""" + from recordstream import batch_regions + + assert set(batch_regions(self._batch(), "target")[0]) == {"boxes", "labels"} + assert set(batch_regions(self._batch(scores=True), "target")[0]) == {"boxes", "labels", "scores"} + + def test_values_keep_their_framework(self) -> None: + """Framework-free by rule: the caller owns dtype and device, as with `batch_values`.""" + import torch + + from recordstream import batch_regions + + assert isinstance(batch_regions(self._batch(), "target")[0]["boxes"], torch.Tensor) + + def test_numpy_boxes_stay_numpy(self) -> None: + import numpy as np + + from recordstream import Regions, batch_regions, collate_records + + batch = collate_records([{"target": Regions(boxes=np.zeros((2, 4)), labels=np.zeros(2))}]) + assert isinstance(batch_regions(batch, "target")[0]["boxes"], np.ndarray) + + def test_a_wrong_type_raises_naming_it(self) -> None: + import pytest + + from recordstream import Label, batch_regions, collate_records + + with pytest.raises(TypeError, match="not a Regions"): + batch_regions(collate_records([{"target": Label(0)}]), "target") + + def test_an_uncollated_regions_raises_naming_the_mistake(self) -> None: + import numpy as np + import pytest + + from recordstream import Regions, batch_regions + + with pytest.raises(ValueError, match="not a COLLATED Regions"): + batch_regions({"target": Regions(boxes=np.zeros((2, 4)))}, "target") + + +# --------------------------------------------------------------------------- # +# The collate is a CHOICE — and both choices read back the same +# --------------------------------------------------------------------------- # +class TestCollateIsAChoice: + """`"record"` stacks, `"list"` does not — and every read-back helper accepts both. + + That last property is what makes the choice free rather than a fork in every consumer: a + trainer picks the batch shape its MODEL needs and reads the batch the same way either way. + """ + + def _records(self, sizes: tuple = (8, 8)) -> Any: + import torch + + from recordstream import Image, Label, Regions + + return [ + { + "image": Image(np.zeros((3, s, s), dtype="float32"), layout="CHW"), + "target": Regions(boxes=torch.rand(n, 4), labels=torch.zeros(n, dtype=torch.int64)), + "class": Label(i), + } + for i, (s, n) in enumerate(zip(sizes, (1, 3))) + ] + + def test_both_keys_are_registered(self) -> None: + from recordstream import registered_collates + + assert {"record", "list"} <= set(registered_collates()) + + def test_record_stacks_and_list_does_not(self) -> None: + from recordstream import Image, collate_list, collate_records + + stacked = collate_records(self._records())["image"] + listed = collate_list(self._records())["image"] + assert isinstance(stacked, Image) and stacked.shape == (2, 3, 8, 8) + assert isinstance(listed, list) and [v.shape for v in listed] == [(3, 8, 8), (3, 8, 8)] + + def test_the_list_collate_keeps_items_as_items(self) -> None: + """Per-record metadata survives — a list of bare arrays would drop every `layout`.""" + from recordstream import Image, collate_list + + column = collate_list(self._records())["image"] + assert all(isinstance(v, Image) for v in column) + assert [v.layout for v in column] == ["CHW", "CHW"] + + def test_a_variable_size_column_is_a_CHOICE_not_a_crash(self) -> None: + """The whole point: ragged is fine when you asked for lists, and the default explains + itself instead of raising numpy's shape error from three frames down.""" + from recordstream import collate_list, collate_records + + ragged = self._records(sizes=(8, 12)) + with pytest.raises(ValueError, match=r"cannot stack the 'image' column"): + collate_records(ragged) + assert len(collate_list(ragged)["image"]) == 2 + + def test_the_stack_error_names_the_shapes_and_the_way_out(self) -> None: + from recordstream import collate_records + + with pytest.raises(ValueError) as excinfo: + collate_records(self._records(sizes=(8, 12))) + message = str(excinfo.value) + assert "(3, 8, 8)" in message and "(3, 12, 12)" in message + assert '"list"' in message, "the message must name the collate that CAN batch this" + + @pytest.mark.parametrize("collate_key", ["record", "list"]) + def test_every_read_back_helper_accepts_both_collates(self, collate_key: str) -> None: + """`batch_values` / `batch_regions` / `batch_metadata` give the SAME answer either way.""" + from recordstream import batch_metadata, batch_regions, batch_values, collate + + batch = collate(self._records(), key=collate_key) + assert batch_values(batch, "class") == [0, 1] + targets = batch_regions(batch, "target") + assert [t["boxes"].shape[0] for t in targets] == [1, 3] + assert batch_metadata(batch, exclude=("image", "target")) == [{"class": 0}, {"class": 1}] diff --git a/tests/test_convert_to_mask.py b/tests/test_convert_to_mask.py index 1bf6562..bccc33f 100644 --- a/tests/test_convert_to_mask.py +++ b/tests/test_convert_to_mask.py @@ -10,13 +10,14 @@ "get past a wrapper item" rule ``iter_key`` and ``batch_values`` already had. """ -from typing import Any, Dict, Tuple +from contextlib import contextmanager +from typing import Any, Dict, Iterator, List, Tuple import numpy as np import pytest from PIL import Image as PILImage -from recordstream import Image, Label, Mask, MultiLabel, collate_records, item_data, item_value +from recordstream import Image, Label, Mask, MultiLabel, Regions, collate_records, item_data, item_value from recordstream.core import _apply_op from recordstream.ops import ConvertToMask, DropField, FormulaOp @@ -237,3 +238,110 @@ def test_num_classes_still_refuses_an_array_target(self) -> None: with pytest.raises((TypeError, ValueError)): num_classes([{"class": Mask(np.array([[0, 1], [1, 0]], dtype=np.int64))}]) + + +@contextmanager +def _captured_warnings() -> Iterator[List[str]]: + """Collect loggair WARNING records emitted inside the block. + + `caplog` cannot see these: loggair is loguru, which does not propagate to stdlib logging. + Neither can `capfd` alone — the sink is ENQUEUED, so the write lands on another thread after + the assertion runs (this looked exactly like "the warning never fired"). `logger.complete()` + is the deterministic flush the workspace mandates instead of sleeping. + """ + from loguru import logger + + collected: List[str] = [] + sink_id = logger.add(lambda message: collected.append(str(message)), level="WARNING") + try: + yield collected + logger.complete() + finally: + logger.remove(sink_id) + + +class TestBoxesKnowTheirFrame: + """`canvas` is the raster a `Regions`' boxes are stated in — so every op that makes or + re-frames one records it, and the op that moves pixels ALONE says so. + + Before this, only the coupled resize set `canvas`, which meant the frame was knowable + exactly when it was least needed (a resize that already moved the boxes correctly) and + unknown in the chain where boxes and pixels can actually drift apart. + """ + + def test_coco_boxes_record_the_image_they_annotate(self) -> None: + from recordstream.ops.target import CocoToTorchVisionDetection + + record = { + "image": Image(np.zeros((300, 400, 3), dtype="uint8")), + "objects": Label({"bbox": [[10.0, 10.0, 20.0, 20.0]], "category": [1]}), + } + out = CocoToTorchVisionDetection(field="objects")(record) + assert out["target"].canvas == (300, 400), "the annotation's frame is the image's" + + def test_mask_derived_boxes_record_the_mask(self) -> None: + from recordstream.ops.target import MasksToDetectionBoxes + + mask = np.zeros((64, 96), dtype="int64") + mask[10:20, 30:40] = 1 + out = MasksToDetectionBoxes()({"mask": Mask(mask)}) + assert out["target"].canvas == (64, 96), "the boxes were derived FROM this raster" + + def test_a_record_with_no_image_still_works(self) -> None: + """`None` is an ordinary answer — the lookup failing must not fail the op.""" + from recordstream.ops.target import CocoToTorchVisionDetection + + out = CocoToTorchVisionDetection(field="objects")( + {"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [0]})} + ) + assert out["target"].canvas is None + + def test_a_regions_box_array_is_never_mistaken_for_a_raster(self) -> None: + """The narrow lookup's whole point: an `[N, 4]` box array is 2-D and must not be read + as an N x 4 image, which a generic first-array search would do confidently.""" + from recordstream.ops.target import CocoToTorchVisionDetection + + record = { + "objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [0]}), + "other": Regions(boxes=np.zeros((7, 4), dtype="float32"), labels=np.zeros((7,), dtype="int64")), + } + assert CocoToTorchVisionDetection(field="objects")(record)["target"].canvas is None + + def test_an_EMPTY_target_is_re_framed_too(self) -> None: + """A negative example must not be the one record whose frame is unknown.""" + from recordstream.ops.target import ResizeDetection + + record = { + "image": Image(np.zeros((100, 100, 3), dtype="uint8")), + "target": Regions(boxes=np.zeros((0, 4), dtype="float32"), labels=np.zeros((0,), dtype="int64")), + } + out = ResizeDetection(width=64, height=32)(record) + assert out["target"].canvas == (32, 64) + + def test_an_image_only_resize_WARNS_when_it_desyncs_boxes(self) -> None: + from recordstream.ops.image import ConvertToImage + + record = { + "image": Image(np.zeros((200, 200, 3), dtype="uint8")), + "target": Regions(boxes=np.array([[10.0, 10.0, 50.0, 50.0]]), labels=np.array([1])), + } + op = ConvertToImage(field="image", width=64, height=64) + with _captured_warnings() as warnings: + op(record) + assert len(warnings) == 1 + assert "ResizeDetection" in warnings[0] and "PIXELS ONLY" in warnings[0] + assert "(200, 200) -> (64, 64)" in warnings[0], "the message states both rasters" + # Once per op instance: the message is about the CONFIGURATION, not about this record. + with _captured_warnings() as second: + op(record) + assert second == [] + + def test_no_warning_without_boxes_or_without_a_resize(self) -> None: + from recordstream.ops.image import ConvertToImage + + image = Image(np.zeros((200, 200, 3), dtype="uint8")) + boxes = Regions(boxes=np.array([[1.0, 2.0, 3.0, 4.0]]), labels=np.array([1])) + with _captured_warnings() as warnings: + ConvertToImage(field="image", width=64, height=64)({"image": image}) # resized, no boxes + ConvertToImage(field="image", max_size=999)({"image": image, "target": boxes}) # boxes, no resize + assert warnings == [] diff --git a/tests/test_huggingface_source.py b/tests/test_huggingface_source.py new file mode 100644 index 0000000..51ce8e4 --- /dev/null +++ b/tests/test_huggingface_source.py @@ -0,0 +1,27 @@ +"""HuggingFaceSource unit pins that need no Hub access (the dataset cache is stubbed).""" + +from recordstream.sources.huggingface import HuggingFaceSource + + +def _with_rows(n: int, count: int = 0) -> HuggingFaceSource: + source = HuggingFaceSource(path="stub/dataset", count=count) + source._dataset = list(range(n)) # the lazy cache — a list satisfies len()/iteration + return source + + +def test_len_reports_the_dataset_when_count_is_unset() -> None: + """``count`` of 0 (or None) means "all records" — a bare ``self.count`` would report 0 and + make the source look empty to a len()-based consumer.""" + assert len(_with_rows(29)) == 29 + + +def test_len_reports_count_when_it_caps() -> None: + assert len(_with_rows(29, count=10)) == 10 + + +def test_len_is_clamped_when_count_exceeds_the_split() -> None: + """A ``count`` larger than the split must never be reported verbatim: a map-style consumer + trusts ``len()`` for its index space, so a lying length surfaces as an ``IndexError`` deep + inside a DataLoader worker, one epoch in (found the hard way — ``count: 32`` over cppe-5's + 29-row test split killed the first validation pass).""" + assert len(_with_rows(29, count=32)) == 29 diff --git a/tests/test_keras_sequence.py b/tests/test_keras_sequence.py index 2db802c..a8eee06 100644 --- a/tests/test_keras_sequence.py +++ b/tests/test_keras_sequence.py @@ -251,3 +251,46 @@ def test_with_no_engine_installed_it_defers_to_keras_own_default(monkeypatch: py if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +# --------------------------------------------------------------------------- # +# The collate is selectable here too — the torch half's `collate_fn`, on the Keras side +# --------------------------------------------------------------------------- # +def test_the_collate_defaults_to_the_record_key() -> None: + from recordstream.keras import RecordSequence + + assert RecordSequence().collate == "record" + + +def test_a_registered_key_selects_the_batch_shape() -> None: + """A `PyDataset` had no way to say "don't stack" — that made the collate a torch-only + choice, which is the gap `register_collate` existed for and nothing used.""" + import numpy as np + + from recordstream import Image + from recordstream.keras import RecordSequence + + records = [{"image": Image(np.zeros((3, 8, 8), dtype="float32"), layout="CHW")} for _ in range(2)] + + stacked = RecordSequence(records, batch_size=2, collate="record").batch(0)["image"] + listed = RecordSequence(records, batch_size=2, collate="list").batch(0)["image"] + assert getattr(stacked, "shape", None) == (2, 3, 8, 8) + assert isinstance(listed, list) and len(listed) == 2 + + +def test_a_collate_FUNCTION_is_accepted_too() -> None: + """Keys serve JSON-carrying tool surfaces; a function stays the normal Python path.""" + from recordstream import collate_list + from recordstream.keras import RecordSequence + + seq = RecordSequence([{"x": 1}, {"x": 2}], batch_size=2, collate=collate_list) + assert seq.batch(0) == {"x": [1, 2]} + + +def test_an_unknown_key_names_the_registered_ones() -> None: + import pytest + + from recordstream.keras import RecordSequence + + with pytest.raises(KeyError, match="known:"): + RecordSequence([{"x": 1}], batch_size=1, collate="nope").batch(0) diff --git a/tests/test_op_families.py b/tests/test_op_families.py index b677ed5..e735735 100644 --- a/tests/test_op_families.py +++ b/tests/test_op_families.py @@ -7,6 +7,7 @@ op, ``field=`` targeting, and the ``WrappedOp``/``FilterOp`` raw-callable routes. """ +from contextlib import contextmanager from pathlib import Path from typing import Dict, Iterator, List, Optional @@ -17,7 +18,7 @@ from confluid import configurable from torchvision.transforms import v2 -from recordstream import FilterOp, Image, Label, Mask, Pipeline, Record, Transform, WrappedOp +from recordstream import FilterOp, Image, Label, Mask, Pipeline, Record, Regions, Transform, WrappedOp from recordstream.core import Stream, _apply_op, _is_albumentations, _is_torchvision_v2 @@ -376,3 +377,210 @@ def test_attribute_reduction_is_not_part_of_the_contract(self) -> None: from recordstream.ops.formula import _FORMULA_NAMESPACE assert {"amax", "amin", "mean", "std", "median"} <= set(_FORMULA_NAMESPACE) + + +@contextmanager +def _captured_warnings() -> Iterator[List[str]]: + """Collect loggair WARNING records emitted inside the block. + + `caplog` cannot see these — loggair is loguru, which does not propagate to stdlib logging — + and its sink is ENQUEUED, so reading a captured stream races the writer. `logger.complete()` + is the deterministic flush (the workspace forbids sleeping for one). + """ + from loguru import logger + + collected: List[str] = [] + sink_id = logger.add(lambda message: collected.append(str(message)), level="WARNING") + try: + yield collected + logger.complete() + finally: + logger.remove(sink_id) + + +class TestGeometryLeavingRegionsBehind: + """A `Regions` is not in albumentations' key vocabulary, so it never reaches the library. + + That is correct for the dispatch — passing a foreign item would break the call — but it + means a geometry-changing transform moves the pixels while the boxes stay put, with no + error of its own. Measured: `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes + on `[10, 10, 100, 100]`; `A.HorizontalFlip` mirrors the pixels while changing NO shape at + all, which is why the condition is the library's spatial/photometric taxonomy rather than + "did the raster change". + """ + + @staticmethod + def _record() -> Record: + return { + "image": Image(np.zeros((200, 200, 3), dtype="uint8")), + "target": Regions(boxes=np.array([[10.0, 10.0, 100.0, 100.0]]), labels=np.array([1])), + } + + @pytest.fixture(autouse=True) + def _forget_previous_warnings(self) -> Iterator[None]: + """The once-per-type memo is module state — clear it so tests do not shadow each other.""" + from recordstream.core.families import _WARNED_SPATIAL + + snapshot = set(_WARNED_SPATIAL) + _WARNED_SPATIAL.clear() + yield + _WARNED_SPATIAL.clear() + _WARNED_SPATIAL.update(snapshot) + + def test_the_desync_is_real_and_silent_without_the_guard(self) -> None: + """The premise, asserted rather than assumed: pixels move, boxes do not.""" + out = _apply_op(self._record(), A.Resize(height=64, width=64)) + assert out is not None + assert out["image"].shape[:2] == (64, 64) + assert out["target"].boxes.tolist() == [[10.0, 10.0, 100.0, 100.0]], "boxes stayed behind" + + def test_a_resize_warns_naming_both_ways_out(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), A.Resize(height=64, width=64)) + assert len(warnings) == 1 + assert "bbox_params" in warnings[0] and "ResizeDetection" in warnings[0] + + def test_a_flip_warns_though_NO_shape_changes(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), A.HorizontalFlip(p=1.0)) + assert len(warnings) == 1, "a raster-change test would miss this one entirely" + + def test_an_image_only_transform_stays_silent(self) -> None: + """`Normalize` is an `ImageOnlyTransform` — it cannot touch geometry, so there is + nothing to warn about. Reading the library's own taxonomy is what makes this exact.""" + with _captured_warnings() as warnings: + _apply_op(self._record(), A.Normalize()) + assert warnings == [] + + def test_a_compose_is_recursed(self) -> None: + composed = A.Compose([A.Normalize(), A.RandomCrop(height=8, width=8)]) + with _captured_warnings() as warnings: + _apply_op(self._record(), composed) + assert len(warnings) == 1, "the spatial transform is nested one level down" + + def test_it_warns_once_per_transform_type(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), A.Resize(height=64, width=64)) + _apply_op(self._record(), A.Resize(height=32, width=32)) + assert len(warnings) == 1, "the message is about the configuration, not the record" + + def test_the_DOCUMENTED_yaml_way_out_actually_runs(self) -> None: + """The warning names a fix, so the fix has to work — this is that exact YAML. + + Note it must go through a `Stream`: deferred `!class:` markers are flowed at route + entry, so applying them straight out of `confluid.load` hands `_apply_op` a marker. + """ + import confluid + + document = """ +ops: + - !class:recordstream.ops.structure.RenameField { src: my_boxes, dst: bboxes } + - !class:albumentations.Compose + transforms: [!class:albumentations.HorizontalFlip { p: 1.0 }] + bbox_params: !class:albumentations.BboxParams { format: pascal_voc, label_fields: [labels] } +""" + record = { + "image": np.zeros((100, 100, 3), dtype="uint8"), + "my_boxes": [[10.0, 10.0, 40.0, 40.0]], + "labels": [1], + } + with _captured_warnings() as warnings: + out = list(Stream(source=[record], ops=confluid.load(document, flow=True)["ops"]))[0] + assert warnings == [] + assert [round(v, 1) for v in out["bboxes"][0]] == [60.0, 10.0, 90.0, 40.0], "mirrored across x" + + def test_the_correct_spelling_is_NOT_warned_about_and_moves_the_boxes(self) -> None: + """Boxes in the library's own vocabulary: it moves them in the same joint draw.""" + composed = A.Compose( + [A.Resize(height=64, width=64)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ) + record = { + "image": np.zeros((200, 200, 3), dtype="uint8"), + "bboxes": [[10.0, 10.0, 100.0, 100.0]], + "labels": [1], + } + with _captured_warnings() as warnings: + out = _apply_op(record, composed) + assert warnings == [] + assert out is not None + assert [round(v, 1) for v in out["bboxes"][0]] == [3.2, 3.2, 32.0, 32.0], "boxes scaled with the image" + + +class TestV2GeometryLeavingRegionsBehind: + """The same gap in the OTHER family, reached by a different route. + + albumentations misses a `Regions` because it is not in the KEY vocabulary; torchvision v2 + misses it because it is not one of v2's tv_tensor TYPES. Measured: `v2.Resize((64, 64))` + takes a 200x200 image to 64x64 with the boxes still on `[10, 10, 100, 100]`, while the same + transform over a `tv_tensors.BoundingBoxes` rescales them to `[3.2, 3.2, 32, 32]`. + """ + + @staticmethod + def _record() -> Record: + return { + "image": Image(np.zeros((200, 200, 3), dtype="uint8")), + "target": Regions(boxes=torch.tensor([[10.0, 10.0, 100.0, 100.0]]), labels=torch.tensor([1])), + } + + @pytest.fixture(autouse=True) + def _forget_previous_warnings(self) -> Iterator[None]: + from recordstream.core.families import _WARNED_SPATIAL + + snapshot = set(_WARNED_SPATIAL) + _WARNED_SPATIAL.clear() + yield + _WARNED_SPATIAL.clear() + _WARNED_SPATIAL.update(snapshot) + + def test_the_desync_is_real(self) -> None: + out = _apply_op(self._record(), v2.Compose([v2.ToImage(), v2.Resize((64, 64))])) + assert out is not None + assert tuple(out["image"].shape[-2:]) == (64, 64) + assert out["target"].boxes.tolist() == [[10.0, 10.0, 100.0, 100.0]], "boxes stayed behind" + + def test_a_geometric_transform_warns_naming_the_way_out(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), v2.RandomHorizontalFlip(p=1.0)) + assert len(warnings) == 1 + assert "BoundingBoxes" in warnings[0] and "ResizeDetection" in warnings[0] + + def test_a_non_geometric_transform_stays_silent(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), v2.ColorJitter(brightness=0.5)) + assert warnings == [] + + def test_a_compose_is_recursed(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), v2.Compose([v2.ToImage(), v2.Resize((32, 32))])) + assert len(warnings) == 1 + + def test_v2s_OWN_box_type_is_transformed_and_not_warned_about(self) -> None: + from torchvision import tv_tensors + + record = { + "image": tv_tensors.Image(torch.zeros(3, 200, 200, dtype=torch.uint8)), + "boxes": tv_tensors.BoundingBoxes( + torch.tensor([[10.0, 10.0, 100.0, 100.0]]), format="XYXY", canvas_size=(200, 200) + ), + } + with _captured_warnings() as warnings: + out = _apply_op(record, v2.Resize((64, 64))) + assert warnings == [] + assert out is not None + assert [round(v, 1) for v in out["boxes"].tolist()[0]] == [3.2, 3.2, 32.0, 32.0] + + def test_the_geometry_signal_still_matches_this_torchvision(self) -> None: + """The signal is a PRIVATE module path, so it can go stale on a torchvision upgrade. + + It fails OPEN (no warning, nothing else changes), which is the right direction for a + diagnostic but also the direction that rots unnoticed — so assert the classification + directly rather than only through a warning that would silently stop appearing. + """ + from recordstream.core.families import _is_v2_geometry + + assert _is_v2_geometry(v2.Resize((8, 8))) + assert _is_v2_geometry(v2.RandomHorizontalFlip()) + assert _is_v2_geometry(v2.RandomCrop(8)) + assert not _is_v2_geometry(v2.ColorJitter()) + assert not _is_v2_geometry(v2.Normalize(mean=[0.0], std=[1.0])) diff --git a/tests/test_outputs.py b/tests/test_outputs.py index 3df80d7..6266b14 100644 --- a/tests/test_outputs.py +++ b/tests/test_outputs.py @@ -5,8 +5,10 @@ from recordstream.outputs import ( ClassificationOutput, DetectionOutput, + RestorationOutput, SegmentationOutput, classification_output, + restoration_output, segmentation_output, ) @@ -38,6 +40,39 @@ def test_segmentation_output_per_pixel_argmax() -> None: assert torch.equal(out["mask"], logits.argmax(dim=1).to(torch.int64)) +def test_restoration_output_carries_the_image_unchanged() -> None: + """The builder transforms NOTHING — a restored image already IS the contract's one key.""" + image = torch.rand(2, 3, 8, 8) + out = restoration_output(image) + + assert set(out.keys()) == {"image"} + assert out["image"] is image + assert out["image"].shape == (2, 3, 8, 8) + + +def test_restoration_output_does_not_clamp_the_range() -> None: + """A residual denoiser can legitimately overshoot [0, 1], and clamping would change the score. + + Whether the output is clipped is the RUN's decision (a metric on clamped values is a + different number), so the contract carries what the model produced. + """ + out = restoration_output(torch.tensor([[[[-0.25, 1.5]]]])) + + assert float(out["image"].min()) == -0.25 + assert float(out["image"].max()) == 1.5 + + +def test_restoration_output_has_no_residual_key() -> None: + """``input - image`` is derivable, and a derivable key is a second place for the two to disagree.""" + assert "residual" not in restoration_output(torch.zeros(1, 3, 4, 4)) + + +def test_restoration_output_is_typed_dict_instance() -> None: + out: RestorationOutput = restoration_output(torch.zeros(1, 3, 4, 4)) + assert isinstance(out, dict) + assert set(out.keys()) == {"image"} + + def test_detection_output_typed_dict_construction() -> None: # DetectionOutput is constructed directly at call sites; verify the # TypedDict's runtime behavior matches a plain dict. @@ -104,8 +139,10 @@ def test_the_package_root_exports_the_contracts_and_builders() -> None: "ClassificationOutput", "DetectionOutput", "DetectionPredictions", + "RestorationOutput", "SegmentationOutput", "classification_output", + "restoration_output", "segmentation_output", ): assert name in recordstream.__all__ and hasattr(recordstream, name) diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py index 753a431..a92efd3 100644 --- a/tests/test_typed_detection_target_ops.py +++ b/tests/test_typed_detection_target_ops.py @@ -193,3 +193,59 @@ def test_discovery_tags(name: str, cls: type) -> None: registry = get_registry() assert name in registry.list_classes(category="op") assert name in registry.list_classes(group="structure") + + +# --------------------------------------------------------------------------- # +# ResizeDetection — the coupled image+boxes resize +# --------------------------------------------------------------------------- # +class TestResizeDetection: + def _record(self) -> dict: + import torch + + from recordstream import Image, Regions + + image = Image((np.arange(40 * 20 * 3) % 256).reshape(40, 20, 3).astype(np.uint8)) # H=40, W=20 + target = Regions(boxes=torch.tensor([[5.0, 10.0, 15.0, 30.0]]), labels=torch.tensor([1])) + return {"image": image, "target": target} + + def test_image_and_boxes_move_together(self) -> None: + import torch + + from recordstream.ops.target import ResizeDetection + + out = ResizeDetection(width=40, height=80)(self._record()) # 2x on both axes + assert np.asarray(out["image"]).shape[:2] == (80, 40) + assert torch.allclose(out["target"].boxes, torch.tensor([[10.0, 20.0, 30.0, 60.0]])) + assert out["target"].canvas == (80, 40) + assert out["target"].labels.tolist() == [1] + + def test_boxes_stay_in_their_framework(self) -> None: + from recordstream import Image, Regions + from recordstream.ops.target import ResizeDetection + + record = { + "image": Image(np.zeros((10, 10, 3), dtype=np.uint8)), + "target": Regions(boxes=np.array([[1.0, 1.0, 5.0, 5.0]]), labels=np.array([0])), + } + out = ResizeDetection(width=20, height=20)(record) + assert isinstance(out["target"].boxes, np.ndarray) + assert out["target"].boxes.tolist() == [[2.0, 2.0, 10.0, 10.0]] + + def test_zero_arg_construction_validates_lazily(self) -> None: + import pytest + + from recordstream.ops.target import ResizeDetection + + op = ResizeDetection() # zero-arg per the lazy-construction mandate + with pytest.raises(ValueError, match="width/height"): + op(self._record()) + + def test_a_float_image_is_rejected_with_the_ordering_hint(self) -> None: + import pytest + + from recordstream import Image + from recordstream.ops.target import ResizeDetection + + record = {"image": Image(np.zeros((8, 8, 3), dtype=np.float32))} + with pytest.raises(TypeError, match="before ToTensor"): + ResizeDetection(width=4, height=4)(record) diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py index 18ac3d6..d0c9736 100644 --- a/tests/test_typed_target_ops.py +++ b/tests/test_typed_target_ops.py @@ -54,6 +54,40 @@ def test_parity_no_normalize(self) -> None: expected = to_tensor(arr, normalize=False).numpy() assert np.array_equal(np.asarray(out["image"]), expected) + # `mode=` must coerce ARRAY payloads too, not only PIL ones: a decoding source hands over + # already-arrayed pixels, and a mixed-mode dataset (cppe-5 ships RGB + RGBA + grayscale rows) + # then reaches the model with ragged channel counts — a channel-mismatch crash MID-EPOCH, + # after the 3-channel rows trained fine. Found the hard way; these pin the array path. + def test_mode_rgb_coerces_a_4_channel_uint8_array(self) -> None: + rgba = (np.arange(4 * 5 * 4).reshape(4, 5, 4) % 256).astype(np.uint8) + tensor = to_tensor(rgba, mode="RGB") + assert tuple(tensor.shape) == (3, 4, 5) + + def test_mode_rgb_coerces_a_2d_grayscale_uint8_array(self) -> None: + gray = (np.arange(4 * 5).reshape(4, 5) % 256).astype(np.uint8) + tensor = to_tensor(gray, mode="RGB") + assert tuple(tensor.shape) == (3, 4, 5) + + def test_mode_rgb_coerces_a_singleton_channel_uint8_array(self) -> None: + gray1 = (np.arange(4 * 5).reshape(4, 5, 1) % 256).astype(np.uint8) + tensor = to_tensor(gray1, mode="RGB") + assert tuple(tensor.shape) == (3, 4, 5) + + def test_mode_rgb_leaves_a_3_channel_array_byte_identical(self) -> None: + arr = _hwc_uint8() + assert np.array_equal(to_tensor(arr, mode="RGB").numpy(), to_tensor(arr).numpy()) + + def test_mode_is_left_alone_for_a_non_uint8_array(self) -> None: + # PIL cannot represent a float RGBA faithfully — the coercion is deliberately uint8-only. + rgba = np.random.rand(4, 5, 4).astype(np.float32) + tensor = to_tensor(rgba, mode="RGB", normalize=False) + assert tuple(tensor.shape) == (4, 4, 5) + + def test_op_mode_reaches_the_array_path(self) -> None: + rgba = (np.arange(4 * 5 * 4).reshape(4, 5, 4) % 256).astype(np.uint8) + out = ToTensor(mode="RGB")({"image": Image(rgba)}) + assert tuple(out["image"].shape) == (3, 4, 5) + def test_output_is_a_plain_live_tensor(self) -> None: # The record model holds arbitrary values: the tensor rides AS-IS (no Image wrap — an # NDArrayItem coerces via np.asarray and cannot hold a live tensor). item_data passes From dadf0a25185f85e5b5b8efd4f2ceb4c80948f68f Mon Sep 17 00:00:00 2001 From: gearlux Date: Fri, 7 Aug 2026 08:58:18 +0200 Subject: [PATCH 078/102] =?UTF-8?q?feat:=20trainer-base=20data=20helpers?= =?UTF-8?q?=20=E2=80=94=20per=5Frecord=5Fpredictions,=20prepare=5Frecord?= =?UTF-8?q?=5Fdataset,=20loader=5Fslots,=20RunnableTask?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data half of the workspace trainer-base consolidation: - batch.py: per_record_predictions (the batched-output -> per-record slices ladder, lifted from three byte-identical consumer copies) - core/stream.py: prepare_record_dataset = ensure_record_dataset + ensure_materialized (the fork-safety normalize+warm pair, documented once) - loaders.py (new, torch-extra-gated): loader_slots -> LoaderSlots NamedTuple of Lazy[DataLoader] train/val/test slots, with collate_fn + **loader_kw passthrough and a shuffle-refusal guard; deliberately NOT root-exported so 'import recordstream' stays torch-free - runnable.py: RunnableTask Literal (the standard four-verb vocabulary) - collate.py docstrings: the task-collate-stays-engine-internal decision --- AGENTS.md | 6 +- docs/kinds.md | 21 +++++++ recordstream/__init__.py | 14 ++++- recordstream/batch.py | 35 +++++++++++- recordstream/collate.py | 37 ++++++++----- recordstream/core/__init__.py | 2 + recordstream/core/stream.py | 20 +++++++ recordstream/loaders.py | 101 ++++++++++++++++++++++++++++++++++ recordstream/runnable.py | 11 +++- tests/test_batch.py | 46 ++++++++++++++++ tests/test_loaders.py | 85 ++++++++++++++++++++++++++++ tests/test_record_source.py | 37 +++++++++++++ 12 files changed, 394 insertions(+), 21 deletions(-) create mode 100644 recordstream/loaders.py create mode 100644 tests/test_loaders.py diff --git a/AGENTS.md b/AGENTS.md index a1ea188..afaaf80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → ONE step-graph engine behind two facades (`Stream`/`JointStream` for the dataset surface, `FlowGraph` for a `flow:` document) → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config; the context ops + the flow⇄ops lowering pass were DELETED 2026-07-30 (one step-graph engine, see the mandate below). Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. -- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of matrainer when matrainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the matrainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of matrainer when matrainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). **The four STANDARD tasks are the closed `RunnableTask = Literal["fit", "evaluate", "test", "predict"]` declared in this module (2026-08-06)** — a consumer's task alias is `MyTask = RunnableTask` so the set is written once; a runnable with a narrower capability set declares its own Literal (the type names the convention, the markers gate dispatch). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the matrainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. - **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). - **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and matrainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. - **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — and since 2026-08-05 the COLLATE ITSELF is selectable here too (`collate=` takes a registered key or a function, resolved per batch so a key registered later still works), because a `PyDataset` otherwise had no way to say "don't stack" and the batch-shape choice was torch-only — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). @@ -33,7 +33,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core.families._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry, And The BATCH SHAPE Is A CHOICE (`recordstream.collate`; the choice landed 2026-08-05):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`. **TWO collates ship, differing in ONE decision — whether array payloads are STACKED — because that decision belongs to the MODEL, not to the data:** `"record"` (the default, `collate_records`) stacks what can stack; `"list"` (`collate_list` = `collate_records(items, stack=False)`) stacks nothing and leaves every key a per-record list with ITEMS KEPT AS ITEMS (so an `Image`'s `layout` survives per record). Left implicit the shape is decided by ACCIDENT — a column stacks if it holds array items and stays a list if it holds PLAIN values, so `ToTensor` running or not silently decides whether a detector gets its `List[Tensor]`; the key lets a consumer DECLARE it (raidar's torch loaders pass `collate_fn=collate_list`, `RecordSequence` takes `collate=`). **Every read-back helper accepts BOTH shapes** (`batch_values` unwraps a list-of-items element-wise, `batch_regions` reads a batched `Regions` or a list of them, `batch_metadata` transposes either) — that is what makes the collate a free choice rather than a fork in every consumer, and it is pinned by a parametrized test over both keys. **A stack failure now EXPLAINS itself** (`_stack_or_explain`): it names the key, the differing shapes and the `"list"` way out, where the raw `ValueError: all input arrays must have the same shape` from inside numpy named none of the three — this closed the old TASKS item about `_stack`'s unreachable "else a list" promise, by making the fallback a DECLARED mode rather than a silent type change. **`register_collate` is signature-PRESERVING** (`TypeVar` bound to `CollateFn`, not a flat `-> CollateFn`), so registering a collate no longer erases its own parameters — that is what lets `collate_list` call `collate_records(items, stack=False)` and type-check. The default `collate_records` behaviour is unchanged: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. Consumers `register_collate` their task collates ADDITIVELY (e.g. a detection collate that hand-builds variable-N `Regions` values); their divergent conventions are deliberately NOT unified. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_regions(batch, key)` (a collated `Regions` column transposed into per-record `{boxes, labels}` dicts — see the detection note below), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device) and `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. **A DETECTION batch needs NO consumer collate, and that was measured (2026-08-05):** `collate_records` already batches one correctly — a variable-size image column stays a per-image LIST (`ToTensor` emits a live tensor as a PLAIN value, and plain values are gathered, never stacked) and a variable-N `Regions` keeps per-record COLUMNS (its `boxes`/`labels` are declared attrs, which the collate lists rather than stacking). So the only piece that was missing is the inverse, `batch_regions`, which is why it lives here: a detection consumer was carrying ~90 lines of its own collate (a `DetectionBatchInput` container, a `Regions`->`{boxes,labels}` unwrap, a metadata transpose) that re-derived exactly these rules, and it was deleted in favour of `collate_records` + `batch_regions` + `batch_metadata`. `batch_regions` is FRAMEWORK-FREE like its neighbours (torch stays torch, numpy stays numpy — a target's dtype/device is the caller's contract), omits a field the item left `None` (so a training target is exactly `{boxes, labels}`), and leaves `canvas`/`extras` on the batched item (per-image frame metadata and an open dict are not per-box columns). **A variable-size `Image` ITEM column is batched by the `"list"` collate** — the default still raises for it, deliberately and with an explanatory message, because a caller who asked for stacking should hear that it could not happen rather than silently receive a different type. +- **Collation Is a Pluggable Registry, And The BATCH SHAPE Is A CHOICE (`recordstream.collate`; the choice landed 2026-08-05):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`. **TWO collates ship, differing in ONE decision — whether array payloads are STACKED — because that decision belongs to the MODEL, not to the data:** `"record"` (the default, `collate_records`) stacks what can stack; `"list"` (`collate_list` = `collate_records(items, stack=False)`) stacks nothing and leaves every key a per-record list with ITEMS KEPT AS ITEMS (so an `Image`'s `layout` survives per record). Left implicit the shape is decided by ACCIDENT — a column stacks if it holds array items and stays a list if it holds PLAIN values, so `ToTensor` running or not silently decides whether a detector gets its `List[Tensor]`; the key lets a consumer DECLARE it (raidar's torch loaders pass `collate_fn=collate_list`, `RecordSequence` takes `collate=`). **Every read-back helper accepts BOTH shapes** (`batch_values` unwraps a list-of-items element-wise, `batch_regions` reads a batched `Regions` or a list of them, `batch_metadata` transposes either) — that is what makes the collate a free choice rather than a fork in every consumer, and it is pinned by a parametrized test over both keys. **A stack failure now EXPLAINS itself** (`_stack_or_explain`): it names the key, the differing shapes and the `"list"` way out, where the raw `ValueError: all input arrays must have the same shape` from inside numpy named none of the three — this closed the old TASKS item about `_stack`'s unreachable "else a list" promise, by making the fallback a DECLARED mode rather than a silent type change. **`register_collate` is signature-PRESERVING** (`TypeVar` bound to `CollateFn`, not a flat `-> CollateFn`), so registering a collate no longer erases its own parameters — that is what lets `collate_list` call `collate_records(items, stack=False)` and type-check. The default `collate_records` behaviour is unchanged: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. **The registry is ENGINE-INTERNAL — only shape-generic, parameter-free collates register (decided 2026-08-06, closing root TASKS item 41 as option (b)):** a TASK's batch shape (a fastai `(x, y)` tuple, a keras array pair) is parameterized by task-decided state (input/target keys, int-id vs multi-hot, a channels-last transpose) that a registry string cannot carry, so a consumer passes its callable straight to the slot that takes one (`DataLoader(collate_fn=...)`, `RecordSequence(transform=...)`) and never registers it — measured before deciding: no workspace consumer ever had. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_regions(batch, key)` (a collated `Regions` column transposed into per-record `{boxes, labels}` dicts — see the detection note below), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device), `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising) and `per_record_predictions(preds)` (the transpose's PREDICTION-side twin, 2026-08-06: a model's batched output — a list, a `{"predictions": [...]}` wrapper, or a batched mapping with agreeing lengths — sliced into ONE entry per record for the per-record sink contract; extracted from three byte-identical consumer copies). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. **A DETECTION batch needs NO consumer collate, and that was measured (2026-08-05):** `collate_records` already batches one correctly — a variable-size image column stays a per-image LIST (`ToTensor` emits a live tensor as a PLAIN value, and plain values are gathered, never stacked) and a variable-N `Regions` keeps per-record COLUMNS (its `boxes`/`labels` are declared attrs, which the collate lists rather than stacking). So the only piece that was missing is the inverse, `batch_regions`, which is why it lives here: a detection consumer was carrying ~90 lines of its own collate (a `DetectionBatchInput` container, a `Regions`->`{boxes,labels}` unwrap, a metadata transpose) that re-derived exactly these rules, and it was deleted in favour of `collate_records` + `batch_regions` + `batch_metadata`. `batch_regions` is FRAMEWORK-FREE like its neighbours (torch stays torch, numpy stays numpy — a target's dtype/device is the caller's contract), omits a field the item left `None` (so a training target is exactly `{boxes, labels}`), and leaves `canvas`/`extras` on the batched item (per-image frame metadata and an open dict are not per-box columns). **A variable-size `Image` ITEM column is batched by the `"list"` collate** — the default still raises for it, deliberately and with an explanatory message, because a caller who asked for stacking should hear that it could not happen rather than silently receive a different type. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. @@ -47,7 +47,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. - **A Source NAMES The Data It Reads — `dataset_uri` + `dataset_url`, And A View Propagates Them VERBATIM (`recordstream.uri`, 2026-08-02):** `SupportsDatasetIdentity` is a `@runtime_checkable` Protocol with TWO properties, and the pair is deliberate rather than one field doing double duty: **`dataset_uri`** is the CANONICAL handle (machine-parseable, stable across machines, the string two runs are COMPARED on — `hf://datasets/ylecun/mnist?split=train`, matching the convention hosted tracking services already use for a dataset source) while **`dataset_url`** is a link a PERSON opens and is `None` whenever the data has no web page. Collapsing them forces a choice between a browsable string that lies about local data and a canonical one nobody can click. `HuggingFaceSource` implements both: a Hub repo id -> `hf://datasets/?…`, a local directory -> its `file://` URI, decided by `Path(path).exists()` (the same question `load_dataset` answers); the query params are SORTED so one configuration has exactly ONE string, and both read STORED CONFIG ONLY — asking never loads, so a source that is never iterated still names itself (`tests/test_dataset_uri.py::test_asking_for_identity_never_loads_the_dataset`). **`revision` is a DECLARED ctor param BECAUSE identity reads it** (it was in the removed `**kwargs`), and `load_options` merges it over `load_kwargs` as a read-only property rather than in `__init__` — a declared key may be set post-construction, and a dict assembled in the constructor would keep the value the object was born with. **Following happens in the FREE FUNCTIONS, not in every wrapper:** `dataset_uri(x)` / `dataset_url(x)` flow a deferred `!class:` marker (as `project` does) and then follow a `.source` attribute when the object holds no handle — depth-capped (`MAX_WRAPPER_DEPTH`) and cycle-safe — so `Stream` / `DatasetSplit` / `_SplitView` / `RangeSource` / `MetadataFilterSource` all work with ZERO code of their own, as does any third-party wrapper using that attribute name. **A wrapper must NOT decorate the URI it passes up** (no `#train`, no `#0:1000`): the handle identifies the DATASET, how much of it a run consumed is already recorded by the wrapper's own configuration, and decorating would mean one dataset reached two ways stops comparing equal — the single property the handle exists to have, and what makes a consumer's dedup exact. **`ConcatSource` answers `None` ON PURPOSE** — several datasets end to end are not one dataset, and picking a member would be a lie; `dataset_uris(source)` is the plural form that fans out over `.sources` (recursive, deduplicated). `None` is an ordinary answer everywhere (unconfigured source, in-memory stream, data with no page), never an error. Adding identity to a new source is TWO properties and no registration. Package-root exports + `__all__`; no entry point (the module holds no `@configurable`, same as `projection.py`). Rationale: `docs/architecture.md` §13. Usage: `docs/sources.md` → "Identifying a dataset". Pins: `tests/test_dataset_uri.py`. -- **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset`; pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise). +- **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset` — and since 2026-08-06 the composition itself ships as **`prepare_record_dataset(source)`** = `ensure_materialized(ensure_record_dataset(source))` with `None` passing through (the `_prepare` helper every training runnable had re-composed; a consumer needing only one half still calls that half). **The torch loader triple ships beside it (`recordstream.loaders.loader_slots`, 2026-08-06):** the train/val/test `LazyClass(DataLoader, ...)` slots every torch training runnable declared identically, returned as the `LoaderSlots` NamedTuple (`slots.train`/`.val`/`.test`, typed `Lazy[DataLoader[Any]]`; train shuffled, eval not, `persistent_workers` derived from `num_workers`, `collate_fn=` the batch-shape choice, further DataLoader kwargs passing through to all three — `shuffle` refused as a shared kwarg because it is the one per-split decision the helper owns). The module is torch-ONLY and deliberately NOT package-root exported (the `ops.torch` pattern — a torch-free import of it raises an `ImportError` naming the extra); consumers import `from recordstream.loaders import loader_slots`. Config-side, the three runnable slots stay whole-value replaceable in YAML exactly as with the inline construction this replaces. Pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise), `tests/test_loaders.py`. - **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). diff --git a/docs/kinds.md b/docs/kinds.md index 4d3d793..1f7789d 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -101,6 +101,7 @@ batch_tensor(batch, "image", device=model.device) # ONE torch tensor, sta batch_tensor(batch, "class", dev, dtype=torch.int64) # ...with the dtype your loss requires multi_hot(batch, "class", num_classes) # a MultiLabel column as an [N, C] numpy matrix batch_metadata(batch, exclude=("image", "class")) # the remaining columns transposed into N dicts +per_record_predictions(model_output) # a BATCHED model output sliced into one entry per record ``` Only `batch_tensor` is torch; the rest return plain values or numpy, so a non-torch backend reuses them and converts in one line. `dtype` is a parameter, not an opinion — the same knob as `device`. What stays task-side is only WHICH call a trainer makes. @@ -149,3 +150,23 @@ class SlidingWindow: Expansion is flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(stream)` / `stream[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(stream)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Stream` engine. + +## The training-side data helpers (`prepare_record_dataset`, `recordstream.loaders`) + +Two helpers every training runnable composes, written once here: + +```python +from recordstream import prepare_record_dataset +from recordstream.loaders import loader_slots # torch-only module, not in the package root + +dataset = prepare_record_dataset(source) # ensure_record_dataset + ensure_materialized; None passes through +slots = loader_slots(batch_size=32, num_workers=0) # deferred DataLoader triple (train shuffled) +loader = flow(slots.train, dataset=dataset) # the dataset arrives at run time +``` + +`prepare_record_dataset` normalizes a wired source's TYPE and its STATE in the calling process +(the fork-safety pair — see the `ensure_materialized` docs). `loader_slots` returns the +train/val/test `LazyClass(DataLoader, ...)` markers as a named tuple; further `DataLoader` +kwargs pass through to all three, and a config can still replace any individual loader slot +wholesale. The module imports torch, so it is deliberately NOT re-exported from the package +root — `import recordstream` stays framework-free. diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 6a2d94c..4ef1fea 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -10,7 +10,14 @@ """ # --- shared infrastructure ----------------------------------------------------------------- -from recordstream.batch import batch_metadata, batch_regions, batch_tensor, batch_values, multi_hot +from recordstream.batch import ( + batch_metadata, + batch_regions, + batch_tensor, + batch_values, + multi_hot, + per_record_predictions, +) from recordstream.collate import ( collate, collate_list, @@ -27,6 +34,7 @@ WrappedOp, ensure_materialized, ensure_record_dataset, + prepare_record_dataset, register_op_family, registered_op_families, ) @@ -86,6 +94,7 @@ from recordstream.runnable import ( ProgressCallback, ProgressReporting, + RunnableTask, TorchRunner, entrypoint, entrypoint_tasks, @@ -135,6 +144,7 @@ "RecordSource", "ensure_materialized", "ensure_record_dataset", + "prepare_record_dataset", "FilterOp", "WrappedOp", "register_op_family", @@ -146,6 +156,7 @@ "batch_tensor", "batch_values", "multi_hot", + "per_record_predictions", "collate_list", "collate_records", "get_collate", @@ -188,6 +199,7 @@ "TorchRunner", "ProgressReporting", "ProgressCallback", + "RunnableTask", "entrypoint", "entrypoint_tasks", "run_entrypoint", diff --git a/recordstream/batch.py b/recordstream/batch.py index 6b26eca..dcae3ea 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -40,7 +40,7 @@ if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only from torch import Tensor -__all__ = ["batch_metadata", "batch_regions", "batch_tensor", "batch_values", "multi_hot"] +__all__ = ["batch_metadata", "batch_regions", "batch_tensor", "batch_values", "multi_hot", "per_record_predictions"] #: The per-box PARALLEL ARRAY fields of a :class:`~recordstream.Regions`, in the order a #: per-record dict presents them. ``canvas`` and ``extras`` are deliberately absent: the first is @@ -234,6 +234,39 @@ def batch_tensor(batch: Record, key: str, device: Any = None, dtype: Any = None) return tensor if device is None else tensor.to(device) +def per_record_predictions(preds: Any) -> List[Any]: + """Split a model's BATCHED prediction output into ONE entry per record. + + The sink contract (``write(prediction, metadata)``) is per-record, but a model emits the + whole batch at once — a classifier's ``probs`` ``[N, C]``, a restorer's ``image`` + ``[N, C, H, W]``. This is the pairing half of prediction, the same transpose + :func:`batch_metadata` performs for the metadata columns, which is why it lives beside it: + a consumer re-deriving the slicing re-derives what a batched output IS. It was extracted + from three byte-identical private copies in the training projects (2026-08-06). + + Handled shapes, in order: + + * a **list** — already per-record (a detector's per-image dicts), returned as-is; + * a ``{"predictions": [...]}`` **wrapper** — that list (a detection convention); + * a **batched mapping** whose values agree on one length ``N`` (a + :class:`~recordstream.ClassificationOutput` and friends) — sliced row-wise into ``N`` + mappings; + * anything else — treated as ONE prediction, wrapped in a singleton list. Handing an + unrecognised batch over whole (the pre-extraction bug) wrote ONE entry for an N-record + batch, with the sink reading row 0 as if it were the whole prediction. + """ + if isinstance(preds, list): + return preds + if isinstance(preds, dict): + if "predictions" in preds: + return list(preds["predictions"]) + lengths = {len(v) for v in preds.values() if hasattr(v, "__len__")} + if len(lengths) == 1: + n = lengths.pop() + return [{k: v[i] for k, v in preds.items()} for i in range(n)] + return [preds] + + def batch_metadata(batch: Record, exclude: Iterable[str] = ()) -> Optional[List[Dict[str, Any]]]: """Per-record metadata dicts recovered from a batched record — the collate's transpose. diff --git a/recordstream/collate.py b/recordstream/collate.py index 72b0682..f2674d8 100644 --- a/recordstream/collate.py +++ b/recordstream/collate.py @@ -2,14 +2,11 @@ Batching in recordstream is two-stage: the engine groups carriers (``Stream.batch`` / ``FlowGraph.batch`` yield ``list``\\ s of N items) and a COLLATE function stacks a group -into one batched carrier. This registry gives consumer packages ONE addressable home for -their task collates — consumers ``register_collate`` their task collates additively, and -callers dispatch by key or by the default record collate. - -The string keys primarily serve AI-callable (MCP) tool surfaces, which pass -JSON-serializable names — never function objects — and enumerate the legal values via -:func:`registered_collates`; in Python (and in YAML via a dotted ``!ref:`` to the -function), passing a collate function directly remains the normal path. +into one batched carrier. The registry keys serve surfaces that pass JSON-serializable +NAMES rather than function objects (an AI-callable tool, ``RecordSequence``'s ``collate=`` +string form), enumerating the legal values via :func:`registered_collates`; in Python (and +in YAML via a dotted ``!ref:`` to the function), passing a collate function directly is the +normal path. TWO collates are registered here, and they differ in ONE decision — whether array payloads are STACKED — because that decision belongs to the model, not to the data: @@ -19,7 +16,15 @@ * ``"list"`` — nothing stacked; every key becomes a per-record list, items kept as items. What a model taking variable-size inputs wants (a torchvision detector's ``List[Tensor]``). -Consumer conventions beyond that are deliberately NOT unified here; the registry is additive. +**The registry is engine-internal — only shape-generic, parameter-free collates register +(decided 2026-08-06, closing the open question).** A TASK's batch shape (a fastai +``(x, y)`` tuple, a keras ``(inputs, targets)`` array pair) is parameterized by task-decided +state — which keys are input and target, an int-id vs multi-hot target, a channels-last +transpose — that a bare registry string cannot carry: a registered ``"tuple"`` with baked +default keys would be the silent-wrong-keys trap. Consumers therefore pass their callable +straight to the slot that takes one (``DataLoader(collate_fn=...)``, +``RecordSequence(transform=...)``) and never register it. Measured before deciding: no +workspace consumer had ever registered one. """ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar @@ -53,15 +58,17 @@ def register_collate(key: str) -> Callable[[F], F]: - """Register a collate function under ``key`` (a task alias). + """Register a collate function under ``key``. - Usable as a decorator:: + For SHAPE-GENERIC, parameter-free collates only (see the module docstring) — a + task-shaped collate carries task state a key cannot, and is passed as a callable to + the slot that takes one instead of being registered. Usable as a decorator:: - @register_collate("yolo") - def yolo_collate(items): ... + @register_collate("record") + def collate_records(items): ... - Re-registering a key overwrites it (logged at debug — consumers may deliberately - replace a default). + Re-registering a key overwrites it (logged at debug — a deliberate replacement of a + default is legal). """ def _register(fn: F) -> F: diff --git a/recordstream/core/__init__.py b/recordstream/core/__init__.py index 6f06ac6..b386627 100644 --- a/recordstream/core/__init__.py +++ b/recordstream/core/__init__.py @@ -51,6 +51,7 @@ ensure_materialized, ensure_record_dataset, linear_steps, + prepare_record_dataset, ) from recordstream.core.wrappers import FilterOp, WrappedOp @@ -66,6 +67,7 @@ "ensure_materialized", "ensure_record_dataset", "linear_steps", + "prepare_record_dataset", "register_op_family", "registered_op_families", ] diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py index f34f43e..5364226 100644 --- a/recordstream/core/stream.py +++ b/recordstream/core/stream.py @@ -500,3 +500,23 @@ def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) # dataset"). Widening that annotation with a Protocol breaks `to_pydantic` for every # Stream, so the exception is documented here instead. See TASKS.md. return Stream(source=cast(Iterable[Any], source)) + + +def prepare_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> Optional["Stream"]: + """Normalize a wired dataset slot's TYPE and its STATE, in THIS process. + + The composition every forking consumer writes: + :func:`ensure_record_dataset` normalizes the TYPE (any wired source into a record-yielding + map-style ``Stream``) and :func:`ensure_materialized` normalizes the STATE (one whole record + read here, so a lazy source is built in the CALLING process rather than in a forked + ``DataLoader`` worker — where a first read SIGSEGVs through ``_scproxy`` / CoreFoundation on + macOS with no Python traceback; the full account lives on :func:`ensure_materialized`). + + ``None`` passes through, so an unwired optional split (``val_set``) needs no guard at the + call site. This function exists because the pair was re-composed identically in every + training runnable across the workspace ("``_prepare``"); a consumer needing only one half + still calls that half directly. + """ + if source is None: + return None + return cast("Stream", ensure_materialized(ensure_record_dataset(source))) diff --git a/recordstream/loaders.py b/recordstream/loaders.py new file mode 100644 index 0000000..397bb0e --- /dev/null +++ b/recordstream/loaders.py @@ -0,0 +1,101 @@ +"""Deferred torch ``DataLoader`` slots for a training runnable — the torch half of batching. + +Batching has two halves (see ``docs/architecture.md`` §10): WHAT a batch contains (a collate) +and WHICH ROWS go in which batch (order, slicing, the short final batch, the per-epoch +reshuffle). torch's second half is the ``DataLoader``, and every training runnable in the +workspace baked the same three deferred loader slots into its constructor — train shuffled, +val/test not, one shared kwarg set. :func:`loader_slots` is that construction written once. + +The slots are :class:`confluid.LazyClass` markers, not live loaders, on purpose: the lazy-init +mandate forbids functional work in a constructor, and the dataset does not exist yet — the run +method flows each slot with ``dataset=`` at run time (``flow(self.train_loader, dataset=ds)``). + +**This module is torch-only and deliberately NOT re-exported from the package root** (the +``recordstream.ops.torch`` pattern): ``import recordstream`` keeps pulling no ML framework, and +a consumer — which is by definition a torch trainer — imports it directly:: + + from recordstream.loaders import loader_slots +""" + +from typing import Any, Callable, List, NamedTuple + +from confluid import Lazy, LazyClass + +from recordstream.collate import collate_records +from recordstream.items import Record + +try: + from torch.utils.data import DataLoader +except ImportError as exc: # pragma: no cover - exercised only on a torch-free install + raise ImportError( + "recordstream.loaders needs torch (it declares DataLoader slots) — install `recordstream[torch]`." + ) from exc + +__all__ = ["LoaderSlots", "loader_slots"] + + +class LoaderSlots(NamedTuple): + """The three deferred loader markers, addressed by split (``slots.train`` / ``.val`` / ``.test``).""" + + train: Lazy[DataLoader[Any]] + val: Lazy[DataLoader[Any]] + test: Lazy[DataLoader[Any]] + + +def loader_slots( + batch_size: int, + num_workers: int, + *, + collate_fn: Callable[[List[Record]], Record] = collate_records, + **loader_kw: Any, +) -> LoaderSlots: + """The train/val/test deferred ``DataLoader`` triple every torch training runnable declares. + + Configurability is TWO-CHANNELLED, and this helper narrows neither: + + * **From code** — any further ``DataLoader`` kwarg (``pin_memory``, ``drop_last``, + ``prefetch_factor``, a ``worker_init_fn``) passes through ``**loader_kw`` and is baked + into all three markers. + * **From config** — the runnable's ``train_loader`` / ``val_loader`` / ``test_loader`` + slots stay whole-value replaceable in YAML (``train_loader: !class:torch.utils.data.DataLoader + {shuffle: false, pin_memory: true, ...}``), exactly as with the inline construction this + replaces; the run method only injects ``dataset=`` at flow time, so every knob a replaced + slot sets survives. + + Args: + batch_size: Rows per batch, baked into all three loaders. + num_workers: Worker processes per loader. ``persistent_workers`` derives from it + (``num_workers != 0``) — there is deliberately no separate knob, because persistent + workers with zero workers is a torch error and the pairing never varies. + collate_fn: The batch-shape choice (see the collate registry) — ``collate_records`` + stacks, ``collate_list`` does not (what a detection consumer passes), and a task + collate is any callable. + **loader_kw: Further ``DataLoader`` kwargs, baked into ALL THREE markers. ``shuffle`` + is refused here because it is the ONE per-split kwarg this helper owns (train + ``True``, eval ``False``) — a different split policy is a whole-slot replacement, + not a shared kwarg. + + Returns: + A :class:`LoaderSlots` named tuple — three ``LazyClass(DataLoader, ...)`` markers + (``slots.train`` with ``shuffle=True``, ``slots.val`` / ``slots.test`` with + ``shuffle=False``). Assign them to the runnable's ``train_loader`` / ``val_loader`` / + ``test_loader`` slots and flow each with ``dataset=`` at run time. + """ + if "shuffle" in loader_kw: + raise ValueError( + "loader_slots: 'shuffle' is per-split (train shuffles, val/test do not) and cannot be " + "a shared kwarg — replace the individual loader slot instead " + "(e.g. train_loader: !class:torch.utils.data.DataLoader {shuffle: false, ...})." + ) + shared = dict( + collate_fn=collate_fn, + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=num_workers != 0, + **loader_kw, + ) + return LoaderSlots( + train=LazyClass(DataLoader, shuffle=True, **shared), + val=LazyClass(DataLoader, shuffle=False, **shared), + test=LazyClass(DataLoader, shuffle=False, **shared), + ) diff --git a/recordstream/runnable.py b/recordstream/runnable.py index 6d7b47f..0764ea3 100644 --- a/recordstream/runnable.py +++ b/recordstream/runnable.py @@ -35,7 +35,7 @@ example (the class + the exact introspector outputs): ``docs/runnable.md``. """ -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Literal, Optional from loggair import get_logger @@ -44,6 +44,14 @@ #: Attribute stamped on a method by :func:`entrypoint`. _ENTRYPOINT_ATTR = "__runnable_entrypoint__" +#: The four standard tasks of a merged train+eval runnable — the ``task`` values its +#: ``@entrypoint`` markers declare and its ``run()`` dispatches on. Declared HERE because they are +#: this module's vocabulary (the markers/dispatch above speak them); a consumer's own task alias +#: is typically ``MyTask = RunnableTask`` so the closed set is written exactly once. A runnable +#: with a narrower or wider capability set declares its own Literal instead — the type names the +#: CONVENTION, it does not gate dispatch (``run_entrypoint`` reads the markers, not this). +RunnableTask = Literal["fit", "evaluate", "test", "predict"] + #: A progress sink: ``(value, total, description) -> None``. ``value`` / ``total`` are #: floats in the same unit (optimizer steps, records); ``description`` is a short stage label. ProgressCallback = Callable[[float, float, str], None] @@ -211,6 +219,7 @@ def run_entrypoint(runnable: object, task: str) -> Any: __all__ = [ "ProgressCallback", "ProgressReporting", + "RunnableTask", "TorchRunner", "entrypoint", "entrypoint_tasks", diff --git a/tests/test_batch.py b/tests/test_batch.py index c3964ca..eb9cb79 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -352,3 +352,49 @@ def test_every_read_back_helper_accepts_both_collates(self, collate_key: str) -> targets = batch_regions(batch, "target") assert [t["boxes"].shape[0] for t in targets] == [1, 3] assert batch_metadata(batch, exclude=("image", "target")) == [{"class": 0}, {"class": 1}] + + +# --------------------------------------------------------------------------- # +# per_record_predictions — the prediction-side transpose +# --------------------------------------------------------------------------- # + + +class TestPerRecordPredictions: + """Batched model output -> one entry per record, for the per-record sink contract. + + Ported from the consumer suites when the three byte-identical private copies were + extracted (2026-08-06) — these are the cases those copies exercised. + """ + + def test_a_list_is_already_per_record(self) -> None: + from recordstream import per_record_predictions + + items = [{"boxes": [1]}, {"boxes": [2]}] + assert per_record_predictions(items) is items + + def test_a_predictions_wrapper_is_unwrapped(self) -> None: + from recordstream import per_record_predictions + + assert per_record_predictions({"predictions": [1, 2, 3]}) == [1, 2, 3] + + def test_a_batched_mapping_is_sliced_row_wise(self) -> None: + """A ClassificationOutput-shaped batch becomes N per-record mappings.""" + from recordstream import per_record_predictions + + batch = {"probs": np.eye(3, dtype="float32"), "class_idx": np.array([0, 1, 2])} + rows = per_record_predictions(batch) + assert len(rows) == 3 + assert rows[1]["class_idx"] == 1 + assert np.allclose(rows[2]["probs"], [0.0, 0.0, 1.0]) + + def test_a_mapping_with_disagreeing_lengths_is_one_prediction(self) -> None: + """Row-slicing is only safe when every column agrees on N.""" + from recordstream import per_record_predictions + + batch = {"a": np.zeros(2), "b": np.zeros(3)} + assert per_record_predictions(batch) == [batch] + + def test_anything_else_is_one_prediction(self) -> None: + from recordstream import per_record_predictions + + assert per_record_predictions(1.5) == [1.5] diff --git a/tests/test_loaders.py b/tests/test_loaders.py new file mode 100644 index 0000000..35bee1b --- /dev/null +++ b/tests/test_loaders.py @@ -0,0 +1,85 @@ +"""The deferred torch DataLoader triple (``recordstream.loaders.loader_slots``). + +Two claims: the slots ARE the construction every torch training runnable used to bake inline +(train shuffled, eval not, one shared kwarg set, ``persistent_workers`` derived), and the +module is torch-ONLY by design — deliberately not reachable from the package root, so +``import recordstream`` keeps pulling no ML framework. +""" + +import ast +from pathlib import Path + +import pytest +from confluid.fluid import Fluid + +import recordstream +from recordstream import Stream, collate_list, collate_records + +torch = pytest.importorskip("torch") + + +def _kwargs(marker: object) -> dict: + """A slot holds a LazyClass MARKER pre-flow; its stored kwargs are what these tests pin.""" + assert isinstance(marker, Fluid) + return marker.kwargs + + +from confluid import flow # noqa: E402 +from torch.utils.data import DataLoader # noqa: E402 + +from recordstream.loaders import LoaderSlots, loader_slots # noqa: E402 + + +def test_the_triple_is_split_addressed_with_the_shuffle_split() -> None: + slots = loader_slots(batch_size=4, num_workers=0) + assert isinstance(slots, LoaderSlots) + assert _kwargs(slots.train)["shuffle"] is True + assert _kwargs(slots.val)["shuffle"] is False + assert _kwargs(slots.test)["shuffle"] is False + + +def test_the_shared_kwargs_are_baked_and_persistent_workers_derives() -> None: + for split, marker in zip(("train", "val", "test"), loader_slots(batch_size=8, num_workers=0)): + assert _kwargs(marker)["batch_size"] == 8, split + assert _kwargs(marker)["collate_fn"] is collate_records, split + assert _kwargs(marker)["persistent_workers"] is False, split + assert _kwargs(loader_slots(batch_size=8, num_workers=2).train)["persistent_workers"] is True + + +def test_the_collate_is_the_batch_shape_choice() -> None: + """A detection consumer passes collate_list; the slot carries it verbatim.""" + assert ( + _kwargs(loader_slots(batch_size=2, num_workers=0, collate_fn=collate_list).train)["collate_fn"] is collate_list + ) + + +def test_further_loader_kwargs_pass_through_to_all_three() -> None: + """The code channel for the rest of the DataLoader surface (pin_memory, drop_last, ...).""" + slots = loader_slots(batch_size=2, num_workers=0, drop_last=True) + assert all(_kwargs(marker)["drop_last"] is True for marker in slots) + + +def test_shuffle_is_refused_as_a_shared_kwarg() -> None: + """It is the one PER-SPLIT kwarg the helper owns; the error names the way out.""" + with pytest.raises(ValueError, match="per-split"): + loader_slots(batch_size=2, num_workers=0, shuffle=False) + + +def test_a_slot_flows_into_a_working_loader() -> None: + """The whole point of the deferral: `flow(slot, dataset=...)` at run time yields batches.""" + stream = Stream(source=[{"x": float(i)} for i in range(4)]) + loader = flow(loader_slots(batch_size=2, num_workers=0).val, dataset=stream) + assert isinstance(loader, DataLoader) + batches = list(loader) + assert len(batches) == 2 + assert [float(v) for v in batches[0]["x"]] == [0.0, 1.0] + + +def test_the_module_is_not_reachable_from_the_package_root() -> None: + """It imports torch at module level, so the root must neither import nor advertise it — + that is what keeps `import recordstream` framework-free (the ops.torch pattern).""" + assert "loader_slots" not in recordstream.__all__ + root_source = (Path(recordstream.__file__)).read_text() + for node in ast.walk(ast.parse(root_source)): + if isinstance(node, ast.ImportFrom): + assert node.module != "recordstream.loaders", "the package root must not import recordstream.loaders" diff --git a/tests/test_record_source.py b/tests/test_record_source.py index afa1a46..d66540a 100644 --- a/tests/test_record_source.py +++ b/tests/test_record_source.py @@ -154,3 +154,40 @@ def __iter__(self) -> Iterator[Dict[str, Any]]: source = _IterableOnly() ensure_materialized(source) assert source.built + + +# --------------------------------------------------------------------------- # +# prepare_record_dataset — the TYPE + STATE composition +# --------------------------------------------------------------------------- # + + +def test_prepare_none_passes_through() -> None: + """An unwired optional split needs no guard at the call site.""" + from recordstream import prepare_record_dataset + + assert prepare_record_dataset(None) is None + + +def test_prepare_wraps_and_warms_in_one_call() -> None: + """The composition IS ensure_record_dataset + ensure_materialized: a lazy source comes + back as a record Stream whose build already happened in THIS process.""" + from recordstream import Stream, prepare_record_dataset + + source = _LazySource(rows=2) + prepared = prepare_record_dataset(source) + assert isinstance(prepared, Stream) + assert source.built, "the lazy build must happen here, not in a forked worker" + + +def test_prepare_returns_a_stream_as_the_same_stream() -> None: + """Identity matters: a label-encoding Stream keeps its class_names.""" + from recordstream import Stream, prepare_record_dataset + + stream = Stream(source=[{"class": 1}]) + assert prepare_record_dataset(stream) is stream + + +def test_prepare_is_exported_from_the_package_root() -> None: + import recordstream + + assert "prepare_record_dataset" in recordstream.__all__ From 9d8426a10ff8361650ad041b95921bd9fe563fd1 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 8 Aug 2026 15:10:50 +0200 Subject: [PATCH 079/102] docs: use namespaced HuggingFace repo ids (ylecun/mnist) in the source examples Current huggingface_hub rejects namespace-less ids (HfUriError); worse, a stale local cache makes a bare id appear to work on one machine and fail on a fresh one. docs/sources.md now states the rule beside the first example. --- docs/architecture.md | 2 +- docs/graph.md | 4 ++-- docs/sources.md | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index bdd9731..62f8832 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1084,7 +1084,7 @@ searches `__init__.py` and raises `OSError: could not find class definition`. ```yaml # canonical — what a generator emits, matching cls.__module__ train_set: !class:recordstream.sources.huggingface.HuggingFaceSource - path: mnist + path: ylecun/mnist split: train my_split: !class:recordstream.sources.split.DatasetSplit() diff --git a/docs/graph.md b/docs/graph.md index 0293184..28939f6 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -81,7 +81,7 @@ too. from recordstream import FlowGraph, Stream from recordstream.sources import HuggingFaceSource -graph = FlowGraph.from_yaml("graph.yaml", source=HuggingFaceSource(path="mnist")) +graph = FlowGraph.from_yaml("graph.yaml", source=HuggingFaceSource(path="ylecun/mnist")) for record in graph: ... @@ -116,7 +116,7 @@ source: from recordstream import Stream from recordstream.sources import HuggingFaceSource -stream = Stream.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="mnist")) +stream = Stream.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="ylecun/mnist")) ``` The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so diff --git a/docs/sources.md b/docs/sources.md index 1d71f65..bd3eee2 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -25,12 +25,14 @@ but a generated one will use the submodule spelling. Rationale: ```yaml hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() - path: mnist + path: ylecun/mnist input_feature: image target_feature: label metadata_features: ["*"] # keep every other column as its own record entry (the default) ``` +> **Use the namespaced repo id** (`ylecun/mnist`, never the legacy bare `mnist`): current `huggingface_hub` rejects namespace-less ids (`HfUriError: Repository id must be 'namespace/name'`, measured 2026-08-06). Worse than the hard failure is the soft one — with a stale local cache present, `datasets` logs "couldn't be found on the Hugging Face Hub" and silently loads the cached copy, so a bare id can appear to work on one machine and fail on a fresh one. + > **Lazy & zero-arg construction** — `HuggingFaceSource` follows the workspace lazy-init convention: the constructor does no work (no network), so `HuggingFaceSource()` is valid and building one is free. The dataset is downloaded only on first access to the read-only `.dataset` property (cached thereafter; reset `_dataset` to reload), and `.resolved_metadata_features` (the `"*"` expansion) is derived lazily from the loaded columns. `path` is therefore optional at construction and validated lazily — accessing `.dataset` with an empty `path` raises a clear `ValueError`. ## Train / val / test splitting (`DatasetSplit`) @@ -49,7 +51,7 @@ The views are disjoint and complementary, computed once over a single determinis ```yaml hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() - path: mnist + path: ylecun/mnist split: train my_split: !class:recordstream.sources.split.DatasetSplit() From 5ae61746e99875fb9af55fc459b830d9d51fd363 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 10 Aug 2026 11:42:37 +0200 Subject: [PATCH 080/102] =?UTF-8?q?chore:=20gitignore=20runs/=20=E2=80=94?= =?UTF-8?q?=20the=20one=20workspace-wide=20run-output=20root?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every run artifact (tracker metrics, checkpoints, exports, framework logs) now writes under runs/, so a project needs exactly one ignored output directory. See the workspace AGENTS.md mandate 'Run Output Lands In A Gitignored runs/'. --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 2d29224..f1d2810 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,10 @@ coverage.xml test-report.xml coverage/ /pyrightconfig.json + +# Run output — every run (training, evaluation, prediction, sweep cell) writes here. +# ONE ignored root per project: matrainer trackers default to `save_dir: ./runs`, and +# Lightning checkpoints follow the tracker's save_dir, so a whole run lands under one +# directory instead of scattering run-named trees into the repository root. +# See the workspace AGENTS.md mandate "Run Output Lands In A Gitignored `runs/`". +runs/ From 0dd6725a28e7a560bea511dadf386f391dd07d11 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 10 Aug 2026 13:20:48 +0200 Subject: [PATCH 081/102] =?UTF-8?q?feat!:=20Regions=20is=20now=20Boxes=20?= =?UTF-8?q?=E2=80=94=20pixel-only,=20half-open=20xyxy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The structured box item carried two coordinate systems in one field (pixel [x0,y0,x1,y1] or signal [f0,f1,t0,t1]) with no discriminator. It is now Boxes and PIXEL-ONLY by contract; the signal-domain region item moved to the signal package, registered through the same open item registry. No back-compat alias — a stored typedrecord-v1 'Regions' tag fails loudly. - connected_component_bboxes -> connected_component_boxes, renamed WITH its contract: half-open xyxy (x=col, y=row) instead of inclusive row/col tuples; ConnectedComponents emits Boxes with canvas set (empty masks included); masks_to_detection's reconciling transpose is gone - batch_regions -> batch_boxes, _REGION_FIELDS -> _BOX_FIELDS - geometry-desync guards renamed; they now correctly stay silent for a domain package's raster-independent region item - docs: record-model example is a clean pixel one; architecture record #14 captures the split + the ConnectedComponents normalization --- AGENTS.md | 12 ++-- README.md | 2 +- docs/architecture.md | 74 ++++++++++++++++++++++-- docs/augmentation.md | 18 +++--- docs/kinds.md | 6 +- docs/record-model.md | 60 +++++++++---------- recordstream/__init__.py | 8 +-- recordstream/batch.py | 48 +++++++-------- recordstream/collate.py | 2 +- recordstream/core/families.py | 32 +++++----- recordstream/io.py | 2 +- recordstream/items.py | 24 ++++---- recordstream/ops/__init__.py | 2 +- recordstream/ops/image.py | 23 ++++---- recordstream/ops/numpy.py | 38 +++++++----- recordstream/ops/target.py | 48 +++++++-------- recordstream/storage/base.py | 2 +- tests/_fixtures.py | 22 +++---- tests/test_batch.py | 52 ++++++++--------- tests/test_convert_to_mask.py | 12 ++-- tests/test_dispatch.py | 6 +- tests/test_io.py | 4 +- tests/test_items.py | 12 ++-- tests/test_op_families.py | 14 ++--- tests/test_structure_ops.py | 6 +- tests/test_transform.py | 8 +-- tests/test_typed_collate.py | 8 +-- tests/test_typed_detection_target_ops.py | 39 ++++++++----- tests/test_typed_generic_ops.py | 69 +++++++++++++--------- tests/test_typed_storage.py | 6 +- 30 files changed, 380 insertions(+), 279 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index afaaf80..d7a983c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,10 +22,10 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — and since 2026-08-05 the COLLATE ITSELF is selectable here too (`collate=` takes a registered key or a function, resolved per batch so a key registered later still works), because a `PyDataset` otherwise had no way to say "don't stack" and the batch-shape choice was torch-only — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Regions` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Regions`/`Label`). `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Regions`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. -- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). **A geometry-changing transform WARNS when a `Regions` sat out the call (2026-08-06):** the vocabulary rule above is what makes a bare library transform work unmodified, but a detection target rides as a `Regions` item under a key of the pipeline's choosing — so it is not in that vocabulary, is not passed, and does not move. Measured: a bare `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes on `[10, 10, 100, 100]`, and a bare `A.HorizontalFlip` mirrors the pixels while changing NO shape at all. Nothing errors either way — the shapes stay valid and only the coordinates become wrong. **The condition is the LIBRARY'S OWN taxonomy, not a raster comparison**: a `DualTransform` is by definition one that applies to boxes, an `ImageOnlyTransform` cannot touch geometry, so `_has_spatial_transform` matches `DualTransform` by MRO class NAME (import-free, like `_is_albumentations`) and recurses a `Compose` through `.transforms` — the flip case proves why a "did the size change" test is not enough, and reading the taxonomy means no list of transform names to drift. WARNING not error (a record may legitimately carry regions describing something else), once per transform TYPE (`_WARNED_SPATIAL` — the message is about the configuration), and silent when `bboxes` WAS passed. The message names both ways out: `bbox_params` on a `Compose`, or `ops.target.ResizeDetection` for a plain coupled resize. Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (incl. `TestGeometryLeavingRegionsBehind`, which asserts the desync PREMISE before the guard). **The v2 family has the SAME gap by the other route and the same guard (2026-08-06):** v2 walks the record natively but transforms only its own `tv_tensors` TYPES, so a `Regions` is passed through untouched — measured, `v2.Resize((64,64))` takes a 200x200 image to 64x64 with the boxes still on `[10,10,100,100]`, while the same transform over a `tv_tensors.BoundingBoxes` gives `[3.2, 3.2, 32, 32]`. `_is_v2_geometry` matches v2's private `torchvision.transforms.v2._geometry` module in the MRO (the library offers no public marker) and shares `_WARNED_SPATIAL` with the albumentations guard. **The behavioural test — apply the transform to a throwaway `BoundingBoxes` and see if they move — is deliberately NOT used: it would draw from the RNG and change the augmentation stream of the run being diagnosed.** The private path can go stale on a torchvision upgrade; it FAILS OPEN (no warning, nothing else changes), so `test_the_geometry_signal_still_matches_this_torchvision` asserts the CLASSIFICATION directly rather than only through a warning that would silently stop appearing (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Boxes` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Boxes`/`Label`). **`Boxes` is PIXEL-ONLY (renamed from `Regions`, 2026-08-10, NO alias): half-open absolute-pixel `[x0, y0, x1, y1]` rows on the `(H, W)` `canvas` raster — the signal-domain time/frequency region item lives in the signal package and registers through the same `register_item` registry; a stored `typedrecord-v1` record carrying `__item_type__: "Regions"` fails loudly on decode and is re-generated (rationale: `docs/architecture.md` §14). `connected_component_boxes` (renamed WITH its contract from `connected_component_bboxes`) emits the SAME half-open xyxy order, `ConnectedComponents` fills `canvas` (empty masks included), and the geometry guards now correctly stay silent for a domain package's raster-independent region item.** `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Boxes, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Boxes`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). **A geometry-changing transform WARNS when a `Boxes` sat out the call (2026-08-06):** the vocabulary rule above is what makes a bare library transform work unmodified, but a detection target rides as a `Boxes` item under a key of the pipeline's choosing — so it is not in that vocabulary, is not passed, and does not move. Measured: a bare `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes on `[10, 10, 100, 100]`, and a bare `A.HorizontalFlip` mirrors the pixels while changing NO shape at all. Nothing errors either way — the shapes stay valid and only the coordinates become wrong. **The condition is the LIBRARY'S OWN taxonomy, not a raster comparison**: a `DualTransform` is by definition one that applies to boxes, an `ImageOnlyTransform` cannot touch geometry, so `_has_spatial_transform` matches `DualTransform` by MRO class NAME (import-free, like `_is_albumentations`) and recurses a `Compose` through `.transforms` — the flip case proves why a "did the size change" test is not enough, and reading the taxonomy means no list of transform names to drift. WARNING not error (a record may legitimately carry regions describing something else), once per transform TYPE (`_WARNED_SPATIAL` — the message is about the configuration), and silent when `bboxes` WAS passed. The message names both ways out: `bbox_params` on a `Compose`, or `ops.target.ResizeDetection` for a plain coupled resize. Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (incl. `TestGeometryLeavingBoxesBehind`, which asserts the desync PREMISE before the guard). **The v2 family has the SAME gap by the other route and the same guard (2026-08-06):** v2 walks the record natively but transforms only its own `tv_tensors` TYPES, so a `Boxes` is passed through untouched — measured, `v2.Resize((64,64))` takes a 200x200 image to 64x64 with the boxes still on `[10,10,100,100]`, while the same transform over a `tv_tensors.BoundingBoxes` gives `[3.2, 3.2, 32, 32]`. `_is_v2_geometry` matches v2's private `torchvision.transforms.v2._geometry` module in the MRO (the library offers no public marker) and shares `_WARNED_SPATIAL` with the albumentations guard. **The behavioural test — apply the transform to a throwaway `BoundingBoxes` and see if they move — is deliberately NOT used: it would draw from the RNG and change the augmentation stream of the run being diagnosed.** The private path can go stale on a torchvision upgrade; it FAILS OPEN (no warning, nothing else changes), so `test_the_geometry_signal_still_matches_this_torchvision` asserts the CLASSIFICATION directly rather than only through a warning that would silently stop appearing (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **OpenCV's Thread Pool Is The SECOND Fork Hazard, And The Guard Fires On USE (`core.families._disable_cv2_threading`, 2026-08-02):** the companion to the `ensure_materialized` mandate below, and **independent of it — neither fixes the other**. albumentations runs on OpenCV, whose thread pool is not fork-safe: once the PARENT has executed a cv2 op, a forked `DataLoader` worker inheriting that pool dies with a **SIGSEGV and no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` — the same symptom as the lazy-source hazard, which is exactly why they get confused for each other. Measured 2026-08-02 with `DataLoader(num_workers=2)` over a `Stream` whose ops list held an `A.Resize`: **with the source warmed and the pool ON the worker still SIGSEGVs, and with the pool off a cold source is still built in the child.** A consumer that forks needs BOTH guards. `cv2.setNumThreads(0)` is fired ONCE at the top of `_invoke_albumentations` — the one place this package INVOKES albumentations — and never at import: a process that never uses albumentations must not have its OpenCV settings changed by importing a data library, and cv2 must not become an import-time dependency of the engine. It costs nothing where it matters, because inside a worker the WORKER is the parallelism and cv2's own threads oversubscribe rather than help (albumentations' own docs recommend exactly this for multiprocessing loaders). Two cv2 quirks a test must not get wrong, both measured: `setNumThreads(0)` makes `getNumThreads()` report **1**, not 0; and `setNumThreads(4)` does not change what it reports at all. **Why the classification consumer never hit this and the segmentation one did:** classification resizes with `ConvertToImage` (PIL), while a per-pixel task must resize the image and its mask in ONE JOINT DRAW — which only a bare albumentations transform does. Segmentation is the first thing in the workspace to put cv2 on the worker path. Pins: `tests/test_fork_safety.py` (the forked-worker subprocess reproduction is the one that matters — removing the guard makes it FAIL, not pass differently). -- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Regions` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. **A `Regions`' `canvas` is the load-bearing case of that rule, and EVERY op that makes or re-frames one fills it in (2026-08-06):** a box means nothing without the raster it is stated in, so `CocoToTorchVisionDetection` records the frame of the image its annotation describes, `MasksToDetectionBoxes` the shape of the mask its boxes were derived from (exact and free), and `ResizeDetection` the size it resized to — **including for an EMPTY target**, because a check that silently skips exactly the records with nothing to check reports a clean bill for the wrong reason. Previously ONLY the resize set it, so the frame was known exactly where it was least needed and unknown in the chain where boxes and pixels actually drift apart. The lookup (`ops.target._source_frame`) is deliberately NARROW — the `"image"` key, then the first `Image` item, then `None` — because a generic "first 2-D array" search reads a `Regions`' own `[N, 4]` box array as an `N x 4` raster and records a confident lie (pinned). `None` stays an ordinary answer: nothing depends on the lookup succeeding. **The desync this makes detectable is otherwise SILENT:** an image-only resize (`ConvertToImage` and friends) moves pixels without moving boxes, every downstream SHAPE stays valid, only the coordinates are wrong, and a model trains against misplaced targets reporting nothing — so `ConvertToImage` WARNS once per op instance when it resizes a record carrying a `Regions`, naming `ResizeDetection`. Once per INSTANCE, not per record: the message is about the configuration, and a copy per record only buries it. **`ops.image.image_frame(value)` is the shared "what raster is this" read** (PIL `.size` transposed, an `Image` item's DECLARED `layout`, else HWC) — it does NOT sniff a channel axis, because this package already carries three deliberately divergent channels-first heuristics and a guessing fourth would mislabel the very frame box coordinates are validated against. Consumer note: a downstream frame check that skipped on `canvas is None` now fires in chains it used to pass (verified on a real COCO-style set — the recorded frame matches the dataset's own `width`/`height` columns). Pins: `tests/test_convert_to_mask.py::TestBoxesKnowTheirFrame` (incl. the box-array-is-not-a-raster case and the once-per-op warning). NOTE for any test asserting on that warning: loggair is loguru, so `caplog` stays EMPTY, and its sink is ENQUEUED so `capfd` alone races it — add a sink and call `logger.complete()` (the mandated flush, never a sleep). Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Boxes` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. **A `Boxes`' `canvas` is the load-bearing case of that rule, and EVERY op that makes or re-frames one fills it in (2026-08-06):** a box means nothing without the raster it is stated in, so `CocoToTorchVisionDetection` records the frame of the image its annotation describes, `MasksToDetectionBoxes` the shape of the mask its boxes were derived from (exact and free), and `ResizeDetection` the size it resized to — **including for an EMPTY target**, because a check that silently skips exactly the records with nothing to check reports a clean bill for the wrong reason. Previously ONLY the resize set it, so the frame was known exactly where it was least needed and unknown in the chain where boxes and pixels actually drift apart. The lookup (`ops.target._source_frame`) is deliberately NARROW — the `"image"` key, then the first `Image` item, then `None` — because a generic "first 2-D array" search reads a `Boxes`' own `[N, 4]` box array as an `N x 4` raster and records a confident lie (pinned). `None` stays an ordinary answer: nothing depends on the lookup succeeding. **The desync this makes detectable is otherwise SILENT:** an image-only resize (`ConvertToImage` and friends) moves pixels without moving boxes, every downstream SHAPE stays valid, only the coordinates are wrong, and a model trains against misplaced targets reporting nothing — so `ConvertToImage` WARNS once per op instance when it resizes a record carrying a `Boxes`, naming `ResizeDetection`. Once per INSTANCE, not per record: the message is about the configuration, and a copy per record only buries it. **`ops.image.image_frame(value)` is the shared "what raster is this" read** (PIL `.size` transposed, an `Image` item's DECLARED `layout`, else HWC) — it does NOT sniff a channel axis, because this package already carries three deliberately divergent channels-first heuristics and a guessing fourth would mislabel the very frame box coordinates are validated against. Consumer note: a downstream frame check that skipped on `canvas is None` now fires in chains it used to pass (verified on a real COCO-style set — the recorded frame matches the dataset's own `width`/`height` columns). Pins: `tests/test_convert_to_mask.py::TestBoxesKnowTheirFrame` (incl. the box-array-is-not-a-raster case and the once-per-op warning). NOTE for any test asserting on that warning: loggair is loguru, so `caplog` stays EMPTY, and its sink is ENQUEUED so `capfd` alone races it — add a sink and call `logger.complete()` (the mandated flush, never a sleep). Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. - **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.execute.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). - **1→N Expanding Steps Fork the REMAINING Subgraph (2026-07-30, supersedes the flat-engine pending-queue rule):** An op carrying `EXPANDS = True` yields N children from one record; the remaining steps then run ONCE PER CHILD over that child's own shallow copy of the step environment (independent name→result maps, shared values), DEPTH-FIRST so sibling order matches the nested-loop intuition. An empty expansion or a `None` child drops that branch. This works in EVERY route — serial, spawn-parallel (the worker returns a LIST), and inside a `flow:` graph (the old `FlowGraph` raised `NotImplementedError` on an expanding step; that limit is gone). CONSEQUENCES: (1) `__len__`/`__getitem__` RAISE on `Stream` AND `FlowGraph` when any step op expands — the expanded index map is unknowable up front, so the pipeline is ITERABLE-ONLY (iterate, wrap in a torch IterableDataset, window at the SOURCE for random access, or `list(...)`); (2) `run_steps` (the strict 1→1 twin used for indexing) raises rather than silently dropping siblings. Pins: `tests/test_typed_flow.py::TestExpandingSteps`. @@ -33,7 +33,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core.families._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). -- **Collation Is a Pluggable Registry, And The BATCH SHAPE Is A CHOICE (`recordstream.collate`; the choice landed 2026-08-05):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`. **TWO collates ship, differing in ONE decision — whether array payloads are STACKED — because that decision belongs to the MODEL, not to the data:** `"record"` (the default, `collate_records`) stacks what can stack; `"list"` (`collate_list` = `collate_records(items, stack=False)`) stacks nothing and leaves every key a per-record list with ITEMS KEPT AS ITEMS (so an `Image`'s `layout` survives per record). Left implicit the shape is decided by ACCIDENT — a column stacks if it holds array items and stays a list if it holds PLAIN values, so `ToTensor` running or not silently decides whether a detector gets its `List[Tensor]`; the key lets a consumer DECLARE it (raidar's torch loaders pass `collate_fn=collate_list`, `RecordSequence` takes `collate=`). **Every read-back helper accepts BOTH shapes** (`batch_values` unwraps a list-of-items element-wise, `batch_regions` reads a batched `Regions` or a list of them, `batch_metadata` transposes either) — that is what makes the collate a free choice rather than a fork in every consumer, and it is pinned by a parametrized test over both keys. **A stack failure now EXPLAINS itself** (`_stack_or_explain`): it names the key, the differing shapes and the `"list"` way out, where the raw `ValueError: all input arrays must have the same shape` from inside numpy named none of the three — this closed the old TASKS item about `_stack`'s unreachable "else a list" promise, by making the fallback a DECLARED mode rather than a silent type change. **`register_collate` is signature-PRESERVING** (`TypeVar` bound to `CollateFn`, not a flat `-> CollateFn`), so registering a collate no longer erases its own parameters — that is what lets `collate_list` call `collate_records(items, stack=False)` and type-check. The default `collate_records` behaviour is unchanged: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. **The registry is ENGINE-INTERNAL — only shape-generic, parameter-free collates register (decided 2026-08-06, closing root TASKS item 41 as option (b)):** a TASK's batch shape (a fastai `(x, y)` tuple, a keras array pair) is parameterized by task-decided state (input/target keys, int-id vs multi-hot, a channels-last transpose) that a registry string cannot carry, so a consumer passes its callable straight to the slot that takes one (`DataLoader(collate_fn=...)`, `RecordSequence(transform=...)`) and never registers it — measured before deciding: no workspace consumer ever had. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_regions(batch, key)` (a collated `Regions` column transposed into per-record `{boxes, labels}` dicts — see the detection note below), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device), `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising) and `per_record_predictions(preds)` (the transpose's PREDICTION-side twin, 2026-08-06: a model's batched output — a list, a `{"predictions": [...]}` wrapper, or a batched mapping with agreeing lengths — sliced into ONE entry per record for the per-record sink contract; extracted from three byte-identical consumer copies). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. **A DETECTION batch needs NO consumer collate, and that was measured (2026-08-05):** `collate_records` already batches one correctly — a variable-size image column stays a per-image LIST (`ToTensor` emits a live tensor as a PLAIN value, and plain values are gathered, never stacked) and a variable-N `Regions` keeps per-record COLUMNS (its `boxes`/`labels` are declared attrs, which the collate lists rather than stacking). So the only piece that was missing is the inverse, `batch_regions`, which is why it lives here: a detection consumer was carrying ~90 lines of its own collate (a `DetectionBatchInput` container, a `Regions`->`{boxes,labels}` unwrap, a metadata transpose) that re-derived exactly these rules, and it was deleted in favour of `collate_records` + `batch_regions` + `batch_metadata`. `batch_regions` is FRAMEWORK-FREE like its neighbours (torch stays torch, numpy stays numpy — a target's dtype/device is the caller's contract), omits a field the item left `None` (so a training target is exactly `{boxes, labels}`), and leaves `canvas`/`extras` on the batched item (per-image frame metadata and an open dict are not per-box columns). **A variable-size `Image` ITEM column is batched by the `"list"` collate** — the default still raises for it, deliberately and with an explanatory message, because a caller who asked for stacking should hear that it could not happen rather than silently receive a different type. +- **Collation Is a Pluggable Registry, And The BATCH SHAPE Is A CHOICE (`recordstream.collate`; the choice landed 2026-08-05):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`. **TWO collates ship, differing in ONE decision — whether array payloads are STACKED — because that decision belongs to the MODEL, not to the data:** `"record"` (the default, `collate_records`) stacks what can stack; `"list"` (`collate_list` = `collate_records(items, stack=False)`) stacks nothing and leaves every key a per-record list with ITEMS KEPT AS ITEMS (so an `Image`'s `layout` survives per record). Left implicit the shape is decided by ACCIDENT — a column stacks if it holds array items and stays a list if it holds PLAIN values, so `ToTensor` running or not silently decides whether a detector gets its `List[Tensor]`; the key lets a consumer DECLARE it (raidar's torch loaders pass `collate_fn=collate_list`, `RecordSequence` takes `collate=`). **Every read-back helper accepts BOTH shapes** (`batch_values` unwraps a list-of-items element-wise, `batch_boxes` reads a batched `Boxes` or a list of them, `batch_metadata` transposes either) — that is what makes the collate a free choice rather than a fork in every consumer, and it is pinned by a parametrized test over both keys. **A stack failure now EXPLAINS itself** (`_stack_or_explain`): it names the key, the differing shapes and the `"list"` way out, where the raw `ValueError: all input arrays must have the same shape` from inside numpy named none of the three — this closed the old TASKS item about `_stack`'s unreachable "else a list" promise, by making the fallback a DECLARED mode rather than a silent type change. **`register_collate` is signature-PRESERVING** (`TypeVar` bound to `CollateFn`, not a flat `-> CollateFn`), so registering a collate no longer erases its own parameters — that is what lets `collate_list` call `collate_records(items, stack=False)` and type-check. The default `collate_records` behaviour is unchanged: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. **The registry is ENGINE-INTERNAL — only shape-generic, parameter-free collates register (decided 2026-08-06, closing root TASKS item 41 as option (b)):** a TASK's batch shape (a fastai `(x, y)` tuple, a keras array pair) is parameterized by task-decided state (input/target keys, int-id vs multi-hot, a channels-last transpose) that a registry string cannot carry, so a consumer passes its callable straight to the slot that takes one (`DataLoader(collate_fn=...)`, `RecordSequence(transform=...)`) and never registers it — measured before deciding: no workspace consumer ever had. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_boxes(batch, key)` (a collated `Boxes` column transposed into per-record `{boxes, labels}` dicts — see the detection note below), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device), `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising) and `per_record_predictions(preds)` (the transpose's PREDICTION-side twin, 2026-08-06: a model's batched output — a list, a `{"predictions": [...]}` wrapper, or a batched mapping with agreeing lengths — sliced into ONE entry per record for the per-record sink contract; extracted from three byte-identical consumer copies). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. **A DETECTION batch needs NO consumer collate, and that was measured (2026-08-05):** `collate_records` already batches one correctly — a variable-size image column stays a per-image LIST (`ToTensor` emits a live tensor as a PLAIN value, and plain values are gathered, never stacked) and a variable-N `Boxes` keeps per-record COLUMNS (its `boxes`/`labels` are declared attrs, which the collate lists rather than stacking). So the only piece that was missing is the inverse, `batch_boxes`, which is why it lives here: a detection consumer was carrying ~90 lines of its own collate (a `DetectionBatchInput` container, a `Boxes`->`{boxes,labels}` unwrap, a metadata transpose) that re-derived exactly these rules, and it was deleted in favour of `collate_records` + `batch_boxes` + `batch_metadata`. `batch_boxes` is FRAMEWORK-FREE like its neighbours (torch stays torch, numpy stays numpy — a target's dtype/device is the caller's contract), omits a field the item left `None` (so a training target is exactly `{boxes, labels}`), and leaves `canvas`/`extras` on the batched item (per-image frame metadata and an open dict are not per-box columns). **A variable-size `Image` ITEM column is batched by the `"list"` collate** — the default still raises for it, deliberately and with an explanatory message, because a caller who asked for stacking should hear that it could not happen rather than silently receive a different type. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. @@ -52,11 +52,11 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Regions` via the shared `connected_component_bboxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` / `ResizeDetection` (`recordstream.ops.target` — the first two emit a `Regions` detection target, lazy-importing torch: one from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the other from a segmentation MASK; `ResizeDetection` is the COUPLED image+boxes resize for fixed-input-size detectors — PIL/uint8 image to `(height, width)` + the `Regions` boxes scaled by the same factors, torch staying torch, `canvas` updated — run it BEFORE any float conversion such as `ToTensor`, and omit it for detectors that resize internally)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Boxes` via the shared `connected_component_boxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` / `ResizeDetection` (`recordstream.ops.target` — the first two emit a `Boxes` detection target, lazy-importing torch: one from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the other from a segmentation MASK; `ResizeDetection` is the COUPLED image+boxes resize for fixed-input-size detectors — PIL/uint8 image to `(height, width)` + the `Boxes` boxes scaled by the same factors, torch staying torch, `canvas` updated — run it BEFORE any float conversion such as `ToTensor`, and omit it for detectors that resize internally)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. - **Generic MASK Conversion Lives Here Too — `ConvertToMask` (2026-08-02):** the segmentation counterpart of `ConvertToImage` and the same op SHAPE (read one field, write a differently-typed item under `output`): a mask-bearing field (an ndarray, a torch tensor, or the PIL image a source handed over) becomes an **`int64` `[H, W]` `Mask`** of per-pixel class ids — what a segmentation dataset actually ships (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map) turned into what a per-pixel loss consumes. It belongs HERE, not in a segmentation project: "a mask PNG's pixels are class ids" mentions no modality (the `Threshold` → `Mask` precedent), and a consumer owning it would be the third package to write the conversion. **It converts and NOTHING else, deliberately** — remapping the ids is `FormulaOp` over its output (`formula: a - 1` for a 1-based trimap) or `EncodeTarget` for a lookup table; resizing/augmenting it TOGETHER WITH THE IMAGE is a bare albumentations transform in the same ops list; dropping the source column is `DropField`. Do NOT grow it an `offset` / `mapping` / `dtype` knob: each one restates an op that already exists. **`output` defaults to `"mask"` and that is load-bearing, not a nicety** — it is albumentations' own key vocabulary (`_ALB_KEYS`), so the engine's op-family dispatch hands `image` AND `mask` to ONE call and a single joint draw moves both with the `Mask` type surviving the re-wrap (measured; an image-only transform like `Normalize` still touches the image alone). **`int64` is not a knob either:** a class-id map is integer by definition and it is what `torch.nn.CrossEntropyLoss` requires (*"expected target dtype to be Long or Byte, but got Int"*); a library that casts on the way past — albumentations returns int32 — is corrected at the MODEL boundary by `batch_tensor(..., dtype=...)`, where the caller names the contract (the `dtype`-is-a-parameter rule). It reads through **`item_value`, never `item_data`**, because a source that does not know a column is a mask ships it as a `Label` (`HuggingFaceSource` does this for every metadata column) — see the record-model mandate. Singleton axes are squeezed (`[H,W,1]` / `[1,H,W]` → `[H,W]`); an **RGB-encoded mask RAISES** rather than being collapsed, because picking one of three channels or decoding a palette is a decision the op must not make silently. Pins: `tests/test_convert_to_mask.py` (incl. the whole `preprocess` chain end to end, and that a `Normalize` leaves the mask untouched). Usage: `docs/image.md` → "Masks". - **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). -- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Regions`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Regions`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) +- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Boxes`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Boxes`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. diff --git a/README.md b/README.md index 19e34af..492b46f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `RecordS ## 🚀 Key Features -- **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Regions`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. +- **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Boxes`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. - **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. - **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. - **Graph pipelines:** readable [`flow:` documents](docs/graph.md) of named steps — `from:` forks, `merge_from:` merges, `bind:` feeds one step's value into another's parameter. An `ops:` list is the same engine's linear spelling; both parse to one step graph. diff --git a/docs/architecture.md b/docs/architecture.md index 62f8832..913cf85 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -49,7 +49,7 @@ families), "where does augmentation come from?" had three answers. Collapse to ONE carrier and ONE op engine: - **A record is a plain `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **typed - values** (`Image`/`Mask`/`Regions`/`Label`, base `NDArrayItem`; open registry `register_item`; + values** (`Image`/`Mask`/`Boxes`/`Label`, base `NDArrayItem`; open registry `register_item`; uniform payload accessors `item_data`/`with_data`). No container class, no roles, no `primary()`: **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"class"`), and a scalar side value is just another key. Metadata is attrs on the typed value (`Image.layout`, @@ -61,7 +61,7 @@ Collapse to ONE carrier and ONE op engine: (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream/dispatch.py`) apply to every handled value, `field=` pins one key. The second sanctioned shape — type-CHANGING ops (`Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, `ConnectedComponents`: - `Mask`→`Regions`, the target ops) — overrides `__call__`, resolves its source by an explicit + `Mask`→`Boxes`, the target ops) — overrides `__call__`, resolves its source by an explicit `field=` or the first value of the natural type, and raises a `ValueError` naming the record's keys on every miss. - **External libraries run AS-IS through the engine's op-family dispatch** @@ -103,10 +103,10 @@ Collapse to ONE carrier and ONE op engine: `ops:` list like any native op (deferred markers flow at route entry). - The albumentations vocabulary is load-bearing: a value augments only if it rides one of the library's key names — routing is an explicit `RenameField`, never engine magic. -- **Contracts that outlive refactors:** the `(row_min, row_max, col_min, col_max)` inclusive - integer bin-box order of `connected_component_bboxes` (a downstream back-projection reads - exactly that order); the `typedrecord-v1` tag + no-back-compat rule; the albumentations key - vocabulary; the `"module:qualname"` callable-path format (§4). +- **Contracts that outlive refactors:** the HALF-OPEN pixel xyxy `(x0, y0, x1, y1)` order of + `connected_component_boxes` (every `Boxes` producer emits it; a downstream back-projection + reads exactly that order); the `typedrecord-v1` tag + no-back-compat rule; the albumentations + key vocabulary; the `"module:qualname"` callable-path format (§4). - Anything that used the old container API must migrate — there are deliberately no aliases and no legacy read path. @@ -1277,3 +1277,65 @@ dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:/// - **Recognise a new wrapper shape**: `recordstream/uri.py` follows `.source`; a wrapper using a different attribute name implements the properties itself instead. - **Usage** is [docs/sources.md](sources.md#identifying-a-dataset). + +## 14. `Boxes` is pixel-only — the signal-domain region type moved out (2026-08-10) + +### Context + +The structured item was born as `Regions`, and its own docstring sanctioned TWO coordinate +systems in one field: pixel `[x0, y0, x1, y1]` rows on an image raster, or signal +`[f0, f1, t0, t1]` rows in time/frequency. Nothing on the item said which one a given +instance held — consumers disambiguated by record key and by which op produced the value. +Every mechanical consumer in this package (the detection target ops, `ResizeDetection`'s +scaling, the three geometry-desync guards, `canvas` itself — an `(H, W)` raster) assumed the +pixel reading; a `Regions` carrying signal coordinates satisfied `isinstance` checks written +for a contract it did not hold. A third convention hid inside the same type: +`connected_component_bboxes` emitted INCLUSIVE `(row_min, row_max, col_min, col_max)` bin +tuples, axis-swapped from every other producer, reconciled by a transpose at exactly one +call site. + +### Decision + +The item is **`Boxes`**, and it is **pixel-only**: half-open absolute-pixel +`[x0, y0, x1, y1]` rows (x rightward, y downward), `labels`/`scores`, the `(H, W)` `canvas` +frame, per-box `extras`. A domain package needing a different coordinate system registers its +OWN item through the SAME open `register_item` registry — exactly the modality-neutrality +story the item registry exists for; this engine keeps zero knowledge of it. +`connected_component_boxes` (renamed WITH its contract, so stale callers break loudly) now +emits the same half-open xyxy convention, `ConnectedComponents` fills `canvas` in (empty +masks included), and the one reconciling transpose in `masks_to_detection` is gone. There is +NO back-compat alias in either direction (the workspace rename convention): a stored +`typedrecord-v1` record carrying `__item_type__: "Regions"` fails loudly on decode and is +re-generated with a current sink. + +### Consequences + +- One name, one convention: `isinstance(value, Boxes)` now implies the pixel contract the + geometry guards and detection consumers were already assuming. +- The guards correctly go SILENT for a domain package's region item — physical-unit + coordinates are raster-independent, so "the pixels moved and the boxes did not" was a false + alarm for them all along. +- Every `Boxes` producer emits the same row shape; the `+1`/transpose fix-ups that existed + only to bridge `connected_component_bboxes`' divergent order are deleted rather than moved. +- Stored datasets from before the rename must be re-generated (loud `KeyError`/`TypeError` on + decode — the established `typedrecord-v1` no-back-compat rule). + +### Example + +```python +from recordstream import Boxes, Mask +from recordstream.ops.numpy import ConnectedComponents + +out = ConnectedComponents()({"mask": Mask(mask_2d)}) +out["boxes"] # Boxes(boxes=[(x0, y0, x1, y1), ...], canvas=mask_2d.shape) +mask_2d[y0:y1, x0:x1] # half-open: covers the component exactly +``` + +### What you may change (and where it's documented) + +- **Add a new pixel-box producer**: emit half-open xyxy and FILL `canvas` in (the + metadata-on-the-value mandate); the pins live in `tests/test_typed_generic_ops.py`. +- **A new coordinate system** is a NEW registered item in the owning domain package, never a + second meaning for `Boxes` — that is the mistake this record exists to prevent. +- **Usage** is [docs/record-model.md](record-model.md); the batch read-back is `batch_boxes` + ([docs/kinds.md](kinds.md)). diff --git a/docs/augmentation.md b/docs/augmentation.md index 82a42e1..4931d2b 100644 --- a/docs/augmentation.md +++ b/docs/augmentation.md @@ -71,12 +71,12 @@ out = Pipeline([flip])(record) # image + mask + bboxes flipped together, on Format handling (`pascal_voc` / `coco` / `yolo` / `albumentations`) is `BboxParams`' knob — the engine adds nothing on top. The detection-target ops (`CocoToTorchVisionDetection` / -`MasksToDetectionBoxes`) produce a `Regions` item for the training boundary; the plain +`MasksToDetectionBoxes`) produce a `Boxes` item for the training boundary; the plain `bboxes`/`labels` list keys are the augmentation-time form the library consumes. -For a plain deterministic resize of the `Regions` form there is `ResizeDetection` +For a plain deterministic resize of the `Boxes` form there is `ResizeDetection` (`recordstream.ops.target`) — the detection twin of the joint image+mask draw: it resizes the -image (PIL or uint8 array) to a fixed `(height, width)` AND scales the `Regions` boxes by the +image (PIL or uint8 array) to a fixed `(height, width)` AND scales the `Boxes` boxes by the same factors in one coupled step, recording the new frame in `canvas`. Fixed-input-size detectors need it; detectors that resize internally simply omit it. Run it BEFORE any float conversion (e.g. before `ToTensor`): @@ -90,7 +90,7 @@ ops: ### Boxes carry the frame they are stated in -A box is only meaningful against a raster, so a `Regions` records that raster in `canvas` — +A box is only meaningful against a raster, so a `Boxes` records that raster in `canvas` — `(H, W)` — and every op that makes or re-frames one fills it in: `CocoToTorchVisionDetection` from the image the annotation describes, `MasksToDetectionBoxes` from the mask the boxes were derived from, `ResizeDetection` from the size it resized to (including for an empty target, so a @@ -108,15 +108,15 @@ ops: Every shape downstream stays valid — only the coordinates are wrong — so a model trains happily against misplaced targets. `ConvertToImage` warns once per op when it resizes a record carrying a -`Regions`, and a consumer that must be certain compares `canvas` against the image itself +`Boxes`, and a consumer that must be certain compares `canvas` against the image itself (`recordstream.ops.image.image_frame` reads the `(H, W)` of either). Use `ResizeDetection`, which moves both. **A bare library transform has the same gap**, for the reason that makes the dispatch work: an -albumentations op receives exactly its own key vocabulary, and a `Regions` is not in it. So a bare +albumentations op receives exactly its own key vocabulary, and a `Boxes` is not in it. So a bare `A.Resize` resizes the image and leaves the boxes; a bare `A.HorizontalFlip` mirrors the pixels and leaves them — *without changing any shape at all*. The engine warns once per transform type -when a geometry-changing transform runs while a `Regions` sat out the call, deciding "geometry- +when a geometry-changing transform runs while a `Boxes` sat out the call, deciding "geometry- changing" by the library's own `DualTransform` / `ImageOnlyTransform` split (so `Normalize` and friends stay silent). Speak the library's vocabulary and it moves them for you, in the same draw: @@ -129,7 +129,7 @@ ops: ``` **torchvision v2 has it too, by the other route.** v2 walks the record natively but transforms -only its OWN `tv_tensors` types, and a `Regions` is not one — so `v2.Resize` moves the pixels and +only its OWN `tv_tensors` types, and a `Boxes` is not one — so `v2.Resize` moves the pixels and leaves the boxes, while the same transform over a `tv_tensors.BoundingBoxes` rescales them correctly. The engine warns once per transform type here as well, using v2's geometric-transform grouping so `ColorJitter` and `Normalize` stay silent. Carry boxes in v2's own type when you want @@ -141,7 +141,7 @@ from torchvision import tv_tensors record["boxes"] = tv_tensors.BoundingBoxes(boxes, format="XYXY", canvas_size=(h, w)) ``` -Either way, `ResizeDetection` remains the plain coupled resize over the `Regions` form. +Either way, `ResizeDetection` remains the plain coupled resize over the `Boxes` form. ## YAML — bare library transforms are ordinary `!class:` nodes diff --git a/docs/kinds.md b/docs/kinds.md index 1f7789d..bf87b5c 100644 --- a/docs/kinds.md +++ b/docs/kinds.md @@ -2,7 +2,7 @@ ## What an op processes — dispatch on value type -A **record** is a plain dict of typed values (`Image`, `Mask`, `Regions`, `Label`, … — see [record-model.md](record-model.md)). A native op is a `Transform`: it declares which value TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per record (`get_params`), then applies the matching kernel to every value whose type it handles, passing untouched values through: +A **record** is a plain dict of typed values (`Image`, `Mask`, `Boxes`, `Label`, … — see [record-model.md](record-model.md)). A native op is a `Transform`: it declares which value TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per record (`get_params`), then applies the matching kernel to every value whose type it handles, passing untouched values through: ```python from recordstream import Record, Transform, Image @@ -23,7 +23,7 @@ Because the parameters are sampled once and shared, an op that handles several t Two smaller shapes round it out: - **A plain function** becomes an op via `as_transform(fn, handles=(Image,), field="image")` — `field=` pins the op to one named key (still type-gated). -- **A type-changing op** — read one key, write a differently-typed item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: `Mask` → `Regions`) — subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. +- **A type-changing op** — read one key, write a differently-typed item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: `Mask` → `Boxes`) — subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. Bare library transforms (torchvision `transforms.v2` walking the dict natively, albumentations dispatching by keyword name) drop straight into any ops list **as-is** — the engine's op-family dispatch invokes each one the way its own library expects. See [record-model.md](record-model.md#mixing-libraries--as-is-no-adapters) and [augmentation.md](augmentation.md). @@ -72,7 +72,7 @@ it holds *plain* values, so whether an op like `ToTensor` ran ends up choosing f Two things make the choice free: * **every read-back helper accepts both shapes** — `batch_values` unwraps a list-of-items - element-wise, `batch_regions` reads a batched `Regions` or a list of them, `batch_metadata` + element-wise, `batch_boxes` reads a batched `Boxes` or a list of them, `batch_metadata` transposes either — so a consumer never branches on which collate ran; * **a stack failure explains itself**, naming the key, the differing shapes and the `"list"` way out, rather than surfacing numpy's bare *"all input arrays must have the same shape"*. diff --git a/docs/record-model.md b/docs/record-model.md index cbbf827..1b3628f 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -1,7 +1,7 @@ # The record model — THE recordstream data model A record is a **plain `dict`** of **typed values**. Import the whole surface from the PACKAGE TOP -LEVEL (`from recordstream import Record, Image, Mask, Regions, Label, Transform, Pipeline, +LEVEL (`from recordstream import Record, Image, Mask, Boxes, Label, Transform, Pipeline, as_transform, item_data, item_value, with_data, register_item, register_kernel, register_io, collate_records, ...`). The design rationale is recorded in [architecture.md](architecture.md#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25). @@ -42,11 +42,11 @@ recordstream is **modality-neutral**, so its core ships only generic items — i labels. (Domain items — a signal, a spectrogram — live in the domain package; see below.) ```python -from recordstream import Image, Mask, Regions, Label, MultiLabel +from recordstream import Image, Mask, Boxes, Label, MultiLabel Image(rgb_hwc, layout="HWC") # an image knows its layout ("HWC" default / "CHW") Mask(seg_hw) # a mask shares its image's frame -Regions(boxes=[[1,1,4,4]], labels=["drone"], canvas=(8, 10), extras={"snr_db": [12.5]}) +Boxes(boxes=[[10,10,40,40]], labels=[1], canvas=(64, 64)) # half-open pixel xyxy on a raster Label("drone_x", classes=["noise", "drone_x"]) # ONE class for this record MultiLabel(["drone_x", "jammer"], classes=[...]) # SEVERAL classes for this record ``` @@ -59,7 +59,7 @@ multi-label target from an ordinary sequence value that happens to sit under the Items are **hybrid**: array-backed items (`Image`, `Mask`) subclass `NDArrayItem` — an `np.ndarray` subclass whose declared `_item_attrs` survive numpy operations via `__array_finalize__` — so a -type-agnostic operation touches them as an array; structured items (`Regions`, `Label`, `MultiLabel`) +type-agnostic operation touches them as an array; structured items (`Boxes`, `Label`, `MultiLabel`) are dataclass wrappers (a bounding-box set is not an array). A uniform payload accessor hides the difference from kernels: @@ -111,7 +111,7 @@ def _brighten_image(value: Image, params: dict) -> Image: The second sanctioned op shape is the **type-changing op** — read one key, write a differently-typed item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: -`Mask` → `Regions`, the target ops). It subclasses `Transform` and overrides `__call__` instead of +`Mask` → `Boxes`, the target ops). It subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `handles` / `consumes` / `produces` truthfully as graph metadata (next section). @@ -148,12 +148,12 @@ the FFT ops), and diverge in two directions: ```python class JointFlip(Transform): - handles = (Image, Mask, Regions) # everything ONE draw may move + handles = (Image, Mask, Boxes) # everything ONE draw may move consumes = (Image,) # the only input it needs to be useful - optional = (Mask, Regions) # moved together with the image when present + optional = (Mask, Boxes) # moved together with the image when present ``` - A record with just an `Image` is fine; a record that also carries a `Mask`/`Regions` gets them + A record with just an `Image` is fine; a record that also carries a `Mask`/`Boxes` gets them moved consistently. Declaring `consumes = handles` here would wrongly tell a reader (or a pipeline linter) that a mask is required. @@ -165,8 +165,8 @@ the FFT ops), and diverge in two directions: ```python class ScaleBoxesToImage(Transform): - handles = (Regions,) # the only type it CHANGES - consumes = (Regions, Image) # ...but it cannot run without the reference Image (its shape) + handles = (Boxes,) # the only type it CHANGES + consumes = (Boxes, Image) # ...but it cannot run without the reference Image (its shape) ``` In short: `handles` = "what I write", `consumes` = "what must be present", `optional` = "what I @@ -180,48 +180,48 @@ constructor param per input slot** — defaulting to the conventional key name, validated lazily in `__call__`: ```python -class KeepRegionsOnMask(Transform): - """Drop regions whose center pixel is OFF in the activity mask. +class KeepBoxesOnMask(Transform): + """Drop boxes whose center pixel is OFF in the activity mask. Args: mask_field: Record key of the activity Mask to test against. Defaults to "mask". - regions_field: Record key of the Regions to filter. Defaults to "regions". - output: Key the filtered Regions are written to; blank (default) replaces regions_field in place. + boxes_field: Record key of the Boxes to filter. Defaults to "boxes". + output: Key the filtered Boxes are written to; blank (default) replaces boxes_field in place. """ - handles = (Regions,) # the only type it CHANGES - consumes = (Mask, Regions) # both inputs must be present - produces = (Regions,) + handles = (Boxes,) # the only type it CHANGES + consumes = (Mask, Boxes) # both inputs must be present + produces = (Boxes,) - def __init__(self, mask_field: str = "mask", regions_field: str = "regions", output: str = "") -> None: + def __init__(self, mask_field: str = "mask", boxes_field: str = "boxes", output: str = "") -> None: super().__init__() self.mask_field = mask_field - self.regions_field = regions_field + self.boxes_field = boxes_field self.output = output def __call__(self, record: Record) -> Record: - for name, want in ((self.mask_field, Mask), (self.regions_field, Regions)): + for name, want in ((self.mask_field, Mask), (self.boxes_field, Boxes)): if name not in record: raise ValueError(f"{type(self).__name__}: no {name!r} key in record (keys: {list(record)})") if not isinstance(record[name], want): raise TypeError(f"{type(self).__name__}: {name!r} is {type(record[name]).__name__}, expected {want.__name__}") - mask, regions = record[self.mask_field], record[self.regions_field] - keep = [b for b in regions.boxes if mask[int((b[1] + b[3]) / 2), int((b[0] + b[2]) / 2)]] - out = Regions(boxes=keep, labels=regions.labels, scores=regions.scores, canvas=regions.canvas) - return {**record, (self.output or self.regions_field): out} + mask, boxes = record[self.mask_field], record[self.boxes_field] + keep = [b for b in boxes.boxes if mask[int((b[1] + b[3]) / 2), int((b[0] + b[2]) / 2)]] + out = Boxes(boxes=keep, labels=boxes.labels, scores=boxes.scores, canvas=boxes.canvas) + return {**record, (self.output or self.boxes_field): out} ``` So a record carrying several masks and several region sets is disambiguated entirely in config — the op looks ONLY at the named entries: ```yaml -- !class:mypkg.KeepRegionsOnMask +- !class:mypkg.KeepBoxesOnMask mask_field: activity_mask # not the segmentation mask under "mask" - regions_field: predictions # not the ground truth under "regions" + boxes_field: predictions # not the ground truth under "boxes" ``` This is the established pattern for every shipped multi-input op (e.g. the region→target ops -take `image_field="image"` + `regions_field="regions"` + `output="target"`). Two rules keep it +take `image_field="image"` + `boxes_field="regions"` + `output="target"`). Two rules keep it predictable: the defaults are the CONVENTIONAL key names (so the common record shape needs zero config), and a wrong/missing key fails lazily in `__call__` with the key list in the message — never silently falls back to a different entry when an explicit name was given. @@ -466,7 +466,7 @@ rules (all records must share the same key set; a mismatch raises): 1. **array-backed item** → payloads stacked into one array/tensor with a leading batch dim, SAME item type back; each declared attr becomes a per-record list; -2. **wrapper item** (`Label`, `Regions`) → ONE item whose fields are per-record LISTS — deliberately +2. **wrapper item** (`Label`, `Boxes`) → ONE item whose fields are per-record LISTS — deliberately not auto-tensorized (turning class names into an `[N]` int64 tensor is the model boundary's one explicit step, not a generic-engine guess); 3. **plain value** → a plain list. @@ -515,13 +515,13 @@ same for its pixel-class mask. What stays task-side is only *which* call to make ### When the generic rules cannot work: register a task collate Stacking is task-shaped, and detection is the canonical failure: each record carries a DIFFERENT -number of boxes, and rule 2 can only give you `Regions(boxes=[<1 box>, <3 boxes>])` — per-record +number of boxes, and rule 2 can only give you `Boxes(boxes=[<1 box>, <3 boxes>])` — per-record lists no detection model accepts. A detection model family has its own batch contract (stacked images + RAGGED per-record target dicts), so the task package registers a collate that produces exactly that: ```python -from recordstream import Image, Regions, collate, register_collate +from recordstream import Image, Boxes, collate, register_collate @register_collate("detection") def detection_collate(items): diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 4ef1fea..41c4557 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -11,8 +11,8 @@ # --- shared infrastructure ----------------------------------------------------------------- from recordstream.batch import ( + batch_boxes, batch_metadata, - batch_regions, batch_tensor, batch_values, multi_hot, @@ -52,13 +52,13 @@ register_io, ) from recordstream.items import ( + Boxes, Image, Label, Mask, MultiLabel, NDArrayItem, Record, - Regions, get_item_type, is_class_id, is_item, @@ -112,7 +112,7 @@ "NDArrayItem", "Image", "Mask", - "Regions", + "Boxes", "Label", "MultiLabel", "is_class_id", @@ -152,7 +152,7 @@ "FlowGraph", "collate", "batch_metadata", - "batch_regions", + "batch_boxes", "batch_tensor", "batch_values", "multi_hot", diff --git a/recordstream/batch.py b/recordstream/batch.py index dcae3ea..2f0628a 100644 --- a/recordstream/batch.py +++ b/recordstream/batch.py @@ -12,7 +12,7 @@ * :func:`batch_values` — the raw values, past the wrapper item. Framework-free. * :func:`multi_hot` — a :class:`~recordstream.MultiLabel` column as an ``[N, C]`` matrix. Framework-free (numpy). -* :func:`batch_regions` — a :class:`~recordstream.Regions` column as per-record +* :func:`batch_boxes` — a :class:`~recordstream.Boxes` column as per-record ``{boxes, labels}`` dicts. Framework-free. * :func:`batch_tensor` — the torch adapter: stack, optional dtype, optional device. * :func:`batch_metadata` — the collate's transpose, for prediction sinks. Framework-free. @@ -35,18 +35,18 @@ import numpy as np -from recordstream.items import Record, Regions, is_item, item_value +from recordstream.items import Boxes, Record, is_item, item_value if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only from torch import Tensor -__all__ = ["batch_metadata", "batch_regions", "batch_tensor", "batch_values", "multi_hot", "per_record_predictions"] +__all__ = ["batch_boxes", "batch_metadata", "batch_tensor", "batch_values", "multi_hot", "per_record_predictions"] -#: The per-box PARALLEL ARRAY fields of a :class:`~recordstream.Regions`, in the order a +#: The per-box PARALLEL ARRAY fields of a :class:`~recordstream.Boxes`, in the order a #: per-record dict presents them. ``canvas`` and ``extras`` are deliberately absent: the first is #: per-IMAGE frame metadata and the second an open dict, neither of which is a per-box column — #: read them off the batched item itself (``batch[key].canvas`` is the per-record list). -_REGION_FIELDS = ("boxes", "labels", "scores") +_BOX_FIELDS = ("boxes", "labels", "scores") def batch_values(batch: Record, key: str) -> Any: @@ -122,10 +122,10 @@ def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") return out -def batch_regions(batch: Record, key: str) -> List[Dict[str, Any]]: - """A collated :class:`~recordstream.Regions` column back into PER-RECORD dicts. +def batch_boxes(batch: Record, key: str) -> List[Dict[str, Any]]: + """A collated :class:`~recordstream.Boxes` column back into PER-RECORD dicts. - The collate cannot stack a region set — every record has its own N — so it leaves each + The collate cannot stack a box set — every record has its own N — so it leaves each declared attr as a per-record LIST (``boxes`` = ``[[N0, 4], [N1, 4], …]``). That is the right batch, and it is also not what a model takes: every detection interface in use wants ONE dict per image. This is that transpose, and it belongs beside :func:`batch_metadata` @@ -141,9 +141,9 @@ def batch_regions(batch: Record, key: str) -> List[Dict[str, Any]]: Args: batch: A batched record — from EITHER collate (``"record"`` leaves one batched - :class:`~recordstream.Regions` with per-record columns; ``"list"`` leaves a list of - per-record ``Regions``; both are read here). - key: The record key holding the collated :class:`~recordstream.Regions`. + :class:`~recordstream.Boxes` with per-record columns; ``"list"`` leaves a list of + per-record ``Boxes``; both are read here). + key: The record key holding the collated :class:`~recordstream.Boxes`. Returns: One dict per record, carrying whichever of ``boxes`` / ``labels`` / ``scores`` that @@ -153,35 +153,35 @@ def batch_regions(batch: Record, key: str) -> List[Dict[str, Any]]: open dict are not per-box columns); read them off ``batch[key]``. Raises: - TypeError: when ``key`` does not hold a :class:`~recordstream.Regions`. + TypeError: when ``key`` does not hold a :class:`~recordstream.Boxes`. ValueError: when the item is not COLLATED (its ``boxes`` is not a per-record list) — - passing a single record's ``Regions`` here is the mistake the message names. + passing a single record's ``Boxes`` here is the mistake the message names. Example:: - targets = batch_regions(batch, "target") # [{"boxes": [N0, 4], "labels": [N0]}, …] + targets = batch_boxes(batch, "target") # [{"boxes": [N0, 4], "labels": [N0]}, …] targets = [{k: v.to(device) for k, v in t.items()} for t in targets] # a torch caller """ item = batch[key] - # The "list" collate leaves a LIST of per-record Regions; the default leaves ONE batched - # Regions whose attrs are per-record lists. Both mean the same thing, so both read the same + # The "list" collate leaves a LIST of per-record Boxes; the default leaves ONE batched + # Boxes whose attrs are per-record lists. Both mean the same thing, so both read the same # — a consumer never branches on which collate ran. if isinstance(item, list): - if not all(isinstance(entry, Regions) for entry in item): - raise TypeError(f"batch_regions: {key!r} holds a list whose entries are not all Regions.") + if not all(isinstance(entry, Boxes) for entry in item): + raise TypeError(f"batch_boxes: {key!r} holds a list whose entries are not all Boxes.") return [ - {name: getattr(entry, name) for name in _REGION_FIELDS if getattr(entry, name, None) is not None} + {name: getattr(entry, name) for name in _BOX_FIELDS if getattr(entry, name, None) is not None} for entry in item ] - if not isinstance(item, Regions): - raise TypeError(f"batch_regions: {key!r} holds {type(item).__name__}, not a Regions.") + if not isinstance(item, Boxes): + raise TypeError(f"batch_boxes: {key!r} holds {type(item).__name__}, not a Boxes.") if not isinstance(item.boxes, list): raise ValueError( - f"batch_regions: {key!r} is not a COLLATED Regions — its `boxes` is " + f"batch_boxes: {key!r} is not a COLLATED Boxes — its `boxes` is " f"{type(item.boxes).__name__}, not the per-record list collate_records leaves. " - "Pass the batched record, not a single record's Regions." + "Pass the batched record, not a single record's Boxes." ) - columns = {name: getattr(item, name) for name in _REGION_FIELDS if isinstance(getattr(item, name, None), list)} + columns = {name: getattr(item, name) for name in _BOX_FIELDS if isinstance(getattr(item, name, None), list)} return [ {name: values[index] for name, values in columns.items() if values[index] is not None} for index in range(len(item.boxes)) diff --git a/recordstream/collate.py b/recordstream/collate.py index f2674d8..2b403bd 100644 --- a/recordstream/collate.py +++ b/recordstream/collate.py @@ -193,7 +193,7 @@ def collate_list(items: Sequence[Record]) -> Record: raises — and the type does not depend on the data, because you asked for lists; * items stay ITEMS (a list of :class:`~recordstream.Image`, not a list of bare arrays), so per-record metadata survives. The read-back helpers (:func:`~recordstream.batch_values`, - :func:`~recordstream.batch_regions`, :func:`~recordstream.batch_metadata`) accept BOTH + :func:`~recordstream.batch_boxes`, :func:`~recordstream.batch_metadata`) accept BOTH shapes, so a consumer reads the batch the same way under either collate. Example:: diff --git a/recordstream/core/families.py b/recordstream/core/families.py index 51e6481..07959ed 100644 --- a/recordstream/core/families.py +++ b/recordstream/core/families.py @@ -11,7 +11,7 @@ from loggair import get_logger -from recordstream.items import NDArrayItem, Record, Regions, with_data +from recordstream.items import Boxes, NDArrayItem, Record, with_data logger = get_logger(__name__) @@ -148,7 +148,7 @@ def _disable_cv2_threading() -> None: logger.debug(f"could not disable OpenCV threading ({exc}); a forked DataLoader worker may crash.") -#: Transform classes already warned about (see :func:`_warn_if_regions_are_left_behind`). Keyed +#: Transform classes already warned about (see :func:`_warn_if_boxes_are_left_behind`). Keyed #: by CLASS, not instance: the message describes a configuration pattern, and two `Resize`s in #: one chain have the same thing wrong with them. _WARNED_SPATIAL: Set[type] = set() @@ -177,11 +177,11 @@ def _has_spatial_transform(op: Any, depth: int = 0) -> bool: return any(_has_spatial_transform(child, depth + 1) for child in children) -def _warn_if_regions_are_left_behind(record: Record, op: Any, passed: Dict[str, Any]) -> None: - """Warn once when a geometry-changing transform ran while a ``Regions`` sat out the call. +def _warn_if_boxes_are_left_behind(record: Record, op: Any, passed: Dict[str, Any]) -> None: + """Warn once when a geometry-changing transform ran while a ``Boxes`` sat out the call. This family passes the op EXACTLY the keys of albumentations' own vocabulary, which is what - lets a bare library transform work unmodified — but a detection target rides as a ``Regions`` + lets a bare library transform work unmodified — but a detection target rides as a ``Boxes`` item under a key of the pipeline's choosing, so it is not in that vocabulary and does not get passed. Measured: a bare ``A.Resize`` moves a 200x200 image to 64x64 and leaves the boxes on ``[10, 10, 100, 100]``; a bare ``A.HorizontalFlip`` mirrors the pixels and leaves the boxes @@ -190,24 +190,24 @@ def _warn_if_regions_are_left_behind(record: Record, op: Any, passed: Dict[str, Nothing errors either way: the shapes stay valid and only the coordinates become wrong, so a model trains against misplaced targets and reports nothing. It stays a WARNING rather than an - error because a record may legitimately carry regions describing something other than the + error because a record may legitimately carry boxes describing something other than the image being augmented — this family cannot know, and refusing the call would break a pipeline that is right. The fix is to speak the library's vocabulary: put boxes under ``bboxes`` with their ``labels`` and declare ``bbox_params`` on the ``Compose``, and the library moves them in the same joint draw. For a plain deterministic resize, ``ops.target.ResizeDetection`` does the - coupled step over the ``Regions`` form directly. + coupled step over the ``Boxes`` form directly. """ if "bboxes" in passed or type(op) in _WARNED_SPATIAL: return - keys = [key for key, value in record.items() if isinstance(value, Regions)] + keys = [key for key, value in record.items() if isinstance(value, Boxes)] if not keys or not _has_spatial_transform(op): return _WARNED_SPATIAL.add(type(op)) logger.warning( f"albumentations {type(op).__name__} changes GEOMETRY, but this record's detection " - f"boxes ({keys}) ride as a Regions item, which is not in the library's key vocabulary " + f"boxes ({keys}) ride as a Boxes item, which is not in the library's key vocabulary " f"({', '.join(_ALB_KEYS)}) — so the pixels moved and the boxes did not. Put boxes under " f"'bboxes' + 'labels' with A.Compose(..., bbox_params=A.BboxParams(...)) so the library " f"moves them in the same draw, or use recordstream.ops.target.ResizeDetection for a " @@ -230,7 +230,7 @@ def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." ) return record - _warn_if_regions_are_left_behind(record, op, kwargs) + _warn_if_boxes_are_left_behind(record, op, kwargs) out = op(**kwargs) merged = dict(record) for key, value in out.items(): @@ -276,28 +276,28 @@ def _invoke_torchvision_v2(record: Record, op: Any) -> Optional[Record]: leaves transformed, everything else passed through — called as-is. "Everything else passed through" is where detection boxes fall: v2 recognises its OWN - ``tv_tensors`` types, and a :class:`~recordstream.Regions` is not one, so a geometric + ``tv_tensors`` types, and a :class:`~recordstream.Boxes` is not one, so a geometric transform moves the pixels and leaves the boxes — the same silent desync the albumentations family has, reached by a different route (there the boxes are not in the key vocabulary; here they are not in the TYPE vocabulary). Measured: ``v2.Resize((64, 64))`` takes a 200x200 image to 64x64 with the boxes still on ``[10, 10, 100, 100]``, while the same transform over a ``tv_tensors.BoundingBoxes`` correctly rescales them to ``[3.2, 3.2, 32, 32]``. """ - _warn_if_v2_leaves_regions_behind(record, op) + _warn_if_v2_leaves_boxes_behind(record, op) return cast(Record, op(record)) -def _warn_if_v2_leaves_regions_behind(record: Record, op: Any) -> None: - """The v2 twin of :func:`_warn_if_regions_are_left_behind` — once per transform type.""" +def _warn_if_v2_leaves_boxes_behind(record: Record, op: Any) -> None: + """The v2 twin of :func:`_warn_if_boxes_are_left_behind` — once per transform type.""" if type(op) in _WARNED_SPATIAL: return - keys = [key for key, value in record.items() if isinstance(value, Regions)] + keys = [key for key, value in record.items() if isinstance(value, Boxes)] if not keys or not _is_v2_geometry(op): return _WARNED_SPATIAL.add(type(op)) logger.warning( f"torchvision v2 {type(op).__name__} changes GEOMETRY, but this record's detection boxes " - f"({keys}) ride as a Regions item, which is not one of v2's tv_tensors types — so v2 " + f"({keys}) ride as a Boxes item, which is not one of v2's tv_tensors types — so v2 " f"passes them through untouched while the pixels move. Carry boxes as " f"torchvision.tv_tensors.BoundingBoxes(..., format=…, canvas_size=…) so v2 transforms " f"them in the same call, or use recordstream.ops.target.ResizeDetection for a plain " diff --git a/recordstream/io.py b/recordstream/io.py index b678583..d2726e0 100644 --- a/recordstream/io.py +++ b/recordstream/io.py @@ -135,7 +135,7 @@ def _has_data_field(cls: type) -> bool: return is_dataclass(cls) and any(f.name == "data" for f in dataclass_fields(cls)) -# --- optional helper: an item with a python-object payload (e.g. Regions boxes) ---- +# --- optional helper: an item with a python-object payload (e.g. Boxes boxes) ---- def default_encoded_attrs(item: Any) -> Dict[str, Any]: """The default codec's attrs view of ``item`` — reusable inside a custom encoder.""" return _attrs(item) diff --git a/recordstream/items.py b/recordstream/items.py index be8aa32..8547478 100644 --- a/recordstream/items.py +++ b/recordstream/items.py @@ -2,7 +2,7 @@ A record is a plain ``dict`` (the :data:`Record` alias) whose values are TYPED: an :class:`Image` carries its ``layout``, a :class:`Label` its ``classes``, a -:class:`Regions` its ``canvas`` reference frame. Ops dispatch on these types (the +:class:`Boxes` its ``canvas`` reference frame. Ops dispatch on these types (the torchvision-v2 ``tv_tensors`` idea) — there is no wrapper container and no role tags; key names ("image", "mask", "label") carry meaning, exactly like every torch batch dict. @@ -12,7 +12,7 @@ subclass) so a type-agnostic operation touches them AS an array while their extra attributes survive numpy operations (``__array_finalize__``). ``Image`` / ``Mask`` are these. -* **Structured items are dataclass wrappers** (:class:`Regions` / :class:`Label`) — a +* **Structured items are dataclass wrappers** (:class:`Boxes` / :class:`Label`) — a bounding-box set or a class label is not an array; a wrapper is also the right home for a payload a domain package does not want to subclass (e.g. complex-IQ signal data, where subclassing an ``np.complex64`` ndarray and preserving attributes through arithmetic is @@ -55,7 +55,7 @@ "NDArrayItem", "Image", "Mask", - "Regions", + "Boxes", "Label", "MultiLabel", "is_class_id", @@ -169,20 +169,24 @@ class Mask(NDArrayItem): # --------------------------------------------------------------------------- @register_item @dataclass -class Regions: - """A set of rectangular regions / bounding boxes with optional labels and scores. +class Boxes: + """A set of pixel-space bounding boxes on an image raster, with optional labels and scores. + + Pixel-ONLY by contract: boxes are HALF-OPEN ``[x0, y0, x1, y1]`` rows in absolute + pixels (x rightward, y downward). A domain package needing a different coordinate + system (e.g. time/frequency regions on a waveform) registers its OWN item type — + this one is what every image-geometry op and detection consumer dispatches on. Attributes: - boxes: The boxes — pixel ``[x0, y0, x1, y1]`` or signal ``[f0, f1, t0, t1]`` rows, as a + boxes: The boxes — half-open absolute-pixel ``[x0, y0, x1, y1]`` rows, as a list OR an ``[N, 4]`` array/tensor (a detection pipeline keeps its framework's type; annotated ``Any`` because list, ndarray and tensor share no useful protocol). labels: Optional per-box class labels (list or ``[N]`` array/tensor, like ``boxes``). scores: Optional per-box confidence scores (list or ``[N]`` array/tensor). - canvas: Optional ``(H, W)`` reference frame — the coordinate system boxes live in, + canvas: Optional ``(H, W)`` reference frame — the raster the boxes are stated in, so a geometric transform (flip / resize) has a self-contained frame. - extras: Auxiliary PER-BOX parallel arrays and region-set measurements keyed by name - (e.g. per-box durations/bandwidths/power readings) — item-scoped metadata that - travels WITH the boxes it describes. + extras: Auxiliary PER-BOX parallel arrays and box-set measurements keyed by name — + item-scoped metadata that travels WITH the boxes it describes. """ boxes: Any = field(default_factory=list) diff --git a/recordstream/ops/__init__.py b/recordstream/ops/__init__.py index d2a1980..2066188 100644 --- a/recordstream/ops/__init__.py +++ b/recordstream/ops/__init__.py @@ -3,7 +3,7 @@ Submodules: - recordstream.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / - connected_component_bboxes / resolve_expression helpers) + connected_component_boxes / resolve_expression helpers) - recordstream.ops.torch: ToTensor (+ to_tensor helper) - recordstream.ops.image: ConvertToImage, ConvertToMask (+ value_to_image / normalize_to_uint8 …) - recordstream.ops.target: EncodeTarget, DecodeTarget, diff --git a/recordstream/ops/image.py b/recordstream/ops/image.py index 571a588..4c9c3e7 100644 --- a/recordstream/ops/image.py +++ b/recordstream/ops/image.py @@ -26,9 +26,10 @@ from PIL import Image, ImageDraw from recordstream._compat import is_torch_tensor +from recordstream.items import Boxes from recordstream.items import Image as ImageItem from recordstream.items import Mask as MaskItem -from recordstream.items import NDArrayItem, Record, Regions, is_item, item_data, item_value +from recordstream.items import NDArrayItem, Record, is_item, item_data, item_value from recordstream.transform import Transform logger = get_logger("recordstream.ops.image") @@ -114,7 +115,7 @@ def _text_to_image(text: str, width: int = 512, height: int = 160) -> np.ndarray def image_frame(value: Any) -> Optional[Tuple[int, int]]: """The ``(H, W)`` raster of an image-bearing value, or ``None`` when it is not one. - The reference frame a :class:`~recordstream.Regions`' boxes are stated in is a raster, so + The reference frame a :class:`~recordstream.Boxes`' boxes are stated in is a raster, so "what raster is this?" is asked wherever boxes and pixels have to agree — the coupled image+boxes resize reads it to derive its scale factors, the ops that CREATE a target read it to record the frame on the item, and any consumer comparing the two reads it to notice a @@ -681,8 +682,8 @@ def __init__( self.field = field self.output = output # Private, so it stays out of the config surface (it is not a knob) — see - # `_warn_if_it_desyncs_regions`, which reports the configuration once, not per record. - self._warned_about_regions = False + # `_warn_if_it_desyncs_boxes`, which reports the configuration once, not per record. + self._warned_about_boxes = False def _find_source(self, record: Record) -> Any: """Resolve the payload to render (``self.field`` or the first array-bearing item).""" @@ -696,26 +697,26 @@ def _find_source(self, record: Record) -> Any: return item_data(item) raise ValueError(f"ConvertToImage: no array-bearing field in record (keys: {list(record)})") - def _warn_if_it_desyncs_regions(self, record: Record, before: Tuple[int, int], after: Tuple[int, int]) -> None: + def _warn_if_it_desyncs_boxes(self, record: Record, before: Tuple[int, int], after: Tuple[int, int]) -> None: """Warn ONCE when this op resized the pixels of a record whose boxes describe them. This op resizes the IMAGE and nothing else, which is correct for what it is — but a - record carrying a :class:`~recordstream.Regions` states its boxes in a raster, and moving + record carrying a :class:`~recordstream.Boxes` states its boxes in a raster, and moving the pixels out from under them leaves the two disagreeing with no error of its own: every shape stays valid and only the coordinates become wrong. Downstream that surfaces as a model quietly training against misplaced targets, which is the expensive way to find out. It is a WARNING and not an error because this op cannot know what the boxes describe — a - record may legitimately carry regions belonging to a different key than the field being + record may legitimately carry boxes belonging to a different key than the field being rendered — so the condition is likely, not certain. It fires once per op instance: the message is about the CONFIGURATION, so a second copy per record only buries it. """ - if self._warned_about_regions or before == after: + if self._warned_about_boxes or before == after: return - keys = [key for key, value in record.items() if isinstance(value, Regions)] + keys = [key for key, value in record.items() if isinstance(value, Boxes)] if not keys: return - self._warned_about_regions = True + self._warned_about_boxes = True logger.warning( f"ConvertToImage resized {before} -> {after} (H, W) on a record whose {keys} " f"carries detection boxes — this op moves PIXELS ONLY, so those boxes now describe " @@ -733,7 +734,7 @@ def __call__(self, record: Record) -> Record: ) else: out_arr = _bound_longest_side(rgb, self.max_size) - self._warn_if_it_desyncs_regions(record, rgb.shape[:2], out_arr.shape[:2]) + self._warn_if_it_desyncs_boxes(record, rgb.shape[:2], out_arr.shape[:2]) return {**record, self.output: ImageItem(out_arr, layout="HWC")} diff --git a/recordstream/ops/numpy.py b/recordstream/ops/numpy.py index d146353..0e54805 100644 --- a/recordstream/ops/numpy.py +++ b/recordstream/ops/numpy.py @@ -7,7 +7,7 @@ from confluid import configurable from loggair import get_logger -from recordstream.items import Mask, NDArrayItem, Record, Regions, item_data +from recordstream.items import Boxes, Mask, NDArrayItem, Record, item_data from recordstream.transform import Transform logger = get_logger(__name__) @@ -187,10 +187,16 @@ def __call__(self, record: Record) -> Record: return {**record, self.output: Mask(mask)} -def connected_component_bboxes( +def connected_component_boxes( mask: np.ndarray, min_area_bins: int = 1, connectivity: int = 4 ) -> List[Tuple[int, int, int, int]]: - """Label connected ``True`` regions of a 2-D bool mask → ``(row_min, row_max, col_min, col_max)`` inclusive tuples. + """Label connected ``True`` regions of a 2-D bool mask → HALF-OPEN xyxy ``(x0, y0, x1, y1)`` tuples. + + The pixel-box convention every :class:`~recordstream.Boxes` producer emits: x = column, + y = row, far edges EXCLUSIVE — ``mask[y0:y1, x0:x1]`` covers the component exactly. + (Renamed from ``connected_component_bboxes``, which returned INCLUSIVE + ``(row_min, row_max, col_min, col_max)`` tuples — the rename makes a stale caller fail + loudly instead of silently mis-reading axes.) Components smaller than ``min_area_bins`` are dropped. ``connectivity`` is ``4`` (orthogonal neighbors) or ``8`` (orthogonal + diagonal). Shared by :class:`ConnectedComponents` @@ -222,10 +228,10 @@ def connected_component_bboxes( continue bboxes.append( ( - int(row_slice.start), - int(row_slice.stop) - 1, int(col_slice.start), - int(col_slice.stop) - 1, + int(row_slice.start), + int(col_slice.stop), + int(row_slice.stop), ) ) return bboxes @@ -233,13 +239,13 @@ def connected_component_bboxes( @configurable(category="op", group="numpy") class ConnectedComponents(Transform): - """A boolean ``Mask`` → a ``Regions`` item. + """A boolean ``Mask`` → a ``Boxes`` item. Reads the :class:`~recordstream.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the first array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into - ``(row_min, row_max, col_min, col_max)`` inclusive bin-box tuples via - :func:`connected_component_bboxes`, writing them as a :class:`~recordstream.Regions` item under - ``output`` (RAW detections, not model predictions). Any other key passes through. + HALF-OPEN pixel xyxy ``(x0, y0, x1, y1)`` tuples via :func:`connected_component_boxes`, + writing them as a :class:`~recordstream.Boxes` item under ``output`` with ``canvas`` set to + the mask's shape (RAW detections, not model predictions). Any other key passes through. Components smaller than ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or 8-neighborhood. Requires ``scipy`` (``pip install recordstream[vision]``). @@ -248,12 +254,12 @@ class ConnectedComponents(Transform): min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). connectivity: Pixel neighborhood — ``4`` (orthogonal only) or ``8`` (orthogonal + diagonal). field: Name of the ``Mask`` field to label; blank (default) picks the first ``Mask`` (else first array). - output: Name of the key the ``Regions`` item is written to (added if new). + output: Name of the key the ``Boxes`` item is written to (added if new). """ handles = (Mask,) consumes = (Mask,) - produces = (Regions,) + produces = (Boxes,) def __init__( self, @@ -298,14 +304,16 @@ def _find_mask(self, record: Record) -> np.ndarray: def __call__(self, record: Record) -> Record: mask = self._find_mask(record) - bboxes = connected_component_bboxes(mask, self.min_area_bins, self.connectivity) - return {**record, self.output: Regions(boxes=list(bboxes))} + boxes = connected_component_boxes(mask, self.min_area_bins, self.connectivity) + # canvas is filled in even for an EMPTY box set — a frame check that silently skips + # exactly the records with nothing to check reports a clean bill for the wrong reason. + return {**record, self.output: Boxes(boxes=list(boxes), canvas=(mask.shape[0], mask.shape[1]))} __all__ = [ "resolve_expression", "threshold_array", - "connected_component_bboxes", + "connected_component_boxes", "LowComparison", "HighComparison", "Threshold", diff --git a/recordstream/ops/target.py b/recordstream/ops/target.py index 3517b03..64dac2d 100644 --- a/recordstream/ops/target.py +++ b/recordstream/ops/target.py @@ -6,11 +6,11 @@ train / eval / predict share one identical ordering. * :class:`CocoToTorchVisionDetection` turns a HuggingFace / COCO ``objects`` annotation (``{bbox, category}``) into a torchvision detection target rendered as a - :class:`~recordstream.Regions` item. + :class:`~recordstream.Boxes` item. * :class:`MasksToDetectionBoxes` derives detection boxes from a segmentation ``Mask``. The detection conversions are the modality-neutral, image-detection counterparts of -waivefront's signal-domain region ops. The encoded target value is written verbatim; wrap +a signal package's domain-specific region ops. The encoded target value is written verbatim; wrap it into a framework tensor downstream (e.g. a collate function) when a loss needs one. """ @@ -19,7 +19,7 @@ import numpy as np from confluid import configurable -from recordstream.items import Label, Mask, MultiLabel, Record, Regions, item_data +from recordstream.items import Boxes, Label, Mask, MultiLabel, Record, item_data from recordstream.transform import Transform #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo @@ -37,7 +37,7 @@ def _source_frame(record: Record) -> Optional[Tuple[int, int]]: yield), then as the first :class:`~recordstream.Image` item. It is deliberately narrow: only a declared image is trusted. A generic "first array with two - dimensions" search would happily read a ``Regions``' own ``[N, 4]`` box array as an ``N x 4`` + dimensions" search would happily read a ``Boxes``' own ``[N, 4]`` box array as an ``N x 4`` raster and record a confident lie. ``None`` is an ordinary answer — it leaves ``canvas`` exactly as it was before this was recorded at all, so nothing depends on the lookup succeeding. @@ -143,10 +143,10 @@ def masks_to_detection( boxes: list = [] if connected: - from recordstream.ops.numpy import connected_component_bboxes + from recordstream.ops.numpy import connected_component_boxes - for r0, r1, c0, c1 in connected_component_bboxes(mask != 0, min_area, connectivity): - boxes.append((float(c0), float(r0), float(c1 + 1), float(r1 + 1))) + for x0, y0, x1, y1 in connected_component_boxes(mask != 0, min_area, connectivity): + boxes.append((float(x0), float(y0), float(x1), float(y1))) else: for value in np.unique(mask): if int(value) == 0: @@ -308,11 +308,11 @@ def __call__(self, record: Record) -> Record: @configurable(category="op", group="structure") class CocoToTorchVisionDetection(Transform): - """A COCO / HF ``objects`` annotation → a target ``Regions``. + """A COCO / HF ``objects`` annotation → a target ``Boxes``. Reads a source field (``field``; blank picks the first :class:`~recordstream.Label`, else the first field) carrying a HuggingFace / COCO ``objects`` mapping and rewrites it to the - torchvision detection target, riding as a :class:`~recordstream.Regions` item under + torchvision detection target, riding as a :class:`~recordstream.Boxes` item under ``output`` (``boxes`` = the ``[N, 4]`` float32 xyxy tensor, ``labels`` = the ``[N]`` int64 class-id tensor). An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the negative-example contract). @@ -323,12 +323,12 @@ class CocoToTorchVisionDetection(Transform): bbox_format: Box layout in pixels — ``xywh`` (COCO, default), ``xyxy``, or ``cxcywh``; output is xyxy. label_offset: Added to each class id (default ``0``). Set ``1`` to reserve class ``0`` for background. field: Source field with the objects mapping; blank (default) picks the first ``Label``, else the first field. - output: Key the target ``Regions`` is written to (added if new). + output: Key the target ``Boxes`` is written to (added if new). """ handles = (Label,) consumes = (Label,) - produces = (Regions,) + produces = (Boxes,) def __init__( self, @@ -368,20 +368,20 @@ def __call__(self, record: Record) -> Record: target = coco_to_detection(objects, self.bbox_key, self.category_key, self.bbox_format, self.label_offset) # `canvas` IS the frame the boxes are stated in — a COCO box is in the source image's # pixel space, so it is knowable here and recording it costs one lookup. Left None when - # no image is in the record, which is what every Regions carried before this. + # no image is in the record, which is what every Boxes carried before this. return { **record, - self.output: Regions(boxes=target["boxes"], labels=target["labels"], canvas=_source_frame(record)), + self.output: Boxes(boxes=target["boxes"], labels=target["labels"], canvas=_source_frame(record)), } @configurable(category="op", group="structure") class MasksToDetectionBoxes(Transform): - """A segmentation ``Mask`` → a target ``Regions``. + """A segmentation ``Mask`` → a target ``Boxes``. Reads the :class:`~recordstream.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the first array-bearing item) as a 2-D integer mask and derives one tight - ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~recordstream.Regions` item + ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~recordstream.Boxes` item under ``output``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. Args: @@ -390,12 +390,12 @@ class MasksToDetectionBoxes(Transform): min_area: Drop objects whose mask area (in pixels) is below this (default ``1``). connectivity: Connected-components neighborhood when ``connected=True`` — ``4`` or ``8`` (default ``4``). field: Name of the ``Mask`` field to read; blank (default) picks the first ``Mask`` (else the first array). - output: Key the target ``Regions`` is written to (added if new). + output: Key the target ``Boxes`` is written to (added if new). """ handles = (Mask,) consumes = (Mask,) - produces = (Regions,) + produces = (Boxes,) def __init__( self, @@ -447,7 +447,7 @@ def __call__(self, record: Record) -> Record: height, width = int(mask.shape[0]), int(mask.shape[1]) return { **record, - self.output: Regions(boxes=target["boxes"], labels=target["labels"], canvas=(height, width)), + self.output: Boxes(boxes=target["boxes"], labels=target["labels"], canvas=(height, width)), } @@ -459,8 +459,8 @@ class ResizeDetection(Transform): resized, and a resize that moved the pixels without moving the boxes would silently train on misplaced targets. Reads the image under ``input_key`` (PIL or a uint8 HWC/2-D array), resizes it to ``(height, width)`` (bilinear, PIL), and scales the - :class:`~recordstream.Regions` boxes under ``target_key`` by the same factors — torch boxes - stay torch, numpy stays numpy. The resized ``Regions`` records the new frame in ``canvas``. + :class:`~recordstream.Boxes` under ``target_key`` by the same factors — torch boxes + stay torch, numpy stays numpy. The resized ``Boxes`` records the new frame in ``canvas``. Ops that need no fixed size (torchvision detectors resize internally) simply omit this op — it exists for the detectors that require pre-sized square inputs. @@ -469,11 +469,11 @@ class ResizeDetection(Transform): width: Target width in pixels; required at use (validated lazily, ``0`` = unset). height: Target height in pixels; required at use (validated lazily, ``0`` = unset). input_key: Record key carrying the image (default ``"image"``). - target_key: Record key carrying the target ``Regions``; a record without it resizes the image alone. + target_key: Record key carrying the target ``Boxes``; a record without it resizes the image alone. """ - consumes = (Regions,) - produces = (Regions,) + consumes = (Boxes,) + produces = (Boxes,) def __init__( self, @@ -534,7 +534,7 @@ def __call__(self, record: Record) -> Record: merged[self.input_key] = with_data(item, resized) if isinstance(item, NDArrayItem) else resized target = record.get(self.target_key) - if isinstance(target, Regions): + if isinstance(target, Boxes): import dataclasses # An EMPTY target is re-framed too. Scaling no boxes is a no-op, but leaving the diff --git a/recordstream/storage/base.py b/recordstream/storage/base.py index 1d8b90b..043db98 100644 --- a/recordstream/storage/base.py +++ b/recordstream/storage/base.py @@ -74,7 +74,7 @@ def flush(self) -> None: # The shared attr wire-format for the record key-group layout (HDF5 attrs / Zarr .zattrs # / directory JSON all speak it): scalars stay native (queryable), array values become # separate datasets, and structured values (list/tuple/dict/None) ride a JSON string with -# TUPLE TAGGING so a round-trip preserves tuple-ness (Regions.canvas == (H, W), not [H, W]). +# TUPLE TAGGING so a round-trip preserves tuple-ness (Boxes.canvas == (H, W), not [H, W]). # -------------------------------------------------------------------------------------- def split_attrs(attrs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: """Split an encoded value's attrs into ``(plain, arrays)`` for storage. diff --git a/tests/_fixtures.py b/tests/_fixtures.py index c13e642..eac19a0 100644 --- a/tests/_fixtures.py +++ b/tests/_fixtures.py @@ -11,16 +11,16 @@ import numpy as np -from recordstream import Image, Mask, Record, Regions, Transform, item_data, with_data +from recordstream import Boxes, Image, Mask, Record, Transform, item_data, with_data class FixtureFlip(Transform): - """Horizontal flip with ONE shared decision across Image + Mask + Regions (test fixture).""" + """Horizontal flip with ONE shared decision across Image + Mask + Boxes (test fixture).""" - handles = (Image, Mask, Regions) + handles = (Image, Mask, Boxes) consumes = (Image,) - optional = (Mask, Regions) - produces = (Image, Mask, Regions) + optional = (Mask, Boxes) + produces = (Image, Mask, Boxes) def __init__(self, p: float = 0.5, field: Optional[str] = None) -> None: super().__init__(field=field) @@ -46,19 +46,19 @@ def _flip_mask(item: Mask, params: Dict[str, Any]) -> Mask: return with_data(item, np.flip(item_data(item), axis=1).copy()) -@FixtureFlip.kernel(Regions) -def _flip_regions(item: Regions, params: Dict[str, Any]) -> Regions: +@FixtureFlip.kernel(Boxes) +def _flip_regions(item: Boxes, params: Dict[str, Any]) -> Boxes: if not params["do"]: return item width = params.get("width") or (item.canvas[1] if item.canvas else None) if width is None: - raise ValueError("FixtureFlip: no reference width to flip Regions") + raise ValueError("FixtureFlip: no reference width to flip Boxes") boxes = [[width - box[2], box[1], width - box[0], box[3]] for box in item.boxes] - return Regions(boxes=boxes, labels=item.labels, scores=item.scores, canvas=item.canvas) + return Boxes(boxes=boxes, labels=item.labels, scores=item.scores, canvas=item.canvas) def _reference_width(record: Record) -> Optional[int]: - """The horizontal extent to flip boxes against — from the first Image/Mask, or a Regions canvas.""" + """The horizontal extent to flip boxes against — from the first Image/Mask, or a Boxes canvas.""" for _, item in record.items(): if isinstance(item, Image): arr = item_data(item) @@ -70,6 +70,6 @@ def _reference_width(record: Record) -> Optional[int]: if arr.ndim >= 2: return int(arr.shape[1]) for _, item in record.items(): - if isinstance(item, Regions) and item.canvas: + if isinstance(item, Boxes) and item.canvas: return int(item.canvas[1]) return None diff --git a/tests/test_batch.py b/tests/test_batch.py index eb9cb79..92186ca 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -207,19 +207,19 @@ def test_a_label_column_contributes_its_values() -> None: # --------------------------------------------------------------------------- # -# batch_regions — the collate's transpose for a region-set column +# batch_boxes — the collate's transpose for a region-set column # --------------------------------------------------------------------------- # -class TestBatchRegions: - """A collated `Regions` back into the per-record dicts every detection interface takes.""" +class TestBatchBoxes: + """A collated `Boxes` back into the per-record dicts every detection interface takes.""" def _batch(self, counts: tuple = (1, 3), scores: bool = False) -> Any: import torch - from recordstream import Regions, collate_records + from recordstream import Boxes, collate_records records = [ { - "target": Regions( + "target": Boxes( boxes=torch.rand(n, 4), labels=torch.zeros(n, dtype=torch.int64), scores=torch.ones(n) if scores else None, @@ -230,52 +230,52 @@ def _batch(self, counts: tuple = (1, 3), scores: bool = False) -> Any: return collate_records(records) def test_it_transposes_a_variable_n_column_into_per_record_dicts(self) -> None: - from recordstream import batch_regions + from recordstream import batch_boxes - targets = batch_regions(self._batch(counts=(1, 3)), "target") + targets = batch_boxes(self._batch(counts=(1, 3)), "target") assert len(targets) == 2 assert [tuple(t["boxes"].shape) for t in targets] == [(1, 4), (3, 4)] assert [tuple(t["labels"].shape) for t in targets] == [(1,), (3,)] def test_an_absent_field_is_OMITTED_not_handed_over_as_none(self) -> None: """A training target is exactly {boxes, labels} — a `None` scores key would reach a model.""" - from recordstream import batch_regions + from recordstream import batch_boxes - assert set(batch_regions(self._batch(), "target")[0]) == {"boxes", "labels"} - assert set(batch_regions(self._batch(scores=True), "target")[0]) == {"boxes", "labels", "scores"} + assert set(batch_boxes(self._batch(), "target")[0]) == {"boxes", "labels"} + assert set(batch_boxes(self._batch(scores=True), "target")[0]) == {"boxes", "labels", "scores"} def test_values_keep_their_framework(self) -> None: """Framework-free by rule: the caller owns dtype and device, as with `batch_values`.""" import torch - from recordstream import batch_regions + from recordstream import batch_boxes - assert isinstance(batch_regions(self._batch(), "target")[0]["boxes"], torch.Tensor) + assert isinstance(batch_boxes(self._batch(), "target")[0]["boxes"], torch.Tensor) def test_numpy_boxes_stay_numpy(self) -> None: import numpy as np - from recordstream import Regions, batch_regions, collate_records + from recordstream import Boxes, batch_boxes, collate_records - batch = collate_records([{"target": Regions(boxes=np.zeros((2, 4)), labels=np.zeros(2))}]) - assert isinstance(batch_regions(batch, "target")[0]["boxes"], np.ndarray) + batch = collate_records([{"target": Boxes(boxes=np.zeros((2, 4)), labels=np.zeros(2))}]) + assert isinstance(batch_boxes(batch, "target")[0]["boxes"], np.ndarray) def test_a_wrong_type_raises_naming_it(self) -> None: import pytest - from recordstream import Label, batch_regions, collate_records + from recordstream import Label, batch_boxes, collate_records - with pytest.raises(TypeError, match="not a Regions"): - batch_regions(collate_records([{"target": Label(0)}]), "target") + with pytest.raises(TypeError, match="not a Boxes"): + batch_boxes(collate_records([{"target": Label(0)}]), "target") def test_an_uncollated_regions_raises_naming_the_mistake(self) -> None: import numpy as np import pytest - from recordstream import Regions, batch_regions + from recordstream import Boxes, batch_boxes - with pytest.raises(ValueError, match="not a COLLATED Regions"): - batch_regions({"target": Regions(boxes=np.zeros((2, 4)))}, "target") + with pytest.raises(ValueError, match="not a COLLATED Boxes"): + batch_boxes({"target": Boxes(boxes=np.zeros((2, 4)))}, "target") # --------------------------------------------------------------------------- # @@ -291,12 +291,12 @@ class TestCollateIsAChoice: def _records(self, sizes: tuple = (8, 8)) -> Any: import torch - from recordstream import Image, Label, Regions + from recordstream import Boxes, Image, Label return [ { "image": Image(np.zeros((3, s, s), dtype="float32"), layout="CHW"), - "target": Regions(boxes=torch.rand(n, 4), labels=torch.zeros(n, dtype=torch.int64)), + "target": Boxes(boxes=torch.rand(n, 4), labels=torch.zeros(n, dtype=torch.int64)), "class": Label(i), } for i, (s, n) in enumerate(zip(sizes, (1, 3))) @@ -344,12 +344,12 @@ def test_the_stack_error_names_the_shapes_and_the_way_out(self) -> None: @pytest.mark.parametrize("collate_key", ["record", "list"]) def test_every_read_back_helper_accepts_both_collates(self, collate_key: str) -> None: - """`batch_values` / `batch_regions` / `batch_metadata` give the SAME answer either way.""" - from recordstream import batch_metadata, batch_regions, batch_values, collate + """`batch_values` / `batch_boxes` / `batch_metadata` give the SAME answer either way.""" + from recordstream import batch_boxes, batch_metadata, batch_values, collate batch = collate(self._records(), key=collate_key) assert batch_values(batch, "class") == [0, 1] - targets = batch_regions(batch, "target") + targets = batch_boxes(batch, "target") assert [t["boxes"].shape[0] for t in targets] == [1, 3] assert batch_metadata(batch, exclude=("image", "target")) == [{"class": 0}, {"class": 1}] diff --git a/tests/test_convert_to_mask.py b/tests/test_convert_to_mask.py index bccc33f..a8a31d9 100644 --- a/tests/test_convert_to_mask.py +++ b/tests/test_convert_to_mask.py @@ -17,7 +17,7 @@ import pytest from PIL import Image as PILImage -from recordstream import Image, Label, Mask, MultiLabel, Regions, collate_records, item_data, item_value +from recordstream import Boxes, Image, Label, Mask, MultiLabel, collate_records, item_data, item_value from recordstream.core import _apply_op from recordstream.ops import ConvertToMask, DropField, FormulaOp @@ -261,7 +261,7 @@ def _captured_warnings() -> Iterator[List[str]]: class TestBoxesKnowTheirFrame: - """`canvas` is the raster a `Regions`' boxes are stated in — so every op that makes or + """`canvas` is the raster a `Boxes`' boxes are stated in — so every op that makes or re-frames one records it, and the op that moves pixels ALONE says so. Before this, only the coupled resize set `canvas`, which meant the frame was knowable @@ -303,7 +303,7 @@ def test_a_regions_box_array_is_never_mistaken_for_a_raster(self) -> None: record = { "objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [0]}), - "other": Regions(boxes=np.zeros((7, 4), dtype="float32"), labels=np.zeros((7,), dtype="int64")), + "other": Boxes(boxes=np.zeros((7, 4), dtype="float32"), labels=np.zeros((7,), dtype="int64")), } assert CocoToTorchVisionDetection(field="objects")(record)["target"].canvas is None @@ -313,7 +313,7 @@ def test_an_EMPTY_target_is_re_framed_too(self) -> None: record = { "image": Image(np.zeros((100, 100, 3), dtype="uint8")), - "target": Regions(boxes=np.zeros((0, 4), dtype="float32"), labels=np.zeros((0,), dtype="int64")), + "target": Boxes(boxes=np.zeros((0, 4), dtype="float32"), labels=np.zeros((0,), dtype="int64")), } out = ResizeDetection(width=64, height=32)(record) assert out["target"].canvas == (32, 64) @@ -323,7 +323,7 @@ def test_an_image_only_resize_WARNS_when_it_desyncs_boxes(self) -> None: record = { "image": Image(np.zeros((200, 200, 3), dtype="uint8")), - "target": Regions(boxes=np.array([[10.0, 10.0, 50.0, 50.0]]), labels=np.array([1])), + "target": Boxes(boxes=np.array([[10.0, 10.0, 50.0, 50.0]]), labels=np.array([1])), } op = ConvertToImage(field="image", width=64, height=64) with _captured_warnings() as warnings: @@ -340,7 +340,7 @@ def test_no_warning_without_boxes_or_without_a_resize(self) -> None: from recordstream.ops.image import ConvertToImage image = Image(np.zeros((200, 200, 3), dtype="uint8")) - boxes = Regions(boxes=np.array([[1.0, 2.0, 3.0, 4.0]]), labels=np.array([1])) + boxes = Boxes(boxes=np.array([[1.0, 2.0, 3.0, 4.0]]), labels=np.array([1])) with _captured_warnings() as warnings: ConvertToImage(field="image", width=64, height=64)({"image": image}) # resized, no boxes ConvertToImage(field="image", max_size=999)({"image": image, "target": boxes}) # boxes, no resize diff --git a/tests/test_dispatch.py b/tests/test_dispatch.py index 070d9c0..9088dc9 100644 --- a/tests/test_dispatch.py +++ b/tests/test_dispatch.py @@ -2,7 +2,7 @@ from typing import Any, Dict -from recordstream import Image, Label, Mask, Regions, Transform +from recordstream import Boxes, Image, Label, Mask, Transform from recordstream.dispatch import dispatch, get_kernel, register_kernel, registered_kernels from tests._fixtures import FixtureFlip @@ -51,6 +51,6 @@ def test_plain_value_type_misses(self) -> None: def test_registered_kernels_lists_pairs(self) -> None: pairs = registered_kernels() assert ("FixtureFlip", "Image") in pairs - assert ("FixtureFlip", "Regions") in pairs + assert ("FixtureFlip", "Boxes") in pairs assert dispatch(FixtureFlip, Label) is None # FixtureFlip does not handle Label - assert dispatch(FixtureFlip, Regions) is get_kernel(FixtureFlip, Regions) + assert dispatch(FixtureFlip, Boxes) is get_kernel(FixtureFlip, Boxes) diff --git a/tests/test_io.py b/tests/test_io.py index 31b8fe6..615f99a 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -7,10 +7,10 @@ import pytest from recordstream import ( + Boxes, EncodedItem, Image, Label, - Regions, decode_item, decode_record, encode_item, @@ -100,7 +100,7 @@ class TestRecordCodec: def test_record_round_trip_keys_order_and_plain_entries(self) -> None: record = { "image": Image(np.zeros((2, 2, 3), dtype=np.float32)), - "regions": Regions(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), + "regions": Boxes(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), "class": Label("x"), "gain_db": -3.0, # a plain scalar rides the same layout under the "plain" tag } diff --git a/tests/test_items.py b/tests/test_items.py index 59406a0..af12b2b 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -1,6 +1,6 @@ """Typed items — array-subclass attribute preservation, wrappers, payload accessors, registry. -Only the MODALITY-NEUTRAL core items live in recordstream (Image / Mask / Regions / Label). The +Only the MODALITY-NEUTRAL core items live in recordstream (Image / Mask / Boxes / Label). The data-bearing-wrapper and multi-attribute-array paths (which the signal-domain items in a domain package exercise for real) are covered here with small test-local item types, so the core stays tested without importing a domain package. @@ -12,11 +12,11 @@ import pytest from recordstream.items import ( + Boxes, Image, Label, Mask, NDArrayItem, - Regions, get_item_type, is_item, item_data, @@ -78,7 +78,7 @@ def test_wrapper_fields(self) -> None: def test_zero_arg_construction(self) -> None: # Wrappers build with no args (fields defaulted) — the workspace lazy/zero-arg convention. - assert _Blob().data is None and Regions().boxes == [] and Label().value is None + assert _Blob().data is None and Boxes().boxes == [] and Label().value is None class TestPayloadAccessors: @@ -91,7 +91,7 @@ def test_item_data_wrapper(self) -> None: assert np.array_equal(item_data(_Blob(np.ones(3))), np.ones(3)) def test_item_data_no_payload_returns_self(self) -> None: - reg = Regions(boxes=[[0, 0, 1, 1]]) + reg = Boxes(boxes=[[0, 0, 1, 1]]) assert item_data(reg) is reg # no `.data` slot — returns the item def test_item_data_plain_value_passes_through(self) -> None: @@ -108,13 +108,13 @@ def test_with_data_wrapper_preserves_meta(self) -> None: def test_with_data_without_payload_raises(self) -> None: with pytest.raises(TypeError, match="no payload slot"): - with_data(Regions(boxes=[]), [[0, 0, 1, 1]]) + with_data(Boxes(boxes=[]), [[0, 0, 1, 1]]) class TestRegistry: def test_builtins_registered(self) -> None: names = item_type_names() - for name in ("Image", "Mask", "Regions", "Label"): + for name in ("Image", "Mask", "Boxes", "Label"): assert name in names assert Image in item_types() diff --git a/tests/test_op_families.py b/tests/test_op_families.py index e735735..b0a2a37 100644 --- a/tests/test_op_families.py +++ b/tests/test_op_families.py @@ -18,7 +18,7 @@ from confluid import configurable from torchvision.transforms import v2 -from recordstream import FilterOp, Image, Label, Mask, Pipeline, Record, Regions, Transform, WrappedOp +from recordstream import Boxes, FilterOp, Image, Label, Mask, Pipeline, Record, Transform, WrappedOp from recordstream.core import Stream, _apply_op, _is_albumentations, _is_torchvision_v2 @@ -398,8 +398,8 @@ def _captured_warnings() -> Iterator[List[str]]: logger.remove(sink_id) -class TestGeometryLeavingRegionsBehind: - """A `Regions` is not in albumentations' key vocabulary, so it never reaches the library. +class TestGeometryLeavingBoxesBehind: + """A `Boxes` is not in albumentations' key vocabulary, so it never reaches the library. That is correct for the dispatch — passing a foreign item would break the call — but it means a geometry-changing transform moves the pixels while the boxes stay put, with no @@ -413,7 +413,7 @@ class TestGeometryLeavingRegionsBehind: def _record() -> Record: return { "image": Image(np.zeros((200, 200, 3), dtype="uint8")), - "target": Regions(boxes=np.array([[10.0, 10.0, 100.0, 100.0]]), labels=np.array([1])), + "target": Boxes(boxes=np.array([[10.0, 10.0, 100.0, 100.0]]), labels=np.array([1])), } @pytest.fixture(autouse=True) @@ -507,10 +507,10 @@ def test_the_correct_spelling_is_NOT_warned_about_and_moves_the_boxes(self) -> N assert [round(v, 1) for v in out["bboxes"][0]] == [3.2, 3.2, 32.0, 32.0], "boxes scaled with the image" -class TestV2GeometryLeavingRegionsBehind: +class TestV2GeometryLeavingBoxesBehind: """The same gap in the OTHER family, reached by a different route. - albumentations misses a `Regions` because it is not in the KEY vocabulary; torchvision v2 + albumentations misses a `Boxes` because it is not in the KEY vocabulary; torchvision v2 misses it because it is not one of v2's tv_tensor TYPES. Measured: `v2.Resize((64, 64))` takes a 200x200 image to 64x64 with the boxes still on `[10, 10, 100, 100]`, while the same transform over a `tv_tensors.BoundingBoxes` rescales them to `[3.2, 3.2, 32, 32]`. @@ -520,7 +520,7 @@ class TestV2GeometryLeavingRegionsBehind: def _record() -> Record: return { "image": Image(np.zeros((200, 200, 3), dtype="uint8")), - "target": Regions(boxes=torch.tensor([[10.0, 10.0, 100.0, 100.0]]), labels=torch.tensor([1])), + "target": Boxes(boxes=torch.tensor([[10.0, 10.0, 100.0, 100.0]]), labels=torch.tensor([1])), } @pytest.fixture(autouse=True) diff --git a/tests/test_structure_ops.py b/tests/test_structure_ops.py index a97fd21..74e7c88 100644 --- a/tests/test_structure_ops.py +++ b/tests/test_structure_ops.py @@ -3,18 +3,18 @@ import numpy as np import pytest -from recordstream import Image, Label, Record, Regions +from recordstream import Boxes, Image, Label, Record from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields def _record() -> Record: - return {"image": Image(np.zeros((2, 2, 3))), "regions": Regions(boxes=[[0, 0, 1, 1]]), "class": Label("x")} + return {"image": Image(np.zeros((2, 2, 3))), "regions": Boxes(boxes=[[0, 0, 1, 1]]), "class": Label("x")} class TestRenameField: def test_renames_preserving_order(self) -> None: out = RenameField(src="regions", dst="boxes")(_record()) - assert "regions" not in out and isinstance(out["boxes"], Regions) + assert "regions" not in out and isinstance(out["boxes"], Boxes) assert list(out.keys()) == ["image", "boxes", "class"] # renamed in place def test_rename_onto_existing_replaces(self) -> None: diff --git a/tests/test_transform.py b/tests/test_transform.py index d9a712d..ba41da5 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -8,7 +8,7 @@ import numpy as np import pytest -from recordstream import Image, Label, Mask, Pipeline, Record, Regions, Transform, as_transform +from recordstream import Boxes, Image, Label, Mask, Pipeline, Record, Transform, as_transform from tests._fixtures import FixtureFlip @@ -16,7 +16,7 @@ def _seg() -> Record: return { "image": Image(np.arange(8 * 10 * 3).reshape(8, 10, 3).astype(np.float32)), "mask": Mask(np.arange(8 * 10).reshape(8, 10)), - "regions": Regions(boxes=[[1, 1, 4, 4]], labels=["a"], canvas=(8, 10)), + "regions": Boxes(boxes=[[1, 1, 4, 4]], labels=["a"], canvas=(8, 10)), "class": Label("a"), "gain_db": -3.0, # a plain scalar side value is just another key } @@ -61,12 +61,12 @@ def test_image_layout_chw(self) -> None: assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, :, ::-1]) def test_regions_uses_canvas_without_image(self) -> None: - s = {"regions": Regions(boxes=[[2, 0, 5, 3]], canvas=(8, 10))} + s = {"regions": Boxes(boxes=[[2, 0, 5, 3]], canvas=(8, 10))} out = FixtureFlip(p=1.0)(s) assert out is not None and out["regions"].boxes == [[5, 0, 8, 3]] def test_regions_without_reference_width_raises(self) -> None: - s = {"regions": Regions(boxes=[[2, 0, 5, 3]])} # no image, no canvas + s = {"regions": Boxes(boxes=[[2, 0, 5, 3]])} # no image, no canvas with pytest.raises(ValueError, match="no reference width"): FixtureFlip(p=1.0)(s) diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py index 45bdc39..d0a82b5 100644 --- a/tests/test_typed_collate.py +++ b/tests/test_typed_collate.py @@ -7,7 +7,7 @@ import pytest import torch -from recordstream import Image, Label, Mask, Record, Regions, collate, collate_records, get_collate, register_item +from recordstream import Boxes, Image, Label, Mask, Record, collate, collate_records, get_collate, register_item @register_item @@ -90,12 +90,12 @@ def _ragged_detection_records() -> List[Record]: return [ { "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), - "target": Regions(boxes=[[0, 0, 2, 2]], labels=[1]), + "target": Boxes(boxes=[[0, 0, 2, 2]], labels=[1]), "pack": "a", }, { "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), - "target": Regions(boxes=[[0, 0, 1, 1], [1, 1, 3, 3], [0, 2, 2, 4]], labels=[0, 1, 0]), + "target": Boxes(boxes=[[0, 0, 1, 1], [1, 1, 3, 3], [0, 2, 2, 4]], labels=[0, 1, 0]), "pack": "b", }, ] @@ -105,7 +105,7 @@ def test_generic_collate_leaves_ragged_regions_as_lists() -> None: # Rule 2: the generic fold can only give per-record lists for a wrapper item — # exactly why detection registers its own collate. batch = collate_records(_ragged_detection_records()) - assert isinstance(batch["target"], Regions) + assert isinstance(batch["target"], Boxes) assert [len(b) for b in batch["target"].boxes] == [1, 3] diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py index a92efd3..505d0f0 100644 --- a/tests/test_typed_detection_target_ops.py +++ b/tests/test_typed_detection_target_ops.py @@ -1,11 +1,11 @@ """The two detection target-shaping ops over dict records. Pins the native transforms that build a detection pipeline's torchvision-style -``{boxes, labels}`` target as a :class:`~recordstream.Regions` item: +``{boxes, labels}`` target as a :class:`~recordstream.Boxes` item: * :class:`recordstream.ops.target.CocoToTorchVisionDetection` — a HuggingFace / COCO ``objects`` - annotation → a target ``Regions``; -* :class:`recordstream.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Regions``. + annotation → a target ``Boxes``; +* :class:`recordstream.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Boxes``. Each op REUSES its conversion helper, so the op's ``boxes`` / ``labels`` tensors are pinned byte-identical to the helper (parity). recordstream-only — no domain-package import. @@ -16,7 +16,7 @@ import torch from confluid.registry import get_registry, resolve_class -from recordstream import Image, Label, Mask, Regions, collate_records +from recordstream import Boxes, Image, Label, Mask, collate_records from recordstream.ops.target import ( CocoToTorchVisionDetection, MasksToDetectionBoxes, @@ -44,7 +44,7 @@ class TestCocoToTorchVisionDetection: def test_produces_target_regions(self) -> None: out = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) regions = out["target"] - assert isinstance(regions, Regions) + assert isinstance(regions, Boxes) assert isinstance(regions.boxes, torch.Tensor) assert isinstance(regions.labels, torch.Tensor) assert regions.boxes.shape == (2, 4) @@ -77,7 +77,7 @@ def test_default_picks_first_label(self) -> None: def test_new_output_key_keeps_source(self) -> None: out = CocoToTorchVisionDetection(field="objects", output="det")({"objects": Label(_OBJECTS)}) - assert isinstance(out["det"], Regions) + assert isinstance(out["det"], Boxes) assert out["objects"].value == _OBJECTS # source left intact def test_missing_field_raises(self) -> None: @@ -101,7 +101,7 @@ class TestMasksToDetectionBoxes: def test_instance_mask_produces_target_regions(self) -> None: out = MasksToDetectionBoxes(field="mask")({"mask": Mask(_instance_mask())}) regions = out["target"] - assert isinstance(regions, Regions) + assert isinstance(regions, Boxes) assert isinstance(regions.boxes, torch.Tensor) assert isinstance(regions.labels, torch.Tensor) assert regions.boxes.shape == (3, 4) # three instances @@ -123,6 +123,17 @@ def test_connected_components_parity(self) -> None: assert torch.equal(out["target"].boxes, expected["boxes"]) assert torch.equal(out["target"].labels, expected["labels"]) + def test_connected_equals_instance_mode_on_disjoint_instances(self) -> None: + # The two derivations agree when every instance is its own connected component. This + # pins the axis order of the connected path: before connected_component_boxes emitted + # xyxy directly, this branch transposed row/col tuples — the bug class this guards. + mask = _instance_mask() + instance = masks_to_detection(mask) + connected = masks_to_detection((mask != 0).astype(np.uint8), connected=True) + boxes_a = sorted(map(tuple, instance["boxes"].tolist())) + boxes_b = sorted(map(tuple, connected["boxes"].tolist())) + assert boxes_a == boxes_b + def test_min_area_drops_small_instances(self) -> None: mask = _instance_mask() out = MasksToDetectionBoxes(field="mask", min_area=10)({"mask": Mask(mask)}) @@ -141,7 +152,7 @@ def test_default_picks_first_mask(self) -> None: def test_new_output_key_keeps_source(self) -> None: out = MasksToDetectionBoxes(field="mask", output="det")({"mask": Mask(_instance_mask())}) - assert isinstance(out["det"], Regions) + assert isinstance(out["det"], Boxes) assert isinstance(out["mask"], Mask) # source left intact def test_missing_field_raises(self) -> None: @@ -154,7 +165,7 @@ def test_no_mask_or_array_field_raises(self) -> None: # --------------------------------------------------------------------------- # -# Collate — per-record Regions gather into a list of detection targets. +# Collate — per-record Boxes gather into a list of detection targets. # --------------------------------------------------------------------------- # def test_record_collate_gathers_regions_as_list() -> None: a = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) @@ -163,7 +174,7 @@ def test_record_collate_gathers_regions_as_list() -> None: ) batch = collate_records([a, c]) # Variable-N boxes can't be stacked → the collate gathers them as a per-record list of tensors. - assert isinstance(batch["target"], Regions) + assert isinstance(batch["target"], Boxes) assert isinstance(batch["target"].boxes, list) and len(batch["target"].boxes) == 2 assert batch["target"].boxes[0].shape == (2, 4) assert batch["target"].boxes[1].shape == (1, 4) @@ -202,10 +213,10 @@ class TestResizeDetection: def _record(self) -> dict: import torch - from recordstream import Image, Regions + from recordstream import Boxes, Image image = Image((np.arange(40 * 20 * 3) % 256).reshape(40, 20, 3).astype(np.uint8)) # H=40, W=20 - target = Regions(boxes=torch.tensor([[5.0, 10.0, 15.0, 30.0]]), labels=torch.tensor([1])) + target = Boxes(boxes=torch.tensor([[5.0, 10.0, 15.0, 30.0]]), labels=torch.tensor([1])) return {"image": image, "target": target} def test_image_and_boxes_move_together(self) -> None: @@ -220,12 +231,12 @@ def test_image_and_boxes_move_together(self) -> None: assert out["target"].labels.tolist() == [1] def test_boxes_stay_in_their_framework(self) -> None: - from recordstream import Image, Regions + from recordstream import Boxes, Image from recordstream.ops.target import ResizeDetection record = { "image": Image(np.zeros((10, 10, 3), dtype=np.uint8)), - "target": Regions(boxes=np.array([[1.0, 1.0, 5.0, 5.0]]), labels=np.array([0])), + "target": Boxes(boxes=np.array([[1.0, 1.0, 5.0, 5.0]]), labels=np.array([0])), } out = ResizeDetection(width=20, height=20)(record) assert isinstance(out["target"].boxes, np.ndarray) diff --git a/tests/test_typed_generic_ops.py b/tests/test_typed_generic_ops.py index e2bb95a..f49bb67 100644 --- a/tests/test_typed_generic_ops.py +++ b/tests/test_typed_generic_ops.py @@ -1,11 +1,11 @@ -"""The generic array→Image→Mask→Regions ops over dict records. +"""The generic array→Image→Mask→Boxes ops over dict records. Pins the three native type-changing transforms that run the detection/segmentation front-end on plain record dicts: * :class:`recordstream.ops.image.ConvertToImage` — array-bearing key → ``Image`` item; * :class:`recordstream.ops.numpy.Threshold` — array key → boolean ``Mask`` item; -* :class:`recordstream.ops.numpy.ConnectedComponents` — ``Mask`` → ``Regions`` item. +* :class:`recordstream.ops.numpy.ConnectedComponents` — ``Mask`` → ``Boxes`` item. Each op REUSES its shared math helper, so the op output is pinned identical to the helper (parity). recordstream-only — no domain-package import. @@ -15,9 +15,9 @@ import pytest from confluid.registry import get_registry, resolve_class -from recordstream import Image, Mask, Regions +from recordstream import Boxes, Image, Mask from recordstream.ops.image import ConvertToImage, _bound_longest_side, _render_rgb -from recordstream.ops.numpy import ConnectedComponents, Threshold, connected_component_bboxes, threshold_array +from recordstream.ops.numpy import ConnectedComponents, Threshold, connected_component_boxes, threshold_array def _ramp_2d() -> np.ndarray: @@ -26,8 +26,8 @@ def _ramp_2d() -> np.ndarray: def _blob_mask() -> np.ndarray: m = np.zeros((6, 6), dtype=bool) - m[0:2, 0:2] = True # blob A (area 4) -> (0, 1, 0, 1) - m[4:6, 4:6] = True # blob B (area 4) -> (4, 5, 4, 5) + m[0:2, 0:2] = True # blob A (area 4) -> xyxy (0, 0, 2, 2) + m[4:6, 4:6] = True # blob B (area 4) -> xyxy (4, 4, 6, 6) return m @@ -74,7 +74,7 @@ def test_missing_explicit_field_raises(self) -> None: def test_no_array_field_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - ConvertToImage()({"lbl": Regions(boxes=[[0, 0, 1, 1]])}) + ConvertToImage()({"lbl": Boxes(boxes=[[0, 0, 1, 1]])}) # --------------------------------------------------------------------------- # @@ -119,7 +119,7 @@ def test_no_bound_raises(self) -> None: def test_default_field_picks_first_array(self) -> None: # No explicit field: first array-bearing item (insertion order). - rec = {"raw": Mask(_ramp_2d()), "other": Regions(boxes=[])} + rec = {"raw": Mask(_ramp_2d()), "other": Boxes(boxes=[])} out = Threshold(low_level=20.0)(rec) assert np.array_equal(np.asarray(out["mask"]), _ramp_2d() > 20.0) @@ -129,28 +129,42 @@ def test_missing_explicit_field_raises(self) -> None: def test_non_array_field_raises(self) -> None: with pytest.raises(TypeError, match="expected an array"): - Threshold(low_level=1.0, field="reg")({"reg": Regions(boxes=[])}) + Threshold(low_level=1.0, field="reg")({"reg": Boxes(boxes=[])}) def test_no_array_field_default_raises(self) -> None: with pytest.raises(ValueError, match="no array-bearing field"): - Threshold(low_level=1.0)({"reg": Regions(boxes=[])}) + Threshold(low_level=1.0)({"reg": Boxes(boxes=[])}) # --------------------------------------------------------------------------- # # ConnectedComponents # --------------------------------------------------------------------------- # class TestConnectedComponents: - def test_produces_regions_bin_box_contract(self) -> None: - out = ConnectedComponents()({"m": Mask(_blob_mask())}) - regions = out["boxes"] - assert isinstance(regions, Regions) - # The pinned generic contract: (row_min, row_max, col_min, col_max) inclusive tuples. - assert regions.boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] + def test_produces_boxes_half_open_xyxy_contract(self) -> None: + mask = _blob_mask() + out = ConnectedComponents()({"m": Mask(mask)}) + boxes = out["boxes"] + assert isinstance(boxes, Boxes) + # The pinned pixel contract: HALF-OPEN xyxy (x0, y0, x1, y1) — x = col, y = row. + assert boxes.boxes == [(0, 0, 2, 2), (4, 4, 6, 6)] + # mask[y0:y1, x0:x1] covers each component exactly (the half-open property). + for x0, y0, x1, y1 in boxes.boxes: + assert mask[y0:y1, x0:x1].all() + + def test_canvas_is_the_mask_shape_even_for_an_empty_mask(self) -> None: + # canvas is filled in for an EMPTY box set too — a frame check that skips exactly + # the records with nothing to check reports a clean bill for the wrong reason. + empty = np.zeros((6, 6), dtype=bool) + out = ConnectedComponents()({"m": Mask(empty)}) + assert out["boxes"].boxes == [] + assert out["boxes"].canvas == (6, 6) + full = ConnectedComponents()({"m": Mask(_blob_mask())}) + assert full["boxes"].canvas == (6, 6) def test_parity_with_helper(self) -> None: mask = _blob_mask() out = ConnectedComponents()({"m": Mask(mask)}) - expected = connected_component_bboxes(mask) + expected = connected_component_boxes(mask) assert out["boxes"].boxes == expected def test_min_area_bins_filters_small_blobs(self) -> None: @@ -158,7 +172,7 @@ def test_min_area_bins_filters_small_blobs(self) -> None: m[0:2, 0:2] = True # area 4 m[5, 5] = True # area 1 -> dropped when min_area_bins=2 out = ConnectedComponents(min_area_bins=2)({"m": Mask(m)}) - assert out["boxes"].boxes == [(0, 1, 0, 1)] + assert out["boxes"].boxes == [(0, 0, 2, 2)] def test_connectivity_parity(self) -> None: # Diagonal touch: 4-connectivity keeps two blobs, 8 merges them. @@ -174,12 +188,12 @@ def test_default_prefers_mask_over_other_array(self) -> None: # An Image is inserted first, but a Mask is preferred by the default resolver. rec = {"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())} out = ConnectedComponents()(rec) - assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] + assert out["boxes"].boxes == [(0, 0, 2, 2), (4, 4, 6, 6)] def test_falls_back_to_first_array_when_no_mask(self) -> None: # No Mask item — a 2-D array item is used. out = ConnectedComponents()({"m": Image(_blob_mask())}) - assert out["boxes"].boxes == [(0, 1, 0, 1), (4, 5, 4, 5)] + assert out["boxes"].boxes == [(0, 0, 2, 2), (4, 4, 6, 6)] def test_non_2d_mask_raises(self) -> None: with pytest.raises(ValueError, match="2-D mask"): @@ -191,26 +205,27 @@ def test_missing_explicit_field_raises(self) -> None: def test_no_mask_or_array_raises(self) -> None: with pytest.raises(ValueError, match="no Mask or array-bearing field"): - ConnectedComponents()({"reg": Regions(boxes=[])}) + ConnectedComponents()({"reg": Boxes(boxes=[])}) # --------------------------------------------------------------------------- # -# End-to-end chain: array -> Image -> Mask -> Regions, all on one record dict. +# End-to-end chain: array -> Image -> Mask -> Boxes, all on one record dict. # --------------------------------------------------------------------------- # -def test_array_to_image_to_mask_to_regions_chain() -> None: +def test_array_to_image_to_mask_to_boxes_chain() -> None: arr = _ramp_2d() record = {"spec": Mask(arr)} out = ConnectedComponents(field="mask")(Threshold(field="spec", low_level=20.0)(ConvertToImage()(record))) # Every stage produced its typed entry. assert isinstance(out["image"], Image) assert isinstance(out["mask"], Mask) - assert isinstance(out["boxes"], Regions) - # Regions carries (row_min, row_max, col_min, col_max) bin-box tuples. + assert isinstance(out["boxes"], Boxes) + # Boxes carries HALF-OPEN xyxy (x0, y0, x1, y1) tuples framed by the mask raster. assert out["boxes"].boxes + assert out["boxes"].canvas == arr.shape for box in out["boxes"].boxes: assert len(box) == 4 - row_min, row_max, col_min, col_max = box - assert row_min <= row_max and col_min <= col_max + x0, y0, x1, y1 = box + assert x0 < x1 and y0 < y1 # The image entry carries the pixel dims via its shape (no separate metadata). assert np.asarray(out["image"]).shape[:2] == arr.shape diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py index e59c0d9..304676e 100644 --- a/tests/test_typed_storage.py +++ b/tests/test_typed_storage.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from recordstream import Image, Label, Regions, register_item +from recordstream import Boxes, Image, Label, register_item from recordstream.storage.base import TYPED_FORMAT, require_record_format, restore_attrs, split_attrs from recordstream.storage.directory import DirectorySink, DirectorySource from recordstream.storage.hdf5 import HDF5Sink, HDF5Source @@ -31,7 +31,7 @@ def _records() -> list: r0 = { "image": Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW"), "sig": _StoreSig(np.arange(8, dtype=np.float32), samplerate=20e6, mask=np.array([1, 0, 1], dtype=np.uint8)), - "regions": Regions(boxes=[[0, 0, 1, 1], [1, 1, 2, 2]], labels=["a", "b"], canvas=(2, 2)), + "regions": Boxes(boxes=[[0, 0, 1, 1], [1, 1, 2, 2]], labels=["a", "b"], canvas=(2, 2)), "label": Label("drone", classes=["x", "drone"]), "gain_db": -3.0, "source_file": "a.iq", @@ -40,7 +40,7 @@ def _records() -> list: r1 = { "image": Image(np.ones((2, 2, 3), dtype=np.float32)), "sig": _StoreSig(np.zeros(4, dtype=np.float32), samplerate=1e6, mask=np.array([0], dtype=np.uint8)), - "regions": Regions(boxes=[[0, 0, 2, 2]], labels=["c"], canvas=(2, 2)), + "regions": Boxes(boxes=[[0, 0, 2, 2]], labels=["c"], canvas=(2, 2)), "label": Label("x", classes=["x", "drone"]), "gain_db": 1.5, "source_file": "b.iq", From 225cb87569591dd6fa8faa6fd2815b9d802330dd Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 10 Aug 2026 13:51:44 +0200 Subject: [PATCH 082/102] fix: view sources answer a still-deferred source: marker with Stream's actionable guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A parens-less !class:X in a source: slot is a CONFIG error (the fix is !class:X() ), and the view sources (RangeSource/ConcatSource/DatasetSplit via sources/base.py::_guard_live_source) now raise the same slot-naming TypeError Stream does instead of a cryptic 'got Class' — deliberately message-only, never flowed (user decision 2026-08-10; distinct from the flow-first free functions). _fluid_source_guidance takes the owning slot name. Docs note in docs/sources.md; pins in tests/test_view_sources_deferred.py. --- AGENTS.md | 2 +- docs/sources.md | 6 ++ recordstream/core/__init__.py | 1 + recordstream/core/stream.py | 14 ++-- recordstream/sources/base.py | 19 ++++++ recordstream/sources/concat.py | 3 +- recordstream/sources/range.py | 3 +- recordstream/sources/split.py | 3 +- tests/test_view_sources_deferred.py | 99 +++++++++++++++++++++++++++++ 9 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 tests/test_view_sources_deferred.py diff --git a/AGENTS.md b/AGENTS.md index d7a983c..6b4eb1c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. - **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.execute.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). - **1→N Expanding Steps Fork the REMAINING Subgraph (2026-07-30, supersedes the flat-engine pending-queue rule):** An op carrying `EXPANDS = True` yields N children from one record; the remaining steps then run ONCE PER CHILD over that child's own shallow copy of the step environment (independent name→result maps, shared values), DEPTH-FIRST so sibling order matches the nested-loop intuition. An empty expansion or a `None` child drops that branch. This works in EVERY route — serial, spawn-parallel (the worker returns a LIST), and inside a `flow:` graph (the old `FlowGraph` raised `NotImplementedError` on an expanding step; that limit is gone). CONSEQUENCES: (1) `__len__`/`__getitem__` RAISE on `Stream` AND `FlowGraph` when any step op expands — the expanded index map is unknowable up front, so the pipeline is ITERABLE-ONLY (iterate, wrap in a torch IterableDataset, window at the SOURCE for random access, or `list(...)`); (2) `run_steps` (the strict 1→1 twin used for indexing) raises rather than silently dropping siblings. Pins: `tests/test_typed_flow.py::TestExpandingSteps`. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. **A `source:` slot holding a still-deferred `!class:X` marker RAISES with guidance, never flows (2026-08-10, user decision):** the parens-less spelling is a CONFIG error (`!class:X()` is the fix), and `Stream._guard_live_source` + the view sources' shared `sources/base.py::_guard_live_source` answer it with the same actionable `TypeError` (naming the slot, the deferred target, and the parens fix — `core.stream._fluid_source_guidance`). Do NOT "fix" a `got Class` failure by flowing the slot — the 2026-07-29 flow-first convention covers the FREE FUNCTIONS (`project`/`dataset_uri`/`LabelMap.encode`) and the `ops:` list (`_check_ops_materialized`), not `source:` slots. Pins: `tests/test_view_sources_deferred.py`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. - **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. - **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core.families._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). - **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). diff --git a/docs/sources.md b/docs/sources.md index bd3eee2..cdc6423 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -99,6 +99,12 @@ val_set: !class:recordstream.sources.split.DatasetSplit() **HuggingFace native slicing** (alternative, no RecordStream split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. +> **Nesting a source inline: write `!class:X()`, with parens.** A parens-less `!class:X` is a +> *deferred marker*, not an instance — a `source:` slot holding one fails at first use with an +> error naming the slot and this fix (the same guidance `Stream` gives). The examples above +> sidestep this with `!ref:` to a top-level instance, which is also what lets several wrappers +> share one loaded source. + ## Identifying a dataset A source can name the data it reads, so a run record, a report, or a log line can point at it. diff --git a/recordstream/core/__init__.py b/recordstream/core/__init__.py index b386627..bc39387 100644 --- a/recordstream/core/__init__.py +++ b/recordstream/core/__init__.py @@ -47,6 +47,7 @@ JointStream, Stream, _check_ops_materialized, + _fluid_source_guidance, _worker_task, ensure_materialized, ensure_record_dataset, diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py index 5364226..2e7f6e8 100644 --- a/recordstream/core/stream.py +++ b/recordstream/core/stream.py @@ -48,14 +48,18 @@ def _describe_deferred_source(source: Any) -> str: return f"{type(source).__name__}(target={target_name!r})" -def _fluid_source_guidance(source: Any) -> str: - """Build an actionable message when Stream.source is still a Confluid Fluid.""" +def _fluid_source_guidance(source: Any, slot: str = "Stream.source") -> str: + """Build an actionable message when a source slot is still a Confluid Fluid. + + ``slot`` names the owning slot (``"Stream.source"``, ``"RangeSource.source"``, …) so the + view sources raise the SAME guidance Stream does — a still-deferred ``source:`` is a + CONFIG error (the parens-less ``!class:X`` spelling), never something the engine flows. + """ return ( - f"Stream.source is still a deferred Confluid marker: {_describe_deferred_source(source)}. " + f"{slot} is still a deferred Confluid marker: {_describe_deferred_source(source)}. " "Confluid has not materialized it yet. Fixes: (a) in YAML, write the source as " "`!class:X()` (with parens) instead of `!class:X` so it becomes an Instance and is " - "materialized at load time; (b) or call `flow(source)` on the source before handing " - "it to Stream." + "materialized at load time; (b) or call `flow(source)` on the source before wiring it." ) diff --git a/recordstream/sources/base.py b/recordstream/sources/base.py index 4578a80..ae60c56 100644 --- a/recordstream/sources/base.py +++ b/recordstream/sources/base.py @@ -2,6 +2,25 @@ from typing import Any +from confluid.fluid import Fluid as _ConfluidFluid + +from recordstream.core import _fluid_source_guidance + + +def _guard_live_source(source: Any, slot: str) -> None: + """Raise Stream's actionable deferred-marker error when ``source`` is still a Fluid. + + A still-deferred ``!class:X`` (parens-less) marker in a ``source:`` slot is a CONFIG + error, and the view sources answer it exactly as ``Stream._guard_live_source`` does — + naming the slot and the deferred target, and pointing at the ``!class:X()`` fix — + instead of the cryptic ``got Class`` a bare ``hasattr`` check produces. Deliberately + message-only: the slot is never flowed here (the raise-with-guidance convention for + ``source:`` slots, distinct from the free functions ``project`` / ``dataset_uri``, + which do materialize a marker first). + """ + if isinstance(source, _ConfluidFluid): + raise TypeError(_fluid_source_guidance(source, slot=slot)) + def _pass_through(item: Any) -> Any: """Pass a wrapped source's item through verbatim. diff --git a/recordstream/sources/concat.py b/recordstream/sources/concat.py index 3b93bd6..238498b 100644 --- a/recordstream/sources/concat.py +++ b/recordstream/sources/concat.py @@ -6,7 +6,7 @@ from confluid import configurable from recordstream.items import Record -from recordstream.sources.base import _pass_through +from recordstream.sources.base import _guard_live_source, _pass_through @configurable(category="source") @@ -43,6 +43,7 @@ def offsets(self) -> List[int]: offsets: List[int] = [] total = 0 for i, src in enumerate(self.sources): + _guard_live_source(src, f"ConcatSource.sources[{i}]") if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): raise TypeError( "ConcatSource requires sources supporting __len__ and __getitem__; " diff --git a/recordstream/sources/range.py b/recordstream/sources/range.py index 0e5771b..6580332 100644 --- a/recordstream/sources/range.py +++ b/recordstream/sources/range.py @@ -6,7 +6,7 @@ from loggair import get_logger from recordstream.items import Record -from recordstream.sources.base import _pass_through +from recordstream.sources.base import _guard_live_source, _pass_through logger = get_logger(__name__) @@ -41,6 +41,7 @@ def indices(self) -> List[int]: """The contiguous ``[start:stop)`` source indices, computed lazily on first access and cached.""" if self._indices is None: source = self.source + _guard_live_source(source, "RangeSource.source") if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): raise TypeError( "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" diff --git a/recordstream/sources/split.py b/recordstream/sources/split.py index 609f4ac..e77e60c 100644 --- a/recordstream/sources/split.py +++ b/recordstream/sources/split.py @@ -6,7 +6,7 @@ from confluid import configurable from recordstream.items import Record -from recordstream.sources.base import _pass_through +from recordstream.sources.base import _guard_live_source, _pass_through # Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer # closed Literals over bare strings — self-documenting + machine-introspectable by UIs / @@ -93,6 +93,7 @@ def __init__( def _validate(self) -> None: """Validate the (post-construction) configuration. Called lazily before the first partition.""" source = self.source + _guard_live_source(source, "DatasetSplit.source") if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): raise TypeError( "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" diff --git a/tests/test_view_sources_deferred.py b/tests/test_view_sources_deferred.py new file mode 100644 index 0000000..8903853 --- /dev/null +++ b/tests/test_view_sources_deferred.py @@ -0,0 +1,99 @@ +"""A still-deferred ``!class:`` marker in a view source's ``source:`` slot explains itself. + +The parens-less ``!class:X`` YAML spelling leaves a Confluid ``Fluid`` marker in the slot — +a CONFIG error (the fix is ``!class:X()``), and the view sources answer it with the same +actionable guidance ``Stream`` gives (naming the slot, the deferred target, and the parens +fix) instead of the cryptic ``got Class`` their bare ``hasattr`` checks used to produce. +Deliberately message-only: the slot is never flowed (the raise-with-guidance convention for +``source:`` slots — distinct from the free functions ``project`` / ``dataset_uri``). +""" + +import pytest +from confluid import load + +from recordstream import Stream +from recordstream.sources import ConcatSource, DatasetSplit, RangeSource + +_LEAF = "recordstream.sources.concat.ConcatSource" + + +def _expect_guidance(excinfo: "pytest.ExceptionInfo[TypeError]", slot: str) -> None: + """The error names the slot, says WHAT was deferred, and states the parens fix.""" + message = str(excinfo.value) + assert slot in message + assert "deferred Confluid marker" in message + assert "ConcatSource" in message # the deferred target, not just "Class" + assert "!class:X()" in message # the actionable fix + + +def test_a_parens_less_marker_under_range_source_raises_the_stream_guidance() -> None: + cfg = load( + f""" +range_src: !class:recordstream.sources.range.RangeSource() + source: !class:{_LEAF} + stop: 3 +""", + flow=True, + ) + with pytest.raises(TypeError) as excinfo: + cfg["range_src"].indices + _expect_guidance(excinfo, "RangeSource.source") + + +def test_a_parens_less_marker_under_concat_source_names_the_offending_index() -> None: + cfg = load( + f""" +concat_src: !class:recordstream.sources.concat.ConcatSource() + sources: + - !class:{_LEAF}() + - !class:{_LEAF} +""", + flow=True, + ) + with pytest.raises(TypeError) as excinfo: + cfg["concat_src"].offsets + _expect_guidance(excinfo, "ConcatSource.sources[1]") + + +def test_a_parens_less_marker_under_dataset_split_raises_the_stream_guidance() -> None: + cfg = load( + f""" +split_src: !class:recordstream.sources.split.DatasetSplit() + source: !class:{_LEAF} +""", + flow=True, + ) + with pytest.raises(TypeError) as excinfo: + len(cfg["split_src"].train) + _expect_guidance(excinfo, "DatasetSplit.source") + + +def test_the_parens_spelling_materializes_and_the_view_sources_work() -> None: + """The positive counterpart: ``!class:X()`` becomes a live instance at load time.""" + cfg = load( + f""" +range_src: !class:recordstream.sources.range.RangeSource() + source: !class:{_LEAF}() + stop: 3 +concat_src: !class:recordstream.sources.concat.ConcatSource() + sources: + - !class:{_LEAF}() +split_src: !class:recordstream.sources.split.DatasetSplit() + source: !class:{_LEAF}() +""", + flow=True, + ) + range_src: RangeSource = cfg["range_src"] + concat_src: ConcatSource = cfg["concat_src"] + split_src: DatasetSplit = cfg["split_src"] + assert isinstance(range_src.source, ConcatSource) + assert range_src.indices == [] # an empty leaf clamps [0:3) to nothing — but it computed + assert concat_src.offsets == [0] + assert len(split_src.train) == 0 + + +def test_streams_own_guidance_still_names_its_slot() -> None: + """The shared message helper's default slot stays ``Stream.source``.""" + cfg = load(f"deferred: !class:{_LEAF}", flow=True) + with pytest.raises(TypeError, match="Stream.source is still a deferred Confluid marker"): + len(Stream(source=cfg["deferred"])) From 799ed1fa72d6e1d6fbb346882ad8bbaf9277c1d5 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 10 Aug 2026 14:07:43 +0200 Subject: [PATCH 083/102] =?UTF-8?q?feat:=20resolve=5Fentry/resolve=5Fitem?= =?UTF-8?q?=20=E2=80=94=20the=20one=20field-or-first=20record-key=20resolu?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An op with a field=-style knob resolves its entry through ONE helper: an explicit field must exist and hold the expected item type (ValueError naming owner, param and the record's keys), a blank/None field falls back to the first value of that type. fallback=False forbids the blank-field scan (mandatory-key ops), required=False turns every miss into None (the probe form); resolve_entry returns (key, value) for ops that write back to the resolved key. Typed overloads keep required=True call sites Optional-free. Extracted from twelve byte-parallel _find_* copies in the signal package — the item_value/first_value story again. --- AGENTS.md | 4 +- docs/record-model.md | 14 +++++ recordstream/__init__.py | 4 ++ recordstream/items.py | 130 ++++++++++++++++++++++++++++++++++++++- tests/test_items.py | 64 +++++++++++++++++++ 5 files changed, 213 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6b4eb1c..f36a0ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — and since 2026-08-05 the COLLATE ITSELF is selectable here too (`collate=` takes a registered key or a function, resolved per batch so a key registered later still works), because a `PyDataset` otherwise had no way to say "don't stack" and the batch-shape choice was torch-only — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). - **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. - **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. -- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Boxes` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Boxes`/`Label`). **`Boxes` is PIXEL-ONLY (renamed from `Regions`, 2026-08-10, NO alias): half-open absolute-pixel `[x0, y0, x1, y1]` rows on the `(H, W)` `canvas` raster — the signal-domain time/frequency region item lives in the signal package and registers through the same `register_item` registry; a stored `typedrecord-v1` record carrying `__item_type__: "Regions"` fails loudly on decode and is re-generated (rationale: `docs/architecture.md` §14). `connected_component_boxes` (renamed WITH its contract from `connected_component_bboxes`) emits the SAME half-open xyxy order, `ConnectedComponents` fills `canvas` (empty masks included), and the geometry guards now correctly stay silent for a domain package's raster-independent region item.** `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Boxes, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Boxes`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Boxes` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Boxes`/`Label`). **`Boxes` is PIXEL-ONLY (renamed from `Regions`, 2026-08-10, NO alias): half-open absolute-pixel `[x0, y0, x1, y1]` rows on the `(H, W)` `canvas` raster — the signal-domain time/frequency region item lives in the signal package and registers through the same `register_item` registry; a stored `typedrecord-v1` record carrying `__item_type__: "Regions"` fails loudly on decode and is re-generated (rationale: `docs/architecture.md` §14). `connected_component_boxes` (renamed WITH its contract from `connected_component_bboxes`) emits the SAME half-open xyxy order, `ConnectedComponents` fills `canvas` (empty masks included), and the geometry guards now correctly stay silent for a domain package's raster-independent region item.** `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; **`resolve_item`/`resolve_entry` are the same rule for the KEY question (2026-08-10)** — a `field=`-style op resolves its entry through them (explicit field must exist + match the item type with a `ValueError` naming owner/param/keys; blank field = first-of-type; `fallback=False` forbids the blank-field scan, `required=False` returns `None` — extracted from twelve per-class `_find_*` copies in one consumer package), never a hand-written find loop; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Boxes, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Boxes`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. - **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). **A geometry-changing transform WARNS when a `Boxes` sat out the call (2026-08-06):** the vocabulary rule above is what makes a bare library transform work unmodified, but a detection target rides as a `Boxes` item under a key of the pipeline's choosing — so it is not in that vocabulary, is not passed, and does not move. Measured: a bare `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes on `[10, 10, 100, 100]`, and a bare `A.HorizontalFlip` mirrors the pixels while changing NO shape at all. Nothing errors either way — the shapes stay valid and only the coordinates become wrong. **The condition is the LIBRARY'S OWN taxonomy, not a raster comparison**: a `DualTransform` is by definition one that applies to boxes, an `ImageOnlyTransform` cannot touch geometry, so `_has_spatial_transform` matches `DualTransform` by MRO class NAME (import-free, like `_is_albumentations`) and recurses a `Compose` through `.transforms` — the flip case proves why a "did the size change" test is not enough, and reading the taxonomy means no list of transform names to drift. WARNING not error (a record may legitimately carry regions describing something else), once per transform TYPE (`_WARNED_SPATIAL` — the message is about the configuration), and silent when `bboxes` WAS passed. The message names both ways out: `bbox_params` on a `Compose`, or `ops.target.ResizeDetection` for a plain coupled resize. Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (incl. `TestGeometryLeavingBoxesBehind`, which asserts the desync PREMISE before the guard). **The v2 family has the SAME gap by the other route and the same guard (2026-08-06):** v2 walks the record natively but transforms only its own `tv_tensors` TYPES, so a `Boxes` is passed through untouched — measured, `v2.Resize((64,64))` takes a 200x200 image to 64x64 with the boxes still on `[10,10,100,100]`, while the same transform over a `tv_tensors.BoundingBoxes` gives `[3.2, 3.2, 32, 32]`. `_is_v2_geometry` matches v2's private `torchvision.transforms.v2._geometry` module in the MRO (the library offers no public marker) and shares `_WARNED_SPATIAL` with the albumentations guard. **The behavioural test — apply the transform to a throwaway `BoundingBoxes` and see if they move — is deliberately NOT used: it would draw from the RNG and change the augmentation stream of the run being diagnosed.** The private path can go stale on a torchvision upgrade; it FAILS OPEN (no warning, nothing else changes), so `test_the_geometry_signal_still_matches_this_torchvision` asserts the CLASSIFICATION directly rather than only through a warning that would silently stop appearing (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). - **OpenCV's Thread Pool Is The SECOND Fork Hazard, And The Guard Fires On USE (`core.families._disable_cv2_threading`, 2026-08-02):** the companion to the `ensure_materialized` mandate below, and **independent of it — neither fixes the other**. albumentations runs on OpenCV, whose thread pool is not fork-safe: once the PARENT has executed a cv2 op, a forked `DataLoader` worker inheriting that pool dies with a **SIGSEGV and no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` — the same symptom as the lazy-source hazard, which is exactly why they get confused for each other. Measured 2026-08-02 with `DataLoader(num_workers=2)` over a `Stream` whose ops list held an `A.Resize`: **with the source warmed and the pool ON the worker still SIGSEGVs, and with the pool off a cold source is still built in the child.** A consumer that forks needs BOTH guards. `cv2.setNumThreads(0)` is fired ONCE at the top of `_invoke_albumentations` — the one place this package INVOKES albumentations — and never at import: a process that never uses albumentations must not have its OpenCV settings changed by importing a data library, and cv2 must not become an import-time dependency of the engine. It costs nothing where it matters, because inside a worker the WORKER is the parallelism and cv2's own threads oversubscribe rather than help (albumentations' own docs recommend exactly this for multiprocessing loaders). Two cv2 quirks a test must not get wrong, both measured: `setNumThreads(0)` makes `getNumThreads()` report **1**, not 0; and `setNumThreads(4)` does not change what it reports at all. **Why the classification consumer never hit this and the segmentation one did:** classification resizes with `ConvertToImage` (PIL), while a per-pixel task must resize the image and its mask in ONE JOINT DRAW — which only a bare albumentations transform does. Segmentation is the first thing in the workspace to put cv2 on the worker path. Pins: `tests/test_fork_safety.py` (the forked-worker subprocess reproduction is the one that matters — removing the guard makes it FAIL, not pass differently). - **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Boxes` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. **A `Boxes`' `canvas` is the load-bearing case of that rule, and EVERY op that makes or re-frames one fills it in (2026-08-06):** a box means nothing without the raster it is stated in, so `CocoToTorchVisionDetection` records the frame of the image its annotation describes, `MasksToDetectionBoxes` the shape of the mask its boxes were derived from (exact and free), and `ResizeDetection` the size it resized to — **including for an EMPTY target**, because a check that silently skips exactly the records with nothing to check reports a clean bill for the wrong reason. Previously ONLY the resize set it, so the frame was known exactly where it was least needed and unknown in the chain where boxes and pixels actually drift apart. The lookup (`ops.target._source_frame`) is deliberately NARROW — the `"image"` key, then the first `Image` item, then `None` — because a generic "first 2-D array" search reads a `Boxes`' own `[N, 4]` box array as an `N x 4` raster and records a confident lie (pinned). `None` stays an ordinary answer: nothing depends on the lookup succeeding. **The desync this makes detectable is otherwise SILENT:** an image-only resize (`ConvertToImage` and friends) moves pixels without moving boxes, every downstream SHAPE stays valid, only the coordinates are wrong, and a model trains against misplaced targets reporting nothing — so `ConvertToImage` WARNS once per op instance when it resizes a record carrying a `Boxes`, naming `ResizeDetection`. Once per INSTANCE, not per record: the message is about the configuration, and a copy per record only buries it. **`ops.image.image_frame(value)` is the shared "what raster is this" read** (PIL `.size` transposed, an `Image` item's DECLARED `layout`, else HWC) — it does NOT sniff a channel axis, because this package already carries three deliberately divergent channels-first heuristics and a guessing fourth would mislabel the very frame box coordinates are validated against. Consumer note: a downstream frame check that skipped on `canvas is None` now fires in chains it used to pass (verified on a real COCO-style set — the recorded frame matches the dataset's own `width`/`height` columns). Pins: `tests/test_convert_to_mask.py::TestBoxesKnowTheirFrame` (incl. the box-array-is-not-a-raster case and the once-per-op warning). NOTE for any test asserting on that warning: loggair is loguru, so `caplog` stays EMPTY, and its sink is ENQUEUED so `capfd` alone races it — add a sink and call `logger.complete()` (the mandated flush, never a sleep). Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. @@ -37,7 +37,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. - **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. -- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. +- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. **`read_record_group(group, slices=…)` is the PUBLIC HDF5 row decoder (2026-08-10)** — `HDF5Source` iterates through it, and the `slices` map ({record key → slice of that field's `data` dataset}) is an h5py PARTIAL read for consumers that window large stored rows (a domain windowing source slices one window out of a whole-capture payload without materializing the row); the sliced decode is pinned identical to full-read-then-slice by `tests/test_typed_storage.py::TestReadRecordGroupSlices`. Do not re-derive the row-decode walk outside this function. - **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. - **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. **`first_value(source, key)` is the ONE-PEEK primitive beside them (2026-08-02)** — the first non-`None` value under `key`, or `None` when there is none. It answers what a column's values ARE without walking the set, and it belongs here rather than in any consumer because it is `iter_key` plus a `next()`: it inherits all three of that helper's properties (a projection-aware source never builds the values it does not ask for, a deferred source is materialized first, the walk is lazy so a normal source costs ONE record) and its unwrapping rules are what make the answer meaningful — a `MultiLabel` arrives as its `.values` LIST, so a sequence IS a multi-label column, decided by the item type rather than by guessing what a list might mean. The canonical call site pairs it with `is_class_id` to decide whether the targets need a `LabelMap` at all. It was extracted from EIGHT byte-identical private copies in one consumer's training backends (2026-08-02); a consumer re-deriving it is re-deriving `iter_key`'s contract. Pins: `tests/test_projection.py`. - **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. diff --git a/docs/record-model.md b/docs/record-model.md index 1b3628f..690425e 100644 --- a/docs/record-model.md +++ b/docs/record-model.md @@ -80,6 +80,20 @@ want, and why the rule is one function rather than three copies of it. registry — the extensibility surface a domain package or user type plugs into (one class + one decorator, no core edit). +An op with a `field=`-style knob resolves the entry it reads with `resolve_item` (or +`resolve_entry` when it also needs the key back) — an explicit field must exist and hold the +expected item type (each miss raises a `ValueError` naming the op, the parameter and the record's +keys), while a blank field falls back to the first value of that type: + +```python +from recordstream import Image, resolve_item +image = resolve_item(record, self.image_field, Image, owner="SaveImage", param="image_field") +``` + +`fallback=False` makes a blank field an error too (for ops whose key is mandatory config); +`required=False` turns every miss into `None` (the probe form). Twelve per-class `_find_*` +copies in one consumer package predated the extraction — don't re-derive the branch. + ### Ops — type dispatch with once-per-record parameters A `Transform` (`recordstream.transform`) samples its parameters ONCE per record diff --git a/recordstream/__init__.py b/recordstream/__init__.py index 41c4557..b25a548 100644 --- a/recordstream/__init__.py +++ b/recordstream/__init__.py @@ -67,6 +67,8 @@ item_types, item_value, register_item, + resolve_entry, + resolve_item, with_data, ) from recordstream.labels import LabelMap, class_counts, inverse_frequency_weights @@ -123,6 +125,8 @@ "is_item", "item_data", "item_value", + "resolve_entry", + "resolve_item", "with_data", "Transform", "Pipeline", diff --git a/recordstream/items.py b/recordstream/items.py index 8547478..e701c11 100644 --- a/recordstream/items.py +++ b/recordstream/items.py @@ -39,7 +39,7 @@ """ from dataclasses import dataclass, field, fields, is_dataclass, replace -from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, cast +from typing import Any, Dict, List, Literal, Optional, Tuple, Type, TypeVar, cast, overload import numpy as np @@ -66,6 +66,8 @@ "is_item", "item_data", "item_value", + "resolve_entry", + "resolve_item", "with_data", ] @@ -334,3 +336,129 @@ def with_data(item: _ItemT, new_data: Any) -> _ItemT: if is_dataclass(item) and not isinstance(item, type) and any(f.name == "data" for f in fields(item)): return cast(_ItemT, replace(cast(Any, item), data=new_data)) raise TypeError(f"with_data: {type(item).__name__} has no payload slot to replace") + + +_ResolvedT = TypeVar("_ResolvedT") + + +@overload +def resolve_entry( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[True] = ..., +) -> Tuple[str, _ResolvedT]: ... + + +@overload +def resolve_entry( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[False] = ..., +) -> Optional[Tuple[str, _ResolvedT]]: ... + + +def resolve_entry( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = "field", + fallback: bool = True, + required: bool = True, +) -> Optional[Tuple[str, _ResolvedT]]: + """Resolve the ``(key, value)`` entry of ``item_type`` a field-pinned op reads. + + THE record-key resolution every ``field=``-style op used to re-derive per class + (twelve byte-parallel private ``_find_*`` copies in one consumer package before the + extraction — the same story as :func:`item_value` and ``first_value``): an explicit + ``field`` must exist and hold an ``item_type`` value — each miss raises a + ``ValueError`` naming ``owner``, ``param`` and the record's keys — while a blank + ``field`` falls back to the FIRST ``item_type`` value in record order. + + Args: + record: The record dict being resolved against. + field: The configured record key; blank/``None`` engages the first-of-type fallback. + item_type: The item class the entry must be an instance of. + owner: The op/class name error messages lead with (e.g. ``"SaveImage"``). + param: The ctor-param name error messages cite (e.g. ``"image_field"``). + fallback: When False, a blank ``field`` is an error like any other missing key — + for ops whose key is mandatory config rather than a convenience default. + required: When False, every miss returns ``None`` instead of raising (the probe + form — "use it if the record has one"). + + Returns: + ``(key, value)`` — the resolved record key and its typed value; ``None`` only + when ``required=False`` missed. + """ + type_name = item_type.__name__ + if field or not fallback: + if field not in record: + if not required: + return None + raise ValueError(f"{owner}: {param} {field!r} not in record (keys: {list(record)})") + value = record[field] + if not isinstance(value, item_type): + if not required: + return None + raise ValueError(f"{owner}: {param} {field!r} is {type(value).__name__}, expected {type_name}") + return field, value + for key, value in record.items(): + if isinstance(value, item_type): + return key, value + if not required: + return None + raise ValueError(f"{owner}: no {type_name} entry in record (keys: {list(record)})") + + +@overload +def resolve_item( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[True] = ..., +) -> _ResolvedT: ... + + +@overload +def resolve_item( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[False] = ..., +) -> Optional[_ResolvedT]: ... + + +def resolve_item( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = "field", + fallback: bool = True, + required: bool = True, +) -> Optional[_ResolvedT]: + """The value half of :func:`resolve_entry` — the common case when the key is not needed.""" + entry = resolve_entry( + record, field, item_type, owner=owner, param=param, fallback=fallback, required=cast(Any, required) + ) + return None if entry is None else entry[1] diff --git a/tests/test_items.py b/tests/test_items.py index af12b2b..6a5cff7 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -134,3 +134,67 @@ def __init__(self, points: list) -> None: self.points = points assert "Keypoints" in item_type_names() and get_item_type("Keypoints") is Keypoints + + +class TestResolveEntry: + """resolve_entry/resolve_item — THE field-or-first record-key resolution.""" + + def _record(self) -> dict: + import numpy as np + + from recordstream import Image, Label + + return {"class": Label("cat"), "picture": Image(np.zeros((4, 6, 3), dtype=np.uint8)), "samplerate": 1.0} + + def test_explicit_field_returns_key_and_value(self) -> None: + from recordstream import Image, resolve_entry + + key, item = resolve_entry(self._record(), "picture", Image, owner="Op", param="image_field") + assert key == "picture" and isinstance(item, Image) + + def test_blank_field_falls_back_to_first_of_type(self) -> None: + from recordstream import Image, resolve_item + + item = resolve_item(self._record(), "", Image, owner="Op") + assert isinstance(item, Image) + + def test_missing_explicit_field_raises_naming_owner_param_and_keys(self) -> None: + import pytest + + from recordstream import Image, resolve_item + + with pytest.raises(ValueError, match=r"Op: image_field 'nope' not in record \(keys: .*picture"): + resolve_item(self._record(), "nope", Image, owner="Op", param="image_field") + + def test_wrong_type_raises_naming_actual_and_expected(self) -> None: + import pytest + + from recordstream import Image, resolve_item + + with pytest.raises(ValueError, match="Op: field 'class' is Label, expected Image"): + resolve_item(self._record(), "class", Image, owner="Op") + + def test_no_match_raises_naming_the_type(self) -> None: + import pytest + + from recordstream import Mask, resolve_item + + with pytest.raises(ValueError, match=r"Op: no Mask entry in record"): + resolve_item(self._record(), "", Mask, owner="Op") + + def test_fallback_false_makes_blank_field_an_error(self) -> None: + import pytest + + from recordstream import Image, resolve_item + + with pytest.raises(ValueError, match="Op: image_field '' not in record"): + resolve_item(self._record(), "", Image, owner="Op", param="image_field", fallback=False) + + def test_required_false_turns_every_miss_into_none(self) -> None: + from recordstream import Image, Mask, resolve_item + + record = self._record() + assert resolve_item(record, "nope", Image, owner="Op", required=False) is None + assert resolve_item(record, "class", Image, owner="Op", required=False) is None + assert resolve_item(record, "", Mask, owner="Op", required=False) is None + assert resolve_item(record, "", Image, owner="Op", required=False) is not None From 117025bb3d1b2c6411afe5fd087c54d88f47d62e Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 10 Aug 2026 14:15:05 +0200 Subject: [PATCH 084/102] feat: public read_record_group with partial payload slices The HDF5 row decoder HDF5Source iterates through is now the public read_record_group(group, slices=...): the slices map ({record key -> slice of that field's data dataset}) is an h5py PARTIAL read, so a consumer windowing large stored rows decodes one window without materializing the row. Sliced decode pinned identical to full-read-then-slice; attrs are never sliced; slicing a payload-free field raises. Documented in docs/storage.md + the storage mandate; the source-windowing TASKS item refreshed to current reality. --- TASKS.md | 2 +- docs/storage.md | 18 ++++++++++++++++ recordstream/storage/hdf5.py | 27 +++++++++++++++++++----- tests/test_typed_storage.py | 40 +++++++++++++++++++++++++++++++++++- 4 files changed, 80 insertions(+), 7 deletions(-) diff --git a/TASKS.md b/TASKS.md index 89a459d..9d944b8 100644 --- a/TASKS.md +++ b/TASKS.md @@ -8,7 +8,7 @@ workspace root `TASKS.md`. Completed items are not archived here — git history - [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor - [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `recordstream.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in recordstream; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low - [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, matrainer, …) were not. @low -- [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — `HDF5WindowSource.__iter__` (waivefront/waivefront/hdf5_source.py) and `RFUAVSource` hard-code the one-capture→N-windows loop in their generators with deliberately-approximate `__len__`. Once 1→N expanding ops land in recordstream, extract the sliding-window + `clip_regions_to_window` logic into a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor +- [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — the one-capture→N-windows loop now lives ONCE in `waivefront/sources/base.py` (`iter_windows` + `ChunkAnnotations`, shared by `waivefront.sources.rfuav.RFUAVSource` and `waivefront.sources.hdf5_window.HDF5WindowSource`, both with EXACT locator-index `__len__`/`__getitem__` since 2026-08-10), and 1→N expanding ops HAVE landed (`EXPANDS = True`). What remains is the extraction itself: a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor - [ ] **Decide whether `PredictionsSink` collapses into `DataSink`** @medium @refactor — since 2026-07-29 this package carries TWO sink protocols: `storage.base.DataSink.write(record)` (adapted into op chains by `RecordSinkOp`) and `predictions.PredictionsSink.write(prediction, metadata)`. The split is real — a model emits a BATCH while the sink contract is per-record, so the prediction and its record's metadata arrive separately — and it is load-bearing downstream, where a visual editor's node palette keys off the signature difference. The alternative: have the consuming runnable build the record (it already holds both halves — it slices the batch per record before calling `write`) and write through the ordinary `DataSink`, deleting `PredictionsSink` and letting prediction sinks become ordinary `category="sink"` canvas nodes. That touches the predict path of every consuming runnable, which is why it was NOT bundled into the move. Decide deliberately; do not let the two protocols blur by drift. Rationale for the current state: `docs/architecture.md` §8. - [ ] **RecordStream Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance - [ ] **Typed-bag PoC → torch-`Tensor`-subclass item base:** array-backed items are `np.ndarray` subclasses only; add a `TensorItem` base (torch `__torch_function__` attr-preservation) so torch payloads can be array items instead of riding wrapper `Signal`s. @low @ml diff --git a/docs/storage.md b/docs/storage.md index de6f648..8401912 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -71,6 +71,24 @@ item_data(loaded["mask"]) # the full mask array, byte-exact (not a truncated r loaded["image"].layout # item attrs round-trip too ``` +## Partial payload reads (`read_record_group`) + +`recordstream.storage.hdf5.read_record_group(group, slices=...)` is the public form of the +HDF5 row decoder `HDF5Source` iterates through. `slices` maps a record KEY to a slice applied +to that field's `data` dataset **at read time** — an h5py partial read, so only the requested +span of the payload ever leaves the file. A consumer windowing large stored rows (e.g. a +whole-capture signal archived once, re-windowed onto many grids at read time) decodes each +window without materializing the row; the decoded item is identical to a full read followed by +an in-memory slice, and attrs are never sliced: + +```python +import h5py +from recordstream.storage.hdf5 import read_record_group + +with h5py.File("ds.h5", "r") as handle: + window = read_record_group(handle["s000000"], slices={"signal": slice(1000, 2000)}) +``` + ## Queryable metadata (`recordstream.storage.query`) Filter stored records by metadata predicates *without loading arrays*: sources implementing the diff --git a/recordstream/storage/hdf5.py b/recordstream/storage/hdf5.py index f2230d3..b9bca9d 100644 --- a/recordstream/storage/hdf5.py +++ b/recordstream/storage/hdf5.py @@ -28,20 +28,32 @@ _ORDER_ATTR = "__field_order__" -def _read_record(group: h5py.Group) -> Record: - """Decode one ``sNNNNNN`` record group of the record key-group layout.""" +def read_record_group(group: h5py.Group, slices: Optional[Dict[str, slice]] = None) -> Record: + """Decode one ``sNNNNNN`` record group of the record key-group layout. + + ``slices`` maps a record KEY to a slice applied to that field's ``data`` dataset at + read time — an h5py partial read, so only the requested span of the payload ever + leaves the file. This is what lets a windowing source slice one window out of a + large stored row (e.g. a whole-capture IQ array) without materializing the row: + the decoded item is identical to reading the full payload and slicing in memory + (pinned by the storage tests). Keys absent from ``slices`` (and every attr) are + read in full; a sliced key whose group has no ``data`` dataset raises ``KeyError``. + """ order = json.loads(group.attrs[_ORDER_ATTR]) record: Record = {} payload: Any for name in order: fgrp = group[name] type_name = str(fgrp.attrs[_TYPE_ATTR]) + window = (slices or {}).get(name) if type_name == PLAIN_TYPE: # A plain value: array payload as the ``data`` dataset, scalar payload as the # ``value`` attr (JSON-marked when structured) — see HDF5Sink._write_record. if "data" in fgrp: - payload = fgrp["data"][()] + payload = fgrp["data"][window] if window is not None else fgrp["data"][()] else: + if window is not None: + raise KeyError(f"read_record_group: field {name!r} has no 'data' dataset to slice") payload = restore_attrs({PLAIN_VALUE: fgrp.attrs[PLAIN_VALUE]}, {})[PLAIN_VALUE] record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs={})) continue @@ -51,7 +63,12 @@ def _read_record(group: h5py.Group) -> Record: if isinstance(agrp, h5py.Group): for key, dset in agrp.items(): arrays[key] = dset[()] - payload = fgrp["data"][()] if "data" in fgrp else None + if window is not None: + if "data" not in fgrp: + raise KeyError(f"read_record_group: field {name!r} has no 'data' dataset to slice") + payload = fgrp["data"][window] + else: + payload = fgrp["data"][()] if "data" in fgrp else None attrs = restore_attrs(dict(plain), arrays) record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=attrs)) return record @@ -91,7 +108,7 @@ def __iter__(self) -> Iterator[Record]: if self._file is None: return for name in sorted(k for k in self._file.keys() if k.startswith("s")): - yield _read_record(self._file[name]) + yield read_record_group(self._file[name]) def __len__(self) -> int: self.open() diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py index 304676e..cf457a7 100644 --- a/tests/test_typed_storage.py +++ b/tests/test_typed_storage.py @@ -10,7 +10,7 @@ from recordstream import Boxes, Image, Label, register_item from recordstream.storage.base import TYPED_FORMAT, require_record_format, restore_attrs, split_attrs from recordstream.storage.directory import DirectorySink, DirectorySource -from recordstream.storage.hdf5 import HDF5Sink, HDF5Source +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source, read_record_group from recordstream.storage.query import MetadataFilterSource, record_metadata, scan_hdf5_metadata, scan_zarr_metadata from recordstream.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource @@ -158,6 +158,44 @@ def test_non_dict_write_raises(self, tmp_path: Path) -> None: sink.write(np.zeros(3)) +class TestReadRecordGroupSlices: + """``read_record_group(group, slices=...)`` — the partial-payload read a windowing source uses.""" + + def _first_group(self, path: Path) -> "h5py.Group": + handle = h5py.File(path, "r") + return handle["s000000"] + + def _write_one(self, tmp_path: Path) -> Path: + path = tmp_path / "t.h5" + with HDF5Sink(path=path, overwrite=True) as sink: + sink.write(_records()[0]) + return path + + def test_sliced_read_equals_full_read_then_slice(self, tmp_path: Path) -> None: + group = self._first_group(self._write_one(tmp_path)) + full = read_record_group(group) + part = read_record_group(group, slices={"sig": slice(2, 6)}) + # The sliced field: identical item type + attrs, payload = the full payload's slice. + assert type(part["sig"]) is type(full["sig"]) + assert part["sig"].samplerate == full["sig"].samplerate + assert np.array_equal(np.asarray(part["sig"].data), np.asarray(full["sig"].data)[2:6]) + # Attrs (incl. array-valued ones) are never sliced. + assert np.array_equal(np.asarray(part["sig"].mask), np.asarray(full["sig"].mask)) + # Every other field is byte-identical to the full read. + assert np.array_equal(np.asarray(part["image"]), np.asarray(full["image"])) + assert part["gain_db"] == full["gain_db"] + + def test_plain_array_field_slices_too(self, tmp_path: Path) -> None: + group = self._first_group(self._write_one(tmp_path)) + part = read_record_group(group, slices={"window": slice(1, 3)}) + assert np.array_equal(part["window"], np.hanning(4)[1:3]) + + def test_slicing_a_payload_free_field_raises(self, tmp_path: Path) -> None: + group = self._first_group(self._write_one(tmp_path)) + with pytest.raises(KeyError, match="no 'data' dataset"): + read_record_group(group, slices={"gain_db": slice(0, 1)}) + + class TestZarr: def test_group_round_trip(self, tmp_path: Path) -> None: path = str(tmp_path / "g.zarr") From 75f5e33f9b05edf7e6a2d4edf3425d3a675b1224 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 10 Aug 2026 14:43:48 +0200 Subject: [PATCH 085/102] refactor: the target ops adopt resolve_item where it genuinely applies ResizeDetection's target ride-along probe is the shared resolver's probe form (required=False, fallback=False) instead of a hand-rolled record.get + isinstance, and the EncodeTarget/DecodeTarget _find_label twins (18 byte-parallel lines each) collapse into one module-level _find_label_key with the pinned messages parameterized by op name. The six payload-kind finders (Threshold/ToTensor/ConvertToImage/ ConvertToMask and the two two-tier mask finders) deliberately do NOT fold: they search by what the payload IS (ndarray/PIL/2-3-D), not by item type, and routing them through resolve_entry would grow it the predicate knob the workspace bans; _find_label_key's docstring states the same boundary for the type-tuple + TypeError contract. --- recordstream/ops/target.py | 68 +++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/recordstream/ops/target.py b/recordstream/ops/target.py index 64dac2d..3e6c6b3 100644 --- a/recordstream/ops/target.py +++ b/recordstream/ops/target.py @@ -19,7 +19,7 @@ import numpy as np from confluid import configurable -from recordstream.items import Boxes, Label, Mask, MultiLabel, Record, item_data +from recordstream.items import Boxes, Label, Mask, MultiLabel, Record, item_data, resolve_item from recordstream.transform import Transform #: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo @@ -75,6 +75,28 @@ def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: ) +def _find_label_key(record: Record, field: Optional[str], op_name: str) -> str: + """Resolve the KEY of the label field to shape (``field`` or the first label item). + + Matches a :class:`~recordstream.Label` OR a :class:`~recordstream.MultiLabel` — both are + label items, and a multi-label target must be shaped through the same op. Shared by + :class:`EncodeTarget` / :class:`DecodeTarget`, which carried byte-parallel copies. + Deliberately NOT :func:`~recordstream.resolve_entry`: the gate is a type TUPLE and the + wrong-type miss is a ``TypeError`` (the callers' pinned API contract), both outside the + one-type/``ValueError`` shape the shared resolver pins. + """ + if field: + if field not in record: + raise ValueError(f"{op_name}: field {field!r} not in record (keys: {list(record)})") + item = record[field] + if not isinstance(item, (Label, MultiLabel)): + raise TypeError(f"{op_name}: field {field!r} is {type(item).__name__}, expected a Label or MultiLabel") + return field + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, (Label, MultiLabel))): + return key + raise ValueError(f"{op_name}: no Label/MultiLabel field in record (keys: {list(record)})") + + def coco_to_detection( objects: Any, bbox_key: str = "bbox", @@ -205,23 +227,8 @@ def __init__( self.output = str(output) def _find_label(self, record: Record) -> str: - """Resolve the KEY of the label field to encode (``self.field`` or the first label item). - - Matches a :class:`~recordstream.Label` OR a :class:`~recordstream.MultiLabel` — both are - label items, and a multi-label target must be encodeable through the same op. - """ - if self.field: - if self.field not in record: - raise ValueError(f"EncodeTarget: field {self.field!r} not in record (keys: {list(record)})") - item = record[self.field] - if not isinstance(item, (Label, MultiLabel)): - raise TypeError( - f"EncodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label or MultiLabel" - ) - return self.field - for key, _item in ((k, v) for k, v in record.items() if isinstance(v, (Label, MultiLabel))): - return key - raise ValueError(f"EncodeTarget: no Label/MultiLabel field in record (keys: {list(record)})") + """Resolve the KEY of the label field to encode (via the shared :func:`_find_label_key`).""" + return _find_label_key(record, self.field, "EncodeTarget") def __call__(self, record: Record) -> Record: if not self.mapping: @@ -275,23 +282,8 @@ def __init__( self.output = str(output) def _find_label(self, record: Record) -> str: - """Resolve the KEY of the label field to decode (``self.field`` or the first label item). - - Matches a :class:`~recordstream.Label` OR a :class:`~recordstream.MultiLabel` — both are - label items, and a multi-label target must be decodeable through the same op. - """ - if self.field: - if self.field not in record: - raise ValueError(f"DecodeTarget: field {self.field!r} not in record (keys: {list(record)})") - item = record[self.field] - if not isinstance(item, (Label, MultiLabel)): - raise TypeError( - f"DecodeTarget: field {self.field!r} is {type(item).__name__}, expected a Label or MultiLabel" - ) - return self.field - for key, _item in ((k, v) for k, v in record.items() if isinstance(v, (Label, MultiLabel))): - return key - raise ValueError(f"DecodeTarget: no Label/MultiLabel field in record (keys: {list(record)})") + """Resolve the KEY of the label field to decode (via the shared :func:`_find_label_key`).""" + return _find_label_key(record, self.field, "DecodeTarget") def __call__(self, record: Record) -> Record: if not self.mapping: @@ -533,8 +525,10 @@ def __call__(self, record: Record) -> Record: merged[self.input_key] = with_data(item, resized) if isinstance(item, NDArrayItem) else resized - target = record.get(self.target_key) - if isinstance(target, Boxes): + target = resolve_item( + record, self.target_key, Boxes, owner="ResizeDetection", param="target_key", fallback=False, required=False + ) + if target is not None: import dataclasses # An EMPTY target is re-framed too. Scaling no boxes is a no-op, but leaving the From aad40aa30e50248823db84ee5bb60b6226eb07fb Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 11 Aug 2026 16:37:56 +0200 Subject: [PATCH 086/102] fix: the deferred-source guidance no longer tells users to add parens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `!class:X` and `!class:X()` are the same target since confluid merged its eager and deferred markers, so "write it with parens" was advice that changes nothing — the user follows it and hits the identical error. The only ways to reach these messages now are asking for deferral explicitly (`_partial_: true`) or wiring a hand-built `PartialClass(...)`, so they say to drop the deferral instead. A source or op slot needs a live object and nothing downstream will flow it. --- docs/sources.md | 11 ++++--- recordstream/core/stream.py | 22 +++++++++---- recordstream/sources/base.py | 8 ++--- tests/test_view_sources_deferred.py | 50 +++++++++++++++++------------ 4 files changed, 55 insertions(+), 36 deletions(-) diff --git a/docs/sources.md b/docs/sources.md index cdc6423..9378df5 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -99,11 +99,12 @@ val_set: !class:recordstream.sources.split.DatasetSplit() **HuggingFace native slicing** (alternative, no RecordStream split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. -> **Nesting a source inline: write `!class:X()`, with parens.** A parens-less `!class:X` is a -> *deferred marker*, not an instance — a `source:` slot holding one fails at first use with an -> error naming the slot and this fix (the same guidance `Stream` gives). The examples above -> sidestep this with `!ref:` to a top-level instance, which is also what lets several wrappers -> share one loaded source. +> **Never mark a nested source `_partial_: true`.** A partial is a *deferred marker*, not an +> instance — a `source:` slot holding one fails at first use with an error naming the slot and +> the fix (the same guidance `Stream` gives), because nothing will flow it for you. A plain +> `_target_:` source is built at load time and is what the slot wants. The examples above use +> `${ref:…}` to a top-level source instead, which is also what lets several wrappers share one +> loaded source. ## Identifying a dataset diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py index 2e7f6e8..8051de6 100644 --- a/recordstream/core/stream.py +++ b/recordstream/core/stream.py @@ -53,13 +53,21 @@ def _fluid_source_guidance(source: Any, slot: str = "Stream.source") -> str: ``slot`` names the owning slot (``"Stream.source"``, ``"RangeSource.source"``, …) so the view sources raise the SAME guidance Stream does — a still-deferred ``source:`` is a - CONFIG error (the parens-less ``!class:X`` spelling), never something the engine flows. + CONFIG error, never something the engine flows. + + The trigger CHANGED with confluid's marker merge (2026-08-11): a parens-less + ``!class:X`` used to leave a deferred stub here, and this message used to say "add + parens". Both spellings build now, so the only way to reach this is asking for + deferral explicitly — ``_partial_: true`` (``!lazy:``) — or wiring a hand-built + ``PartialClass(...)``. Telling the user to add parens would now be advice that + changes nothing. """ return ( f"{slot} is still a deferred Confluid marker: {_describe_deferred_source(source)}. " - "Confluid has not materialized it yet. Fixes: (a) in YAML, write the source as " - "`!class:X()` (with parens) instead of `!class:X` so it becomes an Instance and is " - "materialized at load time; (b) or call `flow(source)` on the source before wiring it." + "Confluid was told NOT to build it. Fixes: (a) in YAML, drop `_partial_: true` from " + "the source so it is built at load time — a source slot needs a live object, and " + "nothing here will flow it for you; (b) or call `flow(source)` yourself before " + "wiring it in." ) @@ -67,9 +75,9 @@ def _fluid_op_guidance(op: Any, index: int) -> str: """Build an actionable message when a Stream op marker cannot be materialized.""" return ( f"Stream.ops[{index}] is a deferred Confluid marker that could not be materialized: " - f"{_describe_deferred_source(op)}. Fixes: (a) in YAML, write the op as `!class:X()` " - "(with parens) so it becomes an Instance and is materialized at load time; (b) or " - "call `flow(op)` on the op before handing it to Stream." + f"{_describe_deferred_source(op)}. Fixes: (a) in YAML, drop `_partial_: true` from " + "the op so it is built at load time; (b) or call `flow(op)` on the op before handing " + "it to Stream." ) diff --git a/recordstream/sources/base.py b/recordstream/sources/base.py index ae60c56..197f02a 100644 --- a/recordstream/sources/base.py +++ b/recordstream/sources/base.py @@ -10,10 +10,10 @@ def _guard_live_source(source: Any, slot: str) -> None: """Raise Stream's actionable deferred-marker error when ``source`` is still a Fluid. - A still-deferred ``!class:X`` (parens-less) marker in a ``source:`` slot is a CONFIG - error, and the view sources answer it exactly as ``Stream._guard_live_source`` does — - naming the slot and the deferred target, and pointing at the ``!class:X()`` fix — - instead of the cryptic ``got Class`` a bare ``hasattr`` check produces. Deliberately + A still-deferred marker in a ``source:`` slot is a CONFIG error — the slot needs a live + object — and the view sources answer it exactly as ``Stream._guard_live_source`` does, + naming the slot, the deferred target and the fix (drop ``_partial_: true``), instead of + the cryptic ``got Partial`` a bare ``hasattr`` check produces. Deliberately message-only: the slot is never flowed here (the raise-with-guidance convention for ``source:`` slots, distinct from the free functions ``project`` / ``dataset_uri``, which do materialize a marker first). diff --git a/tests/test_view_sources_deferred.py b/tests/test_view_sources_deferred.py index 8903853..9fbacad 100644 --- a/tests/test_view_sources_deferred.py +++ b/tests/test_view_sources_deferred.py @@ -1,11 +1,18 @@ -"""A still-deferred ``!class:`` marker in a view source's ``source:`` slot explains itself. - -The parens-less ``!class:X`` YAML spelling leaves a Confluid ``Fluid`` marker in the slot — -a CONFIG error (the fix is ``!class:X()``), and the view sources answer it with the same -actionable guidance ``Stream`` gives (naming the slot, the deferred target, and the parens -fix) instead of the cryptic ``got Class`` their bare ``hasattr`` checks used to produce. -Deliberately message-only: the slot is never flowed (the raise-with-guidance convention for -``source:`` slots — distinct from the free functions ``project`` / ``dataset_uri``). +"""A still-deferred marker in a view source's ``source:`` slot explains itself. + +A ``_partial_: true`` source leaves a Confluid marker in the slot — a CONFIG error, since a +source slot needs a live object and nothing here will flow it — and the view sources answer +it with the same actionable guidance ``Stream`` gives (naming the slot, the deferred target, +and the fix) instead of the cryptic ``got Partial`` their bare ``hasattr`` checks used to +produce. Deliberately message-only: the slot is never flowed (the raise-with-guidance +convention for ``source:`` slots — distinct from the free functions ``project`` / +``dataset_uri``). + +**The TRIGGER changed 2026-08-11.** It used to be the parens-less ``!class:X`` spelling, +which left a deferred stub; confluid merged its eager and deferred markers, so both +spellings build and that footgun is gone. Deferral is now asked for explicitly, which is +the only way left to reach this guard — and the message changed with it (it used to say +"add parens", advice that would now change nothing). """ import pytest @@ -23,14 +30,15 @@ def _expect_guidance(excinfo: "pytest.ExceptionInfo[TypeError]", slot: str) -> N assert slot in message assert "deferred Confluid marker" in message assert "ConcatSource" in message # the deferred target, not just "Class" - assert "!class:X()" in message # the actionable fix + assert "_partial_: true" in message # the actionable fix -def test_a_parens_less_marker_under_range_source_raises_the_stream_guidance() -> None: +def test_a_partial_marker_under_range_source_raises_the_stream_guidance() -> None: cfg = load( f""" -range_src: !class:recordstream.sources.range.RangeSource() - source: !class:{_LEAF} +range_src: + _target_: recordstream.sources.range.RangeSource + source: {{_target_: {_LEAF}, _partial_: true}} stop: 3 """, flow=True, @@ -40,13 +48,14 @@ def test_a_parens_less_marker_under_range_source_raises_the_stream_guidance() -> _expect_guidance(excinfo, "RangeSource.source") -def test_a_parens_less_marker_under_concat_source_names_the_offending_index() -> None: +def test_a_partial_marker_under_concat_source_names_the_offending_index() -> None: cfg = load( f""" -concat_src: !class:recordstream.sources.concat.ConcatSource() +concat_src: + _target_: recordstream.sources.concat.ConcatSource sources: - - !class:{_LEAF}() - - !class:{_LEAF} + - {{_target_: {_LEAF}}} + - {{_target_: {_LEAF}, _partial_: true}} """, flow=True, ) @@ -55,11 +64,12 @@ def test_a_parens_less_marker_under_concat_source_names_the_offending_index() -> _expect_guidance(excinfo, "ConcatSource.sources[1]") -def test_a_parens_less_marker_under_dataset_split_raises_the_stream_guidance() -> None: +def test_a_partial_marker_under_dataset_split_raises_the_stream_guidance() -> None: cfg = load( f""" -split_src: !class:recordstream.sources.split.DatasetSplit() - source: !class:{_LEAF} +split_src: + _target_: recordstream.sources.split.DatasetSplit + source: {{_target_: {_LEAF}, _partial_: true}} """, flow=True, ) @@ -94,6 +104,6 @@ def test_the_parens_spelling_materializes_and_the_view_sources_work() -> None: def test_streams_own_guidance_still_names_its_slot() -> None: """The shared message helper's default slot stays ``Stream.source``.""" - cfg = load(f"deferred: !class:{_LEAF}", flow=True) + cfg = load(f"deferred: {{_target_: {_LEAF}, _partial_: true}}", flow=True) with pytest.raises(TypeError, match="Stream.source is still a deferred Confluid marker"): len(Stream(source=cfg["deferred"])) From ae609d47d8c22f3ba5245814a28c8dad17fd420d Mon Sep 17 00:00:00 2001 From: gearlux Date: Wed, 12 Aug 2026 10:21:32 +0200 Subject: [PATCH 087/102] refactor!: rename confluid's deprecated aliases to their canonical names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 5a. The names became aliases when confluid merged its two construction modes (Class/Instance -> Target) and renamed Lazy -> Partial; this moves every consumer onto the canonical spelling so the aliases can be deleted. lazy_param_names -> partial_param_names LazyClass -> PartialClass Lazy -> Partial Instance -> Target Class -> Target Mechanical, by codemod. `Class` and `Instance` are ordinary English words, so they were renamed ONLY in a file that demonstrably takes them from confluid — an import or a `confluid.`-qualified reference. The other three are confluid-specific and safe everywhere. The lowercase `lazy=True` registration mark is untouched: it is still spelled that way. --- recordstream/core/mapstyle.py | 2 +- recordstream/core/stream.py | 4 ++-- recordstream/core/wrappers.py | 4 ++-- recordstream/flow/graph.py | 2 +- recordstream/labels.py | 4 ++-- recordstream/loaders.py | 18 +++++++++--------- recordstream/ops/configure.py | 2 +- recordstream/ops/enable.py | 6 +++--- recordstream/ops/formula.py | 2 +- recordstream/ops/image.py | 2 +- recordstream/ops/parallel.py | 2 +- recordstream/ops/sink.py | 2 +- recordstream/ops/target.py | 4 ++-- recordstream/predictions.py | 2 +- recordstream/projection.py | 4 ++-- recordstream/sources/concat.py | 2 +- recordstream/sources/huggingface.py | 10 +++++----- recordstream/sources/range.py | 4 ++-- recordstream/sources/split.py | 4 ++-- recordstream/storage/directory.py | 4 ++-- recordstream/storage/hdf5.py | 4 ++-- recordstream/storage/query.py | 2 +- recordstream/storage/zarr.py | 8 ++++---- recordstream/transform.py | 2 +- recordstream/workflow.py | 2 +- tests/test_cli_run.py | 4 ++-- tests/test_docs_links.py | 2 +- tests/test_keras_sequence.py | 2 +- tests/test_labels.py | 6 +++--- tests/test_lazy_construction.py | 6 +++--- tests/test_loaders.py | 2 +- tests/test_pipeline.py | 4 ++-- tests/test_projection.py | 4 ++-- tests/test_workflow.py | 2 +- 34 files changed, 67 insertions(+), 67 deletions(-) diff --git a/recordstream/core/mapstyle.py b/recordstream/core/mapstyle.py index 112035c..419cf6b 100644 --- a/recordstream/core/mapstyle.py +++ b/recordstream/core/mapstyle.py @@ -33,7 +33,7 @@ def __getitem__(self, index: int) -> Any: ... #: named ONCE here rather than restated by every consumer: anything MAP-STYLE (``__len__`` + #: ``__getitem__`` — which a ``Stream`` is), or any iterable of records (a recordstream #: source, a plain list of record dicts). Consumers annotate their slots -#: ``Optional[Lazy[RecordSource]]`` — ``Lazy`` because they flow the slot at run time. +#: ``Optional[Partial[RecordSource]]`` — ``Partial`` because they flow the slot at run time. #: #: Expressed with the structural :class:`MapStyle` rather than ``torch.utils.data.Dataset`` so the #: engine can say "a dataset" without importing a framework; torch's ``DataLoader`` is itself diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py index 8051de6..abf74cb 100644 --- a/recordstream/core/stream.py +++ b/recordstream/core/stream.py @@ -149,7 +149,7 @@ class JointStream: """ def __init__(self, streams: Optional[List["Stream"]] = None) -> None: - # Lazy / zero-arg: store config only; no sub-streams ⇒ an empty stream. + # Partial / zero-arg: store config only; no sub-streams ⇒ an empty stream. self.streams = streams if streams is not None else [] def __iter__(self) -> Iterator[Record]: @@ -428,7 +428,7 @@ def project(self, keys: Collection[str]) -> Iterator[Record]: Implements :class:`recordstream.projection.SupportsProjection`. Stream must run its op chain to produce each record (an op may consume the input), so this is the generic - "iterate, then keep only the requested keys" form. Lazy: a generator. + "iterate, then keep only the requested keys" form. Partial: a generator. """ want = set(keys) for record in self: diff --git a/recordstream/core/wrappers.py b/recordstream/core/wrappers.py index 83e673c..bd2879a 100644 --- a/recordstream/core/wrappers.py +++ b/recordstream/core/wrappers.py @@ -28,7 +28,7 @@ class FilterOp: """ def __init__(self, p: Optional[Callable[[Record], bool]] = None): - # Lazy / zero-arg: store config only; a missing predicate is validated lazily in __call__. + # Partial / zero-arg: store config only; a missing predicate is validated lazily in __call__. self.p = p def __call__(self, record: Record) -> Optional[Record]: @@ -58,7 +58,7 @@ class WrappedOp: def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: Optional[Dict[str, Any]] = None): from recordstream.discovery import get_callable_path - # Lazy / zero-arg: store config only (the empty-path default resolves lazily via the `func` + # Partial / zero-arg: store config only (the empty-path default resolves lazily via the `func` # property). EXPLICIT: always store the string path for serialization. self.f = get_callable_path(f) if callable(f) else f self.key = key diff --git a/recordstream/flow/graph.py b/recordstream/flow/graph.py index 16be4d2..f1ef3cb 100644 --- a/recordstream/flow/graph.py +++ b/recordstream/flow/graph.py @@ -46,7 +46,7 @@ def __init__( outputs: str = "", chunk_size: int = 0, ) -> None: - # Lazy / zero-arg: store config only; parsing/validation happen in the cached property. + # Partial / zero-arg: store config only; parsing/validation happen in the cached property. self.source = source self.flow = flow self.outputs = str(outputs) diff --git a/recordstream/labels.py b/recordstream/labels.py index 425acb2..86597c8 100644 --- a/recordstream/labels.py +++ b/recordstream/labels.py @@ -15,7 +15,7 @@ "mapping pinned in config, not fitted" discipline of the ops; it is how the pin gets created. Zero-arg constructible (``LabelMap()`` succeeds with an empty mapping) and side-effect-free in -``__init__`` per the workspace "Lazy Initialization & Zero-Arg Construction" convention; the +``__init__`` per the workspace "Partial Initialization & Zero-Arg Construction" convention; the non-empty requirement is validated lazily in the properties, not in the constructor. A label is ALWAYS mappable to ids: :meth:`LabelMap.to_ids` accepts a ``Label`` / ``MultiLabel`` @@ -71,7 +71,7 @@ class LabelMap: """ def __init__(self, mapping: Optional[Dict[str, int]] = None) -> None: - # Lazy / zero-arg: store config only. An empty map is a valid object; the non-empty + # Partial / zero-arg: store config only. An empty map is a valid object; the non-empty # requirement is enforced lazily in the properties, never here. self.mapping: Dict[str, int] = {str(k): int(v) for k, v in mapping.items()} if mapping else {} diff --git a/recordstream/loaders.py b/recordstream/loaders.py index 397bb0e..681c8c7 100644 --- a/recordstream/loaders.py +++ b/recordstream/loaders.py @@ -6,7 +6,7 @@ workspace baked the same three deferred loader slots into its constructor — train shuffled, val/test not, one shared kwarg set. :func:`loader_slots` is that construction written once. -The slots are :class:`confluid.LazyClass` markers, not live loaders, on purpose: the lazy-init +The slots are :class:`confluid.PartialClass` markers, not live loaders, on purpose: the lazy-init mandate forbids functional work in a constructor, and the dataset does not exist yet — the run method flows each slot with ``dataset=`` at run time (``flow(self.train_loader, dataset=ds)``). @@ -19,7 +19,7 @@ from typing import Any, Callable, List, NamedTuple -from confluid import Lazy, LazyClass +from confluid import Partial, PartialClass from recordstream.collate import collate_records from recordstream.items import Record @@ -37,9 +37,9 @@ class LoaderSlots(NamedTuple): """The three deferred loader markers, addressed by split (``slots.train`` / ``.val`` / ``.test``).""" - train: Lazy[DataLoader[Any]] - val: Lazy[DataLoader[Any]] - test: Lazy[DataLoader[Any]] + train: Partial[DataLoader[Any]] + val: Partial[DataLoader[Any]] + test: Partial[DataLoader[Any]] def loader_slots( @@ -76,7 +76,7 @@ def loader_slots( not a shared kwarg. Returns: - A :class:`LoaderSlots` named tuple — three ``LazyClass(DataLoader, ...)`` markers + A :class:`LoaderSlots` named tuple — three ``PartialClass(DataLoader, ...)`` markers (``slots.train`` with ``shuffle=True``, ``slots.val`` / ``slots.test`` with ``shuffle=False``). Assign them to the runnable's ``train_loader`` / ``val_loader`` / ``test_loader`` slots and flow each with ``dataset=`` at run time. @@ -95,7 +95,7 @@ def loader_slots( **loader_kw, ) return LoaderSlots( - train=LazyClass(DataLoader, shuffle=True, **shared), - val=LazyClass(DataLoader, shuffle=False, **shared), - test=LazyClass(DataLoader, shuffle=False, **shared), + train=PartialClass(DataLoader, shuffle=True, **shared), + val=PartialClass(DataLoader, shuffle=False, **shared), + test=PartialClass(DataLoader, shuffle=False, **shared), ) diff --git a/recordstream/ops/configure.py b/recordstream/ops/configure.py index 2a93ace..0d3a489 100644 --- a/recordstream/ops/configure.py +++ b/recordstream/ops/configure.py @@ -57,7 +57,7 @@ def __init__( param: str = "", source: str = "", ) -> None: - # Lazy / zero-arg: store config only; target/param/source are validated at first call. + # Partial / zero-arg: store config only; target/param/source are validated at first call. self.ops = list(ops) if ops else [] self.target = target self.param = str(param) diff --git a/recordstream/ops/enable.py b/recordstream/ops/enable.py index 45dfe12..29d904b 100644 --- a/recordstream/ops/enable.py +++ b/recordstream/ops/enable.py @@ -73,7 +73,7 @@ class Enable: Constraints: * ``ops`` is required and must be a non-empty list — validated **lazily** on first call (zero-arg construction stays valid per the recordstream - "Lazy Initialization & Zero-Arg Construction" convention). + "Partial Initialization & Zero-Arg Construction" convention). * ``enabled`` must be a ``bool``; a non-bool raises ``TypeError`` at set time. * Any OTHER boolean attribute set on the wrapper raises ``ValueError`` on first call. That is the migration guard for the retired dynamic-toggle @@ -90,7 +90,7 @@ class Enable: """ def __init__(self, ops: Optional[List] = None, enabled: bool = True, name: str = "") -> None: - # Lazy / zero-arg: store config only; `ops` non-emptiness and stray-toggle + # Partial / zero-arg: store config only; `ops` non-emptiness and stray-toggle # rejection are enforced lazily on first call. self.ops: List = list(ops) if ops else [] self.name = name @@ -111,7 +111,7 @@ def enabled(self, value: Any) -> None: self._enabled = value def _check(self) -> None: - """Lazy one-time validation, run on the first record.""" + """Partial one-time validation, run on the first record.""" if not self.ops: raise ValueError("Enable requires a non-empty 'ops' list.") stray = [key for key, value in vars(self).items() if isinstance(value, bool) and not key.startswith("_")] diff --git a/recordstream/ops/formula.py b/recordstream/ops/formula.py index ab9f31c..24051a7 100644 --- a/recordstream/ops/formula.py +++ b/recordstream/ops/formula.py @@ -40,7 +40,7 @@ class FormulaOp: """ def __init__(self, formula: str = "a", field: str = "", var: str = "a") -> None: - # Lazy / zero-arg: store config only; formula and field are validated at first call. + # Partial / zero-arg: store config only; formula and field are validated at first call. self.formula = str(formula) self.field = str(field) self.var = str(var) diff --git a/recordstream/ops/image.py b/recordstream/ops/image.py index 4c9c3e7..7e34412 100644 --- a/recordstream/ops/image.py +++ b/recordstream/ops/image.py @@ -776,7 +776,7 @@ class ConvertToMask(Transform): produces = (MaskItem,) def __init__(self, field: str = "", output: str = "mask") -> None: - # Lazy / zero-arg: store config only. A missing/unusable field is reported at call time. + # Partial / zero-arg: store config only. A missing/unusable field is reported at call time. super().__init__() self.field = field self.output = output diff --git a/recordstream/ops/parallel.py b/recordstream/ops/parallel.py index f934e63..cda3d6a 100644 --- a/recordstream/ops/parallel.py +++ b/recordstream/ops/parallel.py @@ -39,7 +39,7 @@ class Parallel: """ def __init__(self, ops: Optional[List[Any]] = None, workers: int = 4) -> None: - # Lazy / zero-arg: store config only; ``workers >= 1`` is validated lazily in ``stream``. + # Partial / zero-arg: store config only; ``workers >= 1`` is validated lazily in ``stream``. self.ops = list(ops) if ops else [] self.workers = int(workers) diff --git a/recordstream/ops/sink.py b/recordstream/ops/sink.py index 20790fb..aad62af 100644 --- a/recordstream/ops/sink.py +++ b/recordstream/ops/sink.py @@ -43,7 +43,7 @@ class RecordSinkOp: """ def __init__(self, sink: Any = None) -> None: - # Lazy / zero-arg: store config only; a non-None sink is required lazily in __call__. + # Partial / zero-arg: store config only; a non-None sink is required lazily in __call__. self.sink = sink self._opened = False diff --git a/recordstream/ops/target.py b/recordstream/ops/target.py index 3e6c6b3..a2fb1f8 100644 --- a/recordstream/ops/target.py +++ b/recordstream/ops/target.py @@ -219,7 +219,7 @@ def __init__( output: str = "", ) -> None: super().__init__() - # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + # Partial / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. self.mapping = dict(mapping) if mapping else {} self.ignore_unknown = bool(ignore_unknown) self.default = default @@ -274,7 +274,7 @@ def __init__( output: str = "", ) -> None: super().__init__() - # Lazy / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + # Partial / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. self.mapping = dict(mapping) if mapping else {} self.ignore_unknown = bool(ignore_unknown) self.default = default diff --git a/recordstream/predictions.py b/recordstream/predictions.py index 8014730..29e395e 100644 --- a/recordstream/predictions.py +++ b/recordstream/predictions.py @@ -32,7 +32,7 @@ class PredictionsSink(Protocol): Structural, not a base class: a sink is anything that can take one record's prediction plus that record's metadata, and be closed at the end. Naming it here — beside the sink this - package ships — means consuming runnables annotate ``Optional[Lazy[PredictionsSink]]`` + package ships — means consuming runnables annotate ``Optional[Partial[PredictionsSink]]`` instead of ``Any``, which declared nothing and let a use site call ``.write`` on a slot that might still be a deferred marker. diff --git a/recordstream/projection.py b/recordstream/projection.py index c9fb479..5a4f4ed 100644 --- a/recordstream/projection.py +++ b/recordstream/projection.py @@ -13,7 +13,7 @@ ------------ * :class:`SupportsProjection` is a ``Protocol`` (never a base class), so a source opts in by *defining* ``project``, not by inheriting. -* Every public function is a lazy generator (**Lazy Evaluation** mandate) — +* Every public function is a lazy generator (**Partial Evaluation** mandate) — nothing materializes the whole source. * :func:`num_classes` (integer class-id semantics) is a free function, *not* a method on the generic :class:`~recordstream.core.stream.Stream` engine — counting classes is @@ -43,7 +43,7 @@ def project(source: Any, keys: Collection[str]) -> Iterator[Record]: Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the efficient path that skips building unrequested values); otherwise falls back to a full - iteration that keeps only the requested keys. Lazy: a generator. + iteration that keeps only the requested keys. Partial: a generator. A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized first, so a caller never has to remember which entry point flows and which does not — diff --git a/recordstream/sources/concat.py b/recordstream/sources/concat.py index 238498b..6f51dfc 100644 --- a/recordstream/sources/concat.py +++ b/recordstream/sources/concat.py @@ -27,7 +27,7 @@ class ConcatSource: """ def __init__(self, sources: Optional[List[Any]] = None) -> None: - # Lazy / zero-arg: store config only; sub-source validation + the cumulative-offset precompute + # Partial / zero-arg: store config only; sub-source validation + the cumulative-offset precompute # are deferred to the ``offsets`` property so sources can be configured post-construction. self.sources = list(sources) if sources else [] self._offsets: Optional[List[int]] = None diff --git a/recordstream/sources/huggingface.py b/recordstream/sources/huggingface.py index 8a64d64..111c65f 100644 --- a/recordstream/sources/huggingface.py +++ b/recordstream/sources/huggingface.py @@ -70,8 +70,8 @@ class HuggingFaceSource: * each ``metadata_features`` column -> its own :class:`~recordstream.Label` keyed by the column name, plus the source-provenance ``hf_path`` / ``hf_split`` Labels. - Lazy & zero-arg per the workspace class-design convention (see confluid AGENTS.md - "Lazy Initialization & Zero-Arg Construction"): the constructor only stores values and + Partial & zero-arg per the workspace class-design convention (see confluid AGENTS.md + "Partial Initialization & Zero-Arg Construction"): the constructor only stores values and does NO functional work — ``HuggingFaceSource()`` is valid, and the dataset is downloaded only on first access to :attr:`dataset` (cached thereafter; reset ``_dataset`` to reload). ``path`` is therefore optional at construction and validated lazily when the data is needed. @@ -110,7 +110,7 @@ def __init__( revision: Optional[str] = None, load_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - # Lazy constructor: store config only — never load here. Real work (the network/disk + # Partial constructor: store config only — never load here. Real work (the network/disk # download) is deferred to the ``dataset`` property so the object is cheap to build and # configurable post-construction. self.path = path @@ -133,7 +133,7 @@ def __init__( # `default-`, missed the cache, and went to the Hub. Measured: the same # command passes with no name override and fails with one. self._load_kwargs = dict(load_kwargs or {}) - # Lazy cache for the materialized dataset (see the ``dataset`` property). + # Partial cache for the materialized dataset (see the ``dataset`` property). self._dataset: Any = None @property @@ -226,7 +226,7 @@ def dataset_url(self) -> Optional[str]: def resolved_metadata_features(self) -> List[str]: """``metadata_features`` resolved against the live dataset's columns (expands the ``"*"`` sentinel). - Lazy because the ``"*"`` expansion needs the loaded dataset's ``column_names``; ``None`` / ``[]`` + Partial because the ``"*"`` expansion needs the loaded dataset's ``column_names``; ``None`` / ``[]`` stays "no extra metadata" (backward-compatible). """ return _resolve_metadata_features( diff --git a/recordstream/sources/range.py b/recordstream/sources/range.py index 6580332..05e9783 100644 --- a/recordstream/sources/range.py +++ b/recordstream/sources/range.py @@ -17,7 +17,7 @@ class RangeSource: The plain-slice counterpart to :class:`~recordstream.sources.DatasetSplit` (which shuffles + partitions) — extracted from DatasetSplit's old "range mode". Negative ``start`` / - ``stop`` count from the end; both are clamped to ``[0, len(source)]``. Lazy: only index + ``stop`` count from the end; both are clamped to ``[0, len(source)]``. Partial: only index arithmetic happens up front; records are produced on demand. The wrapped source must implement ``__len__`` and ``__getitem__``. @@ -29,7 +29,7 @@ class RangeSource: """ def __init__(self, source: Any = None, start: Optional[int] = None, stop: Optional[int] = None) -> None: - # Lazy / zero-arg: store config only; the index arithmetic (and source validation) is deferred + # Partial / zero-arg: store config only; the index arithmetic (and source validation) is deferred # to the ``indices`` property so the source can be configured post-construction. self.source = source self.start = start diff --git a/recordstream/sources/split.py b/recordstream/sources/split.py index e77e60c..4b9f90e 100644 --- a/recordstream/sources/split.py +++ b/recordstream/sources/split.py @@ -58,7 +58,7 @@ class DatasetSplit: Omit ``test_fraction`` for a plain two-way train/val split; omit both fractions for a degenerate split where ``train`` is the whole source and ``val`` / ``test`` are empty. - The wrapped source must implement ``__len__`` and ``__getitem__``. Lazy: only index + The wrapped source must implement ``__len__`` and ``__getitem__``. Partial: only index arithmetic happens up front; records are produced on demand. Args: @@ -77,7 +77,7 @@ def __init__( test_fraction: Optional[float] = None, seed: Optional[int] = None, ) -> None: - # Lazy / zero-arg: store config only. All validation is deferred to first materialization + # Partial / zero-arg: store config only. All validation is deferred to first materialization # (``_validate``, invoked from ``_view``) so the source can be configured post-construction. self.source = source self.split = split diff --git a/recordstream/storage/directory.py b/recordstream/storage/directory.py index 6cfdd43..bb2970a 100644 --- a/recordstream/storage/directory.py +++ b/recordstream/storage/directory.py @@ -33,7 +33,7 @@ class DirectorySink(Storage, DataSink): """ def __init__(self, path: Union[str, Path] = "", overwrite: bool = False, use_npz: bool = True) -> None: - # Lazy / zero-arg: store config only; the directory is created lazily in open(). + # Partial / zero-arg: store config only; the directory is created lazily in open(). self.path = Path(path) self.overwrite = overwrite self.use_npz = use_npz @@ -121,7 +121,7 @@ class DirectorySource(Storage, DataSource): """ def __init__(self, path: Union[str, Path] = "") -> None: - # Lazy / zero-arg: store config only; the directory is scanned lazily on iteration. + # Partial / zero-arg: store config only; the directory is scanned lazily on iteration. self.path = Path(path) def _record_dirs(self) -> list: diff --git a/recordstream/storage/hdf5.py b/recordstream/storage/hdf5.py index b9bca9d..8d11cba 100644 --- a/recordstream/storage/hdf5.py +++ b/recordstream/storage/hdf5.py @@ -83,7 +83,7 @@ class HDF5Source(Storage, DataSource): """ def __init__(self, path: Union[str, Path] = "") -> None: - # Lazy / zero-arg: store config only; the file is opened lazily in open() (an unset path + # Partial / zero-arg: store config only; the file is opened lazily in open() (an unset path # surfaces there, not in __init__). self.path = Path(path) self._file: Optional[h5py.File] = None @@ -138,7 +138,7 @@ def __init__( compression: Optional[str] = "gzip", overwrite: bool = False, ) -> None: - # Lazy / zero-arg: store config only; the file is opened lazily in open(). + # Partial / zero-arg: store config only; the file is opened lazily in open(). self.path = Path(path) self.compression = compression self.overwrite = overwrite diff --git a/recordstream/storage/query.py b/recordstream/storage/query.py index ebe5ceb..26bc0a8 100644 --- a/recordstream/storage/query.py +++ b/recordstream/storage/query.py @@ -189,7 +189,7 @@ def __init__( where: str = "", predicate: Optional[Callable[[Dict[str, Any]], bool]] = None, ) -> None: - # Lazy / zero-arg: store config only; matching indices compute lazily on first access. + # Partial / zero-arg: store config only; matching indices compute lazily on first access. self.source = source self.where = str(where) self.predicate = predicate diff --git a/recordstream/storage/zarr.py b/recordstream/storage/zarr.py index 7604d50..1d967d1 100644 --- a/recordstream/storage/zarr.py +++ b/recordstream/storage/zarr.py @@ -64,7 +64,7 @@ class ZarrGroupSink(Storage, DataSink): """ def __init__(self, path: Union[str, Path] = "", overwrite: bool = False) -> None: - # Lazy / zero-arg: store config only; the group is opened lazily in open(). + # Partial / zero-arg: store config only; the group is opened lazily in open(). self.path = str(path) self.overwrite = overwrite self._root: Optional[zarr.Group] = None @@ -132,7 +132,7 @@ class ZarrGroupSource(Storage, DataSource): """ def __init__(self, path: Union[str, Path] = "") -> None: - # Lazy / zero-arg: store config only; the group is opened lazily in open(). + # Partial / zero-arg: store config only; the group is opened lazily in open(). self.path = str(path) self._root: Optional[zarr.Group] = None @@ -181,7 +181,7 @@ def __init__( chunks: Optional[List[int]] = None, overwrite: bool = False, ) -> None: - # Lazy / zero-arg: store config only; the array is created lazily in open() (an unset + # Partial / zero-arg: store config only; the array is created lazily in open() (an unset # path / shape surfaces there). self.path = str(path) self.shape = tuple(shape) if shape else () @@ -254,7 +254,7 @@ class ZarrBatchSource(Storage, DataSource): """ def __init__(self, path: Union[str, Path] = "") -> None: - # Lazy / zero-arg: store config only; the array is opened lazily in open(). + # Partial / zero-arg: store config only; the array is opened lazily in open(). self.path = str(path) self._data_arr: Optional[zarr.Array] = None diff --git a/recordstream/transform.py b/recordstream/transform.py index 69fa097..bf7f6d3 100644 --- a/recordstream/transform.py +++ b/recordstream/transform.py @@ -112,7 +112,7 @@ class Pipeline: """ def __init__(self, transforms: Optional[Sequence[Any]] = None) -> None: - # Lazy / zero-arg: store config only; marker flow happens on first call. + # Partial / zero-arg: store config only; marker flow happens on first call. self.transforms: List[Any] = list(transforms) if transforms else [] def __call__(self, record: Record) -> Optional[Record]: diff --git a/recordstream/workflow.py b/recordstream/workflow.py index 9e5290f..9c944df 100644 --- a/recordstream/workflow.py +++ b/recordstream/workflow.py @@ -113,7 +113,7 @@ class Sequence(TorchRunner, ProgressReporting): """ def __init__(self, steps: Optional[List[Any]] = None) -> None: - # Lazy / zero-arg: store config only; branches are flowed in run(). + # Partial / zero-arg: store config only; branches are flowed in run(). self.steps: List[Any] = list(steps) if steps else [] def run(self) -> None: diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index f03f06a..70b8697 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -2,7 +2,7 @@ from typing import List -from confluid import Class +from confluid import Target from recordstream.cli import run @@ -32,7 +32,7 @@ def run(self) -> None: log.append(self.tag) # A deferred Confluid marker is flowed before run() is called. - run(Class(R, tag="x")) + run(Target(R, tag="x")) assert log == ["x"] diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index b063093..4757b23 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -59,7 +59,7 @@ def _markdown_files() -> List[Path]: def _strip_code(text: str) -> str: """Blank out fenced code blocks before scanning for links. - Python subscript-then-call — `LazyClass[Metric](SomeClass)` — is + Python subscript-then-call — `PartialClass[Metric](SomeClass)` — is indistinguishable from a markdown link to a regex, so a code sample containing one gets reported as a link to a file named `SomeClass`. Found exactly that way: the first run of this check against another project's docs produced a false diff --git a/tests/test_keras_sequence.py b/tests/test_keras_sequence.py index a8eee06..46e328a 100644 --- a/tests/test_keras_sequence.py +++ b/tests/test_keras_sequence.py @@ -149,7 +149,7 @@ def test_on_epoch_end_is_a_no_op_when_not_shuffling() -> None: # --------------------------------------------------------------------------- # -# Lazy construction — the recordstream constructor rule +# Partial construction — the recordstream constructor rule # --------------------------------------------------------------------------- # diff --git a/tests/test_labels.py b/tests/test_labels.py index 84b7e4b..0f15fa0 100644 --- a/tests/test_labels.py +++ b/tests/test_labels.py @@ -293,7 +293,7 @@ def test_encode_refuses_an_already_encoded_id() -> None: def test_encode_flows_a_deferred_source() -> None: """A config-wired `!class:` source works without the caller flowing it first.""" - from confluid import Class as ConfluidClass + from confluid import Target as ConfluidClass from recordstream import Stream, iter_key @@ -454,11 +454,11 @@ def test_a_stream_rejects_non_string_names_at_construction() -> None: def _deferred(records: list) -> Any: """A `!class:` marker as a config hands one over — unbuilt.""" - from confluid import Class + from confluid import Target from recordstream import Stream - return Class(Stream, source=records) + return Target(Stream, source=records) def test_project_materializes_a_deferred_source() -> None: diff --git a/tests/test_lazy_construction.py b/tests/test_lazy_construction.py index a6c3925..2f46f11 100644 --- a/tests/test_lazy_construction.py +++ b/tests/test_lazy_construction.py @@ -1,11 +1,11 @@ -"""Pins the "Lazy Initialization & Zero-Arg Construction" convention for ALL recordstream configurables. +"""Pins the "Partial Initialization & Zero-Arg Construction" convention for ALL recordstream configurables. Every ``@configurable`` class in recordstream MUST be constructible with no arguments and do no functional work in ``__init__`` (no I/O, no network, no eager materialization). This walks the whole package, discovers every ``@configurable`` class, and asserts ``Cls()`` succeeds — so a newly-added class that violates the convention (a required ctor arg, or a constructor that opens a -file / loads a dataset) fails here. See confluid ``AGENTS.md`` → "Lazy Initialization & Zero-Arg -Construction" and recordstream ``AGENTS.md`` → "Lazy Evaluation". +file / loads a dataset) fails here. See confluid ``AGENTS.md`` → "Partial Initialization & Zero-Arg +Construction" and recordstream ``AGENTS.md`` → "Partial Evaluation". """ import importlib diff --git a/tests/test_loaders.py b/tests/test_loaders.py index 35bee1b..5c99449 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -19,7 +19,7 @@ def _kwargs(marker: object) -> dict: - """A slot holds a LazyClass MARKER pre-flow; its stored kwargs are what these tests pin.""" + """A slot holds a PartialClass MARKER pre-flow; its stored kwargs are what these tests pin.""" assert isinstance(marker, Fluid) return marker.kwargs diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 1565677..e66c6bc 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -64,9 +64,9 @@ def close(self) -> None: assert closed == [True] def test_fluid_entry_flowed_and_cached(self) -> None: - # A config-deferred entry (confluid Class marker) is flowed on first call and the + # A config-deferred entry (confluid Target marker) is flowed on first call and the # live op is cached back into the transforms list. - p = Pipeline(transforms=[confluid.Class(RenameField, src="class", dst="klass")]) + p = Pipeline(transforms=[confluid.Target(RenameField, src="class", dst="klass")]) out = p(_rec()) assert out is not None and "klass" in out and "class" not in out assert isinstance(p.transforms[0], RenameField) # cached in place diff --git a/tests/test_projection.py b/tests/test_projection.py index e7337c4..06ad97b 100644 --- a/tests/test_projection.py +++ b/tests/test_projection.py @@ -7,7 +7,7 @@ from typing import Any, Collection, Dict, Iterator, List -from confluid import LazyClass +from confluid import PartialClass from recordstream import Label, MultiLabel, first_value, is_class_id from recordstream.items import Record @@ -101,7 +101,7 @@ def test_first_value_asks_only_for_the_requested_key() -> None: def test_first_value_materializes_a_deferred_source() -> None: """``project()`` flows a ``!class:`` marker, so a caller writes no ``flow()`` here.""" - marker = LazyClass(_Source, records=[{"class": "cat"}]) + marker = PartialClass(_Source, records=[{"class": "cat"}]) assert first_value(marker, "class") == "cat" diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 7a5504f..c866532 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -68,7 +68,7 @@ def test_sequence_skips_none_entries() -> None: def test_sequence_caches_flowed_step() -> None: - seq = Sequence([confluid.Class(_RunStep, tag="x")]) + seq = Sequence([confluid.Target(_RunStep, tag="x")]) seq.run() # After run, the deferred marker has been replaced by the live, flowed object. assert isinstance(seq.steps[0], _RunStep) From 621f805732fc124d03cc0f191fecdccb467e40b5 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 17 Aug 2026 15:03:30 +0200 Subject: [PATCH 088/102] docs: DatasetSplit views are selected with split=, not read by attribute reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit confluid removed attribute references (record 19, phase 2): `!ref:my_split.train` is refused. A view is a DatasetSplit marker with `split:` set — the recipe anchored on the train view and <<:-merged into the others; every view !ref:s the same upstream source, which therefore still loads once. No code change (the split= selector already existed): the class docstring, docs/sources.md, docs/runnable.md and AGENTS.md now show that spelling. --- AGENTS.md | 2 +- docs/runnable.md | 2 +- docs/sources.md | 30 +++++++++++++++++++----------- recordstream/sources/split.py | 31 ++++++++++++++++++------------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f36a0ed..3cd776c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). - - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`); in YAML reference them via `!ref:my_split.train` (Confluid's dotted-ref reuses the single materialized instance, so the upstream source loads once — confluid `test_dotted_attribute_ref_reuses_single_instance`). Passing `split=` makes the instance iterate that one view (select-one, `None`⇒`train`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`) for CODE; in YAML a view is a `DatasetSplit` with `split=` set (select-one, `None`⇒`train`) — write the recipe once (`&split_recipe` on the train view) and `<<:`-merge it into the others; every view `!ref:`s the same upstream source, which therefore loads once (the attribute-reference spelling `!ref:my_split.train` is REFUSED by confluid since 2026-08 — record 19 phase 2 — and the refusal names this rewrite; `docs/sources.md`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Boxes` via the shared `connected_component_boxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` / `ResizeDetection` (`recordstream.ops.target` — the first two emit a `Boxes` detection target, lazy-importing torch: one from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the other from a segmentation MASK; `ResizeDetection` is the COUPLED image+boxes resize for fixed-input-size detectors — PIL/uint8 image to `(height, width)` + the `Boxes` boxes scaled by the same factors, torch staying torch, `canvas` updated — run it BEFORE any float conversion such as `ToTensor`, and omit it for detectors that resize internally)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. - **Generic MASK Conversion Lives Here Too — `ConvertToMask` (2026-08-02):** the segmentation counterpart of `ConvertToImage` and the same op SHAPE (read one field, write a differently-typed item under `output`): a mask-bearing field (an ndarray, a torch tensor, or the PIL image a source handed over) becomes an **`int64` `[H, W]` `Mask`** of per-pixel class ids — what a segmentation dataset actually ships (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map) turned into what a per-pixel loss consumes. It belongs HERE, not in a segmentation project: "a mask PNG's pixels are class ids" mentions no modality (the `Threshold` → `Mask` precedent), and a consumer owning it would be the third package to write the conversion. **It converts and NOTHING else, deliberately** — remapping the ids is `FormulaOp` over its output (`formula: a - 1` for a 1-based trimap) or `EncodeTarget` for a lookup table; resizing/augmenting it TOGETHER WITH THE IMAGE is a bare albumentations transform in the same ops list; dropping the source column is `DropField`. Do NOT grow it an `offset` / `mapping` / `dtype` knob: each one restates an op that already exists. **`output` defaults to `"mask"` and that is load-bearing, not a nicety** — it is albumentations' own key vocabulary (`_ALB_KEYS`), so the engine's op-family dispatch hands `image` AND `mask` to ONE call and a single joint draw moves both with the `Mask` type surviving the re-wrap (measured; an image-only transform like `Normalize` still touches the image alone). **`int64` is not a knob either:** a class-id map is integer by definition and it is what `torch.nn.CrossEntropyLoss` requires (*"expected target dtype to be Long or Byte, but got Int"*); a library that casts on the way past — albumentations returns int32 — is corrected at the MODEL boundary by `batch_tensor(..., dtype=...)`, where the caller names the contract (the `dtype`-is-a-parameter rule). It reads through **`item_value`, never `item_data`**, because a source that does not know a column is a mask ships it as a `Label` (`HuggingFaceSource` does this for every metadata column) — see the record-model mandate. Singleton axes are squeezed (`[H,W,1]` / `[1,H,W]` → `[H,W]`); an **RGB-encoded mask RAISES** rather than being collapsed, because picking one of three channels or decoding a palette is a decision the op must not make silently. Pins: `tests/test_convert_to_mask.py` (incl. the whole `preprocess` chain end to end, and that a `Normalize` leaves the mask untouched). Usage: `docs/image.md` → "Masks". - **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. diff --git a/docs/runnable.md b/docs/runnable.md index 9e16c03..3d5cc40 100644 --- a/docs/runnable.md +++ b/docs/runnable.md @@ -7,7 +7,7 @@ processor, a workflow. It is the unit `recordstream run` executes: # config.yaml — the ONE runner shape for every kind of run runnable: !class:mypkg.Classifier task: fit # ← the one knob: fit / evaluate / test / predict - train_set: !ref:my_split.train + train_set: !ref:train_split # a DatasetSplit with `split: train` ``` ```bash diff --git a/docs/sources.md b/docs/sources.md index 9378df5..b87f6f6 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -47,24 +47,32 @@ split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% ``` -The views are disjoint and complementary, computed once over a single deterministic shuffle (cached), so the underlying source is consumed once. In Confluid YAML they're reachable by **attribute reference** — `!ref:my_split.train` / `.val` / `.test`. All three refs resolve to the *same* `DatasetSplit` instance, so the upstream source is loaded **exactly once**: +The views are disjoint and complementary, computed once over a single deterministic shuffle (cached). In a config each view is a `DatasetSplit` with its `split` selector set — write the recipe once (a YAML anchor on the first view) and merge it (`<<:`) into the others. Every view references the *same* `hf_train` (`!ref:` shares the instance), so the upstream source is loaded **exactly once**; a `DatasetSplit`'s own partition is one seeded shuffle over `len(source)`: ```yaml hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() path: ylecun/mnist split: train -my_split: !class:recordstream.sources.split.DatasetSplit() - source: !ref:hf_train - val_fraction: 0.1 - test_fraction: 0.1 - seed: 42 - -train_set: !class:recordstream.core.stream.Stream() { source: !ref:my_split.train } -val_set: !class:recordstream.core.stream.Stream() { source: !ref:my_split.val } -test_set: !class:recordstream.core.stream.Stream() { source: !ref:my_split.test } +train_set: !class:recordstream.core.stream.Stream() + source: &split_recipe !class:recordstream.sources.split.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + split: train +val_set: !class:recordstream.core.stream.Stream() + source: !class:recordstream.sources.split.DatasetSplit() + <<: *split_recipe + split: val +test_set: !class:recordstream.core.stream.Stream() + source: !class:recordstream.sources.split.DatasetSplit() + <<: *split_recipe + split: test ``` +(Reading a view by attribute reference — `!ref:my_split.train` — is no longer a config spelling; the config engine refuses it and names this rewrite.) + Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). **Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `recordstream.SplitName`. @@ -184,4 +192,4 @@ class MyStoreSource: Rationale: [docs/architecture.md §13](architecture.md#13-dataset-identity-is-a-protocol-and-a-view-propagates-it-verbatim-recordstreamuri-2026-08-02). -> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key (including attribute refs like `!ref:my_split.train`), so a single `HuggingFaceSource` is loaded once and shared. Use `!clone:` when you want an independent deep copy instead. +> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key, so a single `HuggingFaceSource` is loaded once and shared. Write the marker again when you want an independent instance instead. diff --git a/recordstream/sources/split.py b/recordstream/sources/split.py index 4b9f90e..5bbf6bf 100644 --- a/recordstream/sources/split.py +++ b/recordstream/sources/split.py @@ -36,21 +36,26 @@ class DatasetSplit: split.test # ≈10% The views are disjoint and complementary, computed once (cached) over a single - deterministic shuffle, so the underlying source is consumed once. In Confluid YAML the - views are reachable by **attribute reference** — ``!ref:my_split.train`` / ``.val`` / - ``.test`` — and because two ``!ref:`` to the same key flow the *same* instance, the - partition and the source load are shared across all three references:: - - my_split: !class:recordstream.sources.split.DatasetSplit() - source: !ref:hf_train - val_fraction: 0.1 - test_fraction: 0.1 - seed: 42 + deterministic shuffle. In a config each view is a ``DatasetSplit`` of its own with the + ``split`` selector set — the recipe is written once (a YAML anchor) and merged (``<<:``) + into the other views. Every view references the SAME ``source`` (``!ref:`` shares the + instance), so the upstream source is loaded exactly once; a ``DatasetSplit``'s own + partition is one seeded shuffle over ``len(source)``, cheap to repeat:: train_set: !class:recordstream.core.stream.Stream() - source: !ref:my_split.train + source: &split_recipe !class:recordstream.sources.split.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + split: train val_set: !class:recordstream.core.stream.Stream() - source: !ref:my_split.val + source: !class:recordstream.sources.split.DatasetSplit() + <<: *split_recipe + split: val + + (Reading a view by attribute reference — ``!ref:my_split.train`` — is no longer a + config spelling; the config engine refuses it and names this rewrite.) **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one view (``split=None`` ⇒ ``train``), so it is directly usable as a single ``source:``. @@ -169,7 +174,7 @@ class _SplitView: """An indexable view of ``source`` restricted (and reordered) to ``indices``. Internal to :class:`DatasetSplit` — produced by its ``train`` / ``val`` / ``test`` - properties (and reachable in Confluid YAML via ``!ref:my_split.train``). Deliberately + properties (in a config, select a view with the ``split`` parameter). Deliberately NOT a ``@configurable``: it is never constructed directly in a config, only read off a live ``DatasetSplit`` instance, so it carries no discovery surface of its own. It stays in this module for the same reason — it is DatasetSplit's own return type, not a From 021a194b256dddbb90c2ac4a65124689082bc6b8 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 18 Aug 2026 09:37:49 +0200 Subject: [PATCH 089/102] =?UTF-8?q?chore:=20migrate=20to=20confluid=20load?= =?UTF-8?q?(until=3D=E2=80=A6)=20=E2=80=94=20materialize/resolve/flow=3D?= =?UTF-8?q?=20folded=20(cli,=20stream,=20flow.graph,=20tests,=20docs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/graph.md | 2 +- recordstream/cli.py | 6 +++--- recordstream/core/stream.py | 5 ++--- recordstream/flow/graph.py | 6 +++--- tests/test_cli_materialize.py | 6 +++--- tests/test_dataset_uri.py | 2 +- tests/test_op_families.py | 2 +- tests/test_typed_flow.py | 4 ++-- tests/test_view_sources_deferred.py | 6 +----- 9 files changed, 17 insertions(+), 22 deletions(-) diff --git a/docs/graph.md b/docs/graph.md index 28939f6..f941a83 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -119,7 +119,7 @@ from recordstream.sources import HuggingFaceSource stream = Stream.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="ylecun/mnist")) ``` -The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.materialize`) so +The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.load`) so a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: `Stream` also flows any still-deferred marker in place at engine-route entry, which is what lets a bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. diff --git a/recordstream/cli.py b/recordstream/cli.py index 8c55672..3b07b56 100644 --- a/recordstream/cli.py +++ b/recordstream/cli.py @@ -47,7 +47,7 @@ def materialize_runnable(runnable: Any) -> Any: the constructor default and looks configured. So this reaches back to the loaded document through liquifai's context and calls - ``materialize(node, context=document)``. Nested stubs still ride the normal deferred + ``load(node, context=document)``. Nested stubs still ride the normal deferred path — a ``!lazy:`` marker stays deferred for the runnable to flow at run time. **liquifai 0.1.1 fixes this at its own layer** (``di.deep_flow`` now takes the @@ -74,7 +74,7 @@ def run(runnable: Any) -> None: runnable = materialize_runnable(runnable) runnable.run() """ - from confluid import flow, materialize + from confluid import flow, load from confluid.fluid import Fluid from liquifai.context import get_context @@ -84,7 +84,7 @@ def run(runnable: Any) -> None: context = get_context() document = getattr(context, "config_data", None) if context is not None else None if isinstance(document, dict): - return materialize(runnable, context=document) + return load(runnable, context=document) return flow(runnable) diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py index abf74cb..fa0c549 100644 --- a/recordstream/core/stream.py +++ b/recordstream/core/stream.py @@ -25,7 +25,6 @@ from confluid import configurable from confluid import load as _confluid_load -from confluid import materialize as _confluid_materialize from confluid.fluid import Fluid as _ConfluidFluid from loggair import get_logger @@ -225,11 +224,11 @@ def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "St ``path`` is the ``{ops: [!class:...()]}`` document produced by an external graph exporter's ops-export. Op markers are materialized to live callables before being attached (``confluid.load`` leaves ``!class:`` markers nested under a mapping key - deferred, so ``confluid.materialize`` flows them into live ops). + deferred, so ``confluid.load`` flows them into live ops). """ loaded = _confluid_load(path) raw_ops = loaded.get("ops", []) if isinstance(loaded, dict) else [] - ops = list(_confluid_materialize(raw_ops)) + ops = list(_confluid_load(raw_ops)) return cls(source=source, ops=ops) @property diff --git a/recordstream/flow/graph.py b/recordstream/flow/graph.py index f1ef3cb..ba89602 100644 --- a/recordstream/flow/graph.py +++ b/recordstream/flow/graph.py @@ -11,7 +11,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast from confluid import configurable -from confluid import resolve as _confluid_resolve +from confluid import load as _confluid_load from loggair import get_logger from recordstream.core.families import _extra_op_families, _op_expands @@ -92,10 +92,10 @@ def _ensure_readers(self) -> Dict[str, List[Tuple[int, str]]]: def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": """Build a FlowGraph from a ``{flow: {...}, outputs: ...}`` YAML document (or inline string). - Uses ``confluid.resolve`` so step markers stay UNbuilt until :func:`parse_flow` + Uses ``confluid.load(until="settled")`` so step markers stay UNbuilt until :func:`parse_flow` pops the reserved step keys and flows each op itself. """ - doc = _confluid_resolve(path) + doc = _confluid_load(path, until="settled") if not isinstance(doc, dict) or "flow" not in doc: raise ValueError(f"FlowGraph.from_yaml: {path!r} has no 'flow:' mapping") return cls(source=source, flow=doc["flow"], outputs=str(doc.get("outputs", "") or "")) diff --git a/tests/test_cli_materialize.py b/tests/test_cli_materialize.py index 645ea15..05f961f 100644 --- a/tests/test_cli_materialize.py +++ b/tests/test_cli_materialize.py @@ -59,8 +59,8 @@ def _install(config_data: Any) -> None: def _node(text: str = FLAT_CONFIG) -> Any: - """The document + its ``runnable:`` node, loaded the way liquifai loads it (flow=False).""" - document = confluid.load(text, flow=False) + """The document + its ``runnable:`` node, loaded the way liquifai loads it (until="document").""" + document = confluid.load(text, until="document") return document, document["runnable"] @@ -108,7 +108,7 @@ def test_no_liquifai_context_falls_back_to_flow(liquifai_context: Any) -> None: def test_a_root_fluid_document_falls_back_to_flow(liquifai_context: Any) -> None: """A YAML whose root is a single `!class:` has no siblings — nothing is lost.""" - document = confluid.load("!class:tests.test_cli_materialize._Runner\nmax_epochs: 5\n", flow=False) + document = confluid.load("!class:tests.test_cli_materialize._Runner\nmax_epochs: 5\n", until="document") liquifai_context(document) # not a dict runner = materialize_runnable(document) diff --git a/tests/test_dataset_uri.py b/tests/test_dataset_uri.py index bd605c0..4c65624 100644 --- a/tests/test_dataset_uri.py +++ b/tests/test_dataset_uri.py @@ -199,7 +199,7 @@ def test_a_deferred_config_marker_is_materialized_before_being_asked() -> None: node = load( "!class:recordstream.sources.huggingface.HuggingFaceSource()\n path: ylecun/mnist\n split: train\n", - flow=False, + until="document", ) assert dataset_uri(node) == "hf://datasets/ylecun/mnist?split=train" diff --git a/tests/test_op_families.py b/tests/test_op_families.py index b0a2a37..1479023 100644 --- a/tests/test_op_families.py +++ b/tests/test_op_families.py @@ -485,7 +485,7 @@ def test_the_DOCUMENTED_yaml_way_out_actually_runs(self) -> None: "labels": [1], } with _captured_warnings() as warnings: - out = list(Stream(source=[record], ops=confluid.load(document, flow=True)["ops"]))[0] + out = list(Stream(source=[record], ops=confluid.load(document)["ops"]))[0] assert warnings == [] assert [round(v, 1) for v in out["bboxes"][0]] == [60.0, 10.0, 90.0, 40.0], "mirrored across x" diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py index 83f5f01..eeef1e5 100644 --- a/tests/test_typed_flow.py +++ b/tests/test_typed_flow.py @@ -212,7 +212,7 @@ def test_yaml_bind_runs_the_same_from_either_loader(self, tmp_path: Path) -> Non from_yaml = list(FlowGraph.from_yaml(str(path), source=[dict(record)])) import confluid - doc = confluid.resolve(str(path)) + doc = confluid.load(str(path), until="settled") parsed = list(FlowGraph(source=[dict(record)], flow=doc["flow"], outputs=str(doc.get("outputs", "")))) assert len(from_yaml) == len(parsed) == 1 assert set(from_yaml[0]) == set(parsed[0]) @@ -233,7 +233,7 @@ def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path: Path) - ) import confluid - marker = confluid.resolve(str(path))["flow"]["gated"] + marker = confluid.load(str(path), until="settled")["flow"]["gated"] assert "bind" not in marker.kwargs # consumed as addressed configuration diff --git a/tests/test_view_sources_deferred.py b/tests/test_view_sources_deferred.py index 9fbacad..1b7249f 100644 --- a/tests/test_view_sources_deferred.py +++ b/tests/test_view_sources_deferred.py @@ -41,7 +41,6 @@ def test_a_partial_marker_under_range_source_raises_the_stream_guidance() -> Non source: {{_target_: {_LEAF}, _partial_: true}} stop: 3 """, - flow=True, ) with pytest.raises(TypeError) as excinfo: cfg["range_src"].indices @@ -57,7 +56,6 @@ def test_a_partial_marker_under_concat_source_names_the_offending_index() -> Non - {{_target_: {_LEAF}}} - {{_target_: {_LEAF}, _partial_: true}} """, - flow=True, ) with pytest.raises(TypeError) as excinfo: cfg["concat_src"].offsets @@ -71,7 +69,6 @@ def test_a_partial_marker_under_dataset_split_raises_the_stream_guidance() -> No _target_: recordstream.sources.split.DatasetSplit source: {{_target_: {_LEAF}, _partial_: true}} """, - flow=True, ) with pytest.raises(TypeError) as excinfo: len(cfg["split_src"].train) @@ -91,7 +88,6 @@ def test_the_parens_spelling_materializes_and_the_view_sources_work() -> None: split_src: !class:recordstream.sources.split.DatasetSplit() source: !class:{_LEAF}() """, - flow=True, ) range_src: RangeSource = cfg["range_src"] concat_src: ConcatSource = cfg["concat_src"] @@ -104,6 +100,6 @@ def test_the_parens_spelling_materializes_and_the_view_sources_work() -> None: def test_streams_own_guidance_still_names_its_slot() -> None: """The shared message helper's default slot stays ``Stream.source``.""" - cfg = load(f"deferred: {{_target_: {_LEAF}, _partial_: true}}", flow=True) + cfg = load(f"deferred: {{_target_: {_LEAF}, _partial_: true}}") with pytest.raises(TypeError, match="Stream.source is still a deferred Confluid marker"): len(Stream(source=cfg["deferred"])) From 36c7d52b47aaa731fc6e520208c58c205631f141 Mon Sep 17 00:00:00 2001 From: gearlux Date: Sat, 22 Aug 2026 10:25:14 +0200 Subject: [PATCH 090/102] chore(ci): install loggair from git now that its source is local MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerated after `aisland source set loggair local`: loggair joins confluid and liquifai in the internal-dependency block, installed with --no-deps from git+https://github.com/Gearlux/loggair.git@main before `-e .[dev]` so the extra resolves it pre-satisfied instead of reaching for PyPI. Generated artefact — the template is aisland's, not this file. --- .github/workflows/ci.yml | 4 ++++ Jenkinsfile | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e63c3bc..c7132c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" @@ -58,6 +59,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" @@ -92,6 +94,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" @@ -130,6 +133,7 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. + uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" diff --git a/Jenkinsfile b/Jenkinsfile index c6c71e3..02a12eb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,6 +33,7 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). + sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" From 0e81301a1c82d97da5c08c5e18a7eb0318c61ad8 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 10:29:49 +0200 Subject: [PATCH 091/102] feat(loaders): let `loader_slots` take an explicit `persistent_workers` override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `persistent_workers` still DERIVES from `num_workers != 0` when the new keyword is left at `None`, which is what nearly every run wants. An explicit value covers the case the derivation cannot see — a HOST fact: macOS terminates persistent workers slowly enough that a short run spends longer stopping than training, so a config there sets `false` while keeping its workers. `True` with `num_workers=0` is refused HERE with a message naming the fix; torch otherwise raises for that pairing when the loader is first iterated, minutes into a run. That invariant was the one thing the derived-only value protected by construction. --- AGENTS.md | 2 +- docs/architecture.md | 27 +++++++++++++++++++++++++++ recordstream/loaders.py | 24 +++++++++++++++++++----- tests/test_loaders.py | 23 +++++++++++++++++++++++ 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3cd776c..292b1e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. - **A Source NAMES The Data It Reads — `dataset_uri` + `dataset_url`, And A View Propagates Them VERBATIM (`recordstream.uri`, 2026-08-02):** `SupportsDatasetIdentity` is a `@runtime_checkable` Protocol with TWO properties, and the pair is deliberate rather than one field doing double duty: **`dataset_uri`** is the CANONICAL handle (machine-parseable, stable across machines, the string two runs are COMPARED on — `hf://datasets/ylecun/mnist?split=train`, matching the convention hosted tracking services already use for a dataset source) while **`dataset_url`** is a link a PERSON opens and is `None` whenever the data has no web page. Collapsing them forces a choice between a browsable string that lies about local data and a canonical one nobody can click. `HuggingFaceSource` implements both: a Hub repo id -> `hf://datasets/?…`, a local directory -> its `file://` URI, decided by `Path(path).exists()` (the same question `load_dataset` answers); the query params are SORTED so one configuration has exactly ONE string, and both read STORED CONFIG ONLY — asking never loads, so a source that is never iterated still names itself (`tests/test_dataset_uri.py::test_asking_for_identity_never_loads_the_dataset`). **`revision` is a DECLARED ctor param BECAUSE identity reads it** (it was in the removed `**kwargs`), and `load_options` merges it over `load_kwargs` as a read-only property rather than in `__init__` — a declared key may be set post-construction, and a dict assembled in the constructor would keep the value the object was born with. **Following happens in the FREE FUNCTIONS, not in every wrapper:** `dataset_uri(x)` / `dataset_url(x)` flow a deferred `!class:` marker (as `project` does) and then follow a `.source` attribute when the object holds no handle — depth-capped (`MAX_WRAPPER_DEPTH`) and cycle-safe — so `Stream` / `DatasetSplit` / `_SplitView` / `RangeSource` / `MetadataFilterSource` all work with ZERO code of their own, as does any third-party wrapper using that attribute name. **A wrapper must NOT decorate the URI it passes up** (no `#train`, no `#0:1000`): the handle identifies the DATASET, how much of it a run consumed is already recorded by the wrapper's own configuration, and decorating would mean one dataset reached two ways stops comparing equal — the single property the handle exists to have, and what makes a consumer's dedup exact. **`ConcatSource` answers `None` ON PURPOSE** — several datasets end to end are not one dataset, and picking a member would be a lie; `dataset_uris(source)` is the plural form that fans out over `.sources` (recursive, deduplicated). `None` is an ordinary answer everywhere (unconfigured source, in-memory stream, data with no page), never an error. Adding identity to a new source is TWO properties and no registration. Package-root exports + `__all__`; no entry point (the module holds no `@configurable`, same as `projection.py`). Rationale: `docs/architecture.md` §13. Usage: `docs/sources.md` → "Identifying a dataset". Pins: `tests/test_dataset_uri.py`. -- **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset` — and since 2026-08-06 the composition itself ships as **`prepare_record_dataset(source)`** = `ensure_materialized(ensure_record_dataset(source))` with `None` passing through (the `_prepare` helper every training runnable had re-composed; a consumer needing only one half still calls that half). **The torch loader triple ships beside it (`recordstream.loaders.loader_slots`, 2026-08-06):** the train/val/test `LazyClass(DataLoader, ...)` slots every torch training runnable declared identically, returned as the `LoaderSlots` NamedTuple (`slots.train`/`.val`/`.test`, typed `Lazy[DataLoader[Any]]`; train shuffled, eval not, `persistent_workers` derived from `num_workers`, `collate_fn=` the batch-shape choice, further DataLoader kwargs passing through to all three — `shuffle` refused as a shared kwarg because it is the one per-split decision the helper owns). The module is torch-ONLY and deliberately NOT package-root exported (the `ops.torch` pattern — a torch-free import of it raises an `ImportError` naming the extra); consumers import `from recordstream.loaders import loader_slots`. Config-side, the three runnable slots stay whole-value replaceable in YAML exactly as with the inline construction this replaces. Pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise), `tests/test_loaders.py`. +- **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset` — and since 2026-08-06 the composition itself ships as **`prepare_record_dataset(source)`** = `ensure_materialized(ensure_record_dataset(source))` with `None` passing through (the `_prepare` helper every training runnable had re-composed; a consumer needing only one half still calls that half). **The torch loader triple ships beside it (`recordstream.loaders.loader_slots`, 2026-08-06):** the train/val/test `LazyClass(DataLoader, ...)` slots every torch training runnable declared identically, returned as the `LoaderSlots` NamedTuple (`slots.train`/`.val`/`.test`, typed `Lazy[DataLoader[Any]]`; train shuffled, eval not, `persistent_workers` derived from `num_workers` unless the caller NAMES it (`persistent_workers=` — `None` derives, an explicit value wins, and `True` with zero workers is refused at construction instead of at first iteration; the knob exists for a HOST fact no derivation can see, e.g. macOS terminating persistent workers slowly), `collate_fn=` the batch-shape choice, further DataLoader kwargs passing through to all three — `shuffle` refused as a shared kwarg because it is the one per-split decision the helper owns). The module is torch-ONLY and deliberately NOT package-root exported (the `ops.torch` pattern — a torch-free import of it raises an `ImportError` naming the extra); consumers import `from recordstream.loaders import loader_slots`. Config-side, the three runnable slots stay whole-value replaceable in YAML exactly as with the inline construction this replaces. Pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise), `tests/test_loaders.py`. - **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). diff --git a/docs/architecture.md b/docs/architecture.md index 913cf85..82809a8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -591,6 +591,33 @@ accepts_broadcast(Enable, "enabled") # True — the bare --enabled form no than teaching one wrapper several toggles — each name is independently addressable, and the broadcast form still flips them all. +### Amendment: a DERIVED value is a knob nobody can reach either (2026-08-22) + +`loader_slots` derived `persistent_workers` from `num_workers` (`!= 0`) and its docstring said +there was "deliberately no separate knob, because the pairing never varies". The pairing does vary, +along an axis the derivation cannot see: macOS terminates persistent workers slowly enough that a +short run spends longer stopping than training, so a config there wants workers WITHOUT persistence. +Reaching that meant replacing all three loader slots wholesale — twelve YAML lines that also had to +restate `collate_fn`, because a replaced slot loses the code default and torch's `default_collate` +then crashes on string metadata. + +The reason a plain top-level key was not enough is worth stating, because the neighbouring case +looks identical and behaves differently: a flat config key DOES reach these code-created markers by +broadcasting — `multiprocessing_context: fork` lands in all three slots with no parameter anywhere +(measured) — but only for a kwarg the marker does not already carry. `persistent_workers` is BAKED +here at construction, so there was nothing for a config to reach. + +The rule above answers it unchanged: **if a front-end must set it, declare it.** The parameter is +`persistent_workers: Optional[bool] = None` — `None` derives exactly as before (so nothing that +does not pass it changes), an explicit value wins, and `True` with `num_workers=0` raises at +construction rather than inside torch at first iteration, which is the one invariant the +derived-only version had protected by construction. The same parameter is declared by every +consuming runnable, so it broadcasts from a flat config like `batch_size` does. + +The generalisation for the next time: deriving a value is not a way to avoid declaring it. A +derivation is a good DEFAULT and a bad ONLY option — the moment one caller knows something the +derivation cannot, the undeclared value costs a wholesale replacement of the object that holds it. + ## 7. The `@entrypoint` markers ARE the dispatch table (`run_entrypoint`, 2026-07-29) ### Context diff --git a/recordstream/loaders.py b/recordstream/loaders.py index 681c8c7..95e1ff4 100644 --- a/recordstream/loaders.py +++ b/recordstream/loaders.py @@ -17,7 +17,7 @@ from recordstream.loaders import loader_slots """ -from typing import Any, Callable, List, NamedTuple +from typing import Any, Callable, List, NamedTuple, Optional from confluid import Partial, PartialClass @@ -47,6 +47,7 @@ def loader_slots( num_workers: int, *, collate_fn: Callable[[List[Record]], Record] = collate_records, + persistent_workers: Optional[bool] = None, **loader_kw: Any, ) -> LoaderSlots: """The train/val/test deferred ``DataLoader`` triple every torch training runnable declares. @@ -64,9 +65,16 @@ def loader_slots( Args: batch_size: Rows per batch, baked into all three loaders. - num_workers: Worker processes per loader. ``persistent_workers`` derives from it - (``num_workers != 0``) — there is deliberately no separate knob, because persistent - workers with zero workers is a torch error and the pairing never varies. + num_workers: Worker processes per loader. ``persistent_workers`` DERIVES from it + (``num_workers != 0``) unless the caller says otherwise. + persistent_workers: Override the derived pairing. ``None`` (the default) derives as + above and is what nearly every run wants; an explicit value is for the case the + derivation cannot see — a HOST fact. macOS terminates persistent workers slowly + enough that a short run spends longer stopping than training, so a config there + says ``persistent_workers: false`` while keeping its workers. ``True`` with + ``num_workers=0`` is refused HERE (torch raises for that pairing when the loader + is first iterated, minutes into a run); it is the one invariant the previously + derived-only value protected by construction. collate_fn: The batch-shape choice (see the collate registry) — ``collate_records`` stacks, ``collate_list`` does not (what a detection consumer passes), and a task collate is any callable. @@ -87,11 +95,17 @@ def loader_slots( "a shared kwarg — replace the individual loader slot instead " "(e.g. train_loader: !class:torch.utils.data.DataLoader {shuffle: false, ...})." ) + if persistent_workers and num_workers == 0: + raise ValueError( + "loader_slots: persistent_workers=True needs num_workers > 0 — torch keeps worker " + "processes alive between epochs and there are none. Raise num_workers, or leave " + "persistent_workers unset to derive it." + ) shared = dict( collate_fn=collate_fn, batch_size=batch_size, num_workers=num_workers, - persistent_workers=num_workers != 0, + persistent_workers=(num_workers != 0) if persistent_workers is None else persistent_workers, **loader_kw, ) return LoaderSlots( diff --git a/tests/test_loaders.py b/tests/test_loaders.py index 5c99449..f8b5c47 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -46,6 +46,29 @@ def test_the_shared_kwargs_are_baked_and_persistent_workers_derives() -> None: assert _kwargs(loader_slots(batch_size=8, num_workers=2).train)["persistent_workers"] is True +def test_an_explicit_persistent_workers_wins_over_the_derived_default() -> None: + """The host-scoped case: `persistent_workers: false` in YAML reaches all three slots + while `num_workers` stays 8 (macOS terminates persistent workers slowly).""" + slots = loader_slots(batch_size=8, num_workers=8, persistent_workers=False) + for split, marker in zip(("train", "val", "test"), slots): + assert _kwargs(marker)["persistent_workers"] is False, split + assert _kwargs(marker)["num_workers"] == 8, split + + +def test_none_still_derives_from_num_workers() -> None: + """The default is unchanged: not passing the knob is exactly the old behaviour.""" + assert _kwargs(loader_slots(batch_size=8, num_workers=2, persistent_workers=None).train) == _kwargs( + loader_slots(batch_size=8, num_workers=2).train + ) + + +def test_persistent_workers_true_without_workers_is_refused() -> None: + """torch raises for this pairing at ITERATION time; the knob makes it a construction-time + error instead, which is the invariant the derived-only version protected by construction.""" + with pytest.raises(ValueError, match="persistent_workers=True needs num_workers > 0"): + loader_slots(batch_size=2, num_workers=0, persistent_workers=True) + + def test_the_collate_is_the_batch_shape_choice() -> None: """A detection consumer passes collate_list; the slot carries it verbatim.""" assert ( From ded2bbb5f5447dbf824eb4e1ffb138ce74dc24ae Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 10:50:07 +0200 Subject: [PATCH 092/102] chore: ignore prefixed junit reports (*test-report.xml) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generated slow-test pipeline writes `slow-test-report.xml` (aisland/aisland/services/jenkins.py:864 for the Jenkins stage, :1068 for the GH Actions job), which the exact-name `test-report.xml` entry never matched — so a local slow run dropped an untracked artifact one non-ignored file away from being committable. The glob covers both names. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f1d2810..81bde28 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,7 @@ dir_store/ flake8.txt mypy.txt coverage.xml -test-report.xml +*test-report.xml coverage/ /pyrightconfig.json From 46131a485a227378eab125f952c17403125a6fb3 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 15:45:15 +0200 Subject: [PATCH 093/102] feat: dataset_uri never builds a marker whose class cannot answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A deferred marker is materialized only when its target class declares the identity protocol (dataset_uri / dataset_url) or a source slot the walk would follow; anything else answers None without being constructed — a run provenance capture used to build a full pl.Trainer and attempt a dataset-less DataLoader per walk just to learn each names no dataset. An unresolvable target keeps the old build path, so nothing that answered before answers differently. --- AGENTS.md | 2 +- recordstream/uri.py | 56 ++++++++++++++++++++++++++++++++++++++- tests/test_dataset_uri.py | 44 ++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 292b1e4..d694d62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. - **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. - **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. -- **A Source NAMES The Data It Reads — `dataset_uri` + `dataset_url`, And A View Propagates Them VERBATIM (`recordstream.uri`, 2026-08-02):** `SupportsDatasetIdentity` is a `@runtime_checkable` Protocol with TWO properties, and the pair is deliberate rather than one field doing double duty: **`dataset_uri`** is the CANONICAL handle (machine-parseable, stable across machines, the string two runs are COMPARED on — `hf://datasets/ylecun/mnist?split=train`, matching the convention hosted tracking services already use for a dataset source) while **`dataset_url`** is a link a PERSON opens and is `None` whenever the data has no web page. Collapsing them forces a choice between a browsable string that lies about local data and a canonical one nobody can click. `HuggingFaceSource` implements both: a Hub repo id -> `hf://datasets/?…`, a local directory -> its `file://` URI, decided by `Path(path).exists()` (the same question `load_dataset` answers); the query params are SORTED so one configuration has exactly ONE string, and both read STORED CONFIG ONLY — asking never loads, so a source that is never iterated still names itself (`tests/test_dataset_uri.py::test_asking_for_identity_never_loads_the_dataset`). **`revision` is a DECLARED ctor param BECAUSE identity reads it** (it was in the removed `**kwargs`), and `load_options` merges it over `load_kwargs` as a read-only property rather than in `__init__` — a declared key may be set post-construction, and a dict assembled in the constructor would keep the value the object was born with. **Following happens in the FREE FUNCTIONS, not in every wrapper:** `dataset_uri(x)` / `dataset_url(x)` flow a deferred `!class:` marker (as `project` does) and then follow a `.source` attribute when the object holds no handle — depth-capped (`MAX_WRAPPER_DEPTH`) and cycle-safe — so `Stream` / `DatasetSplit` / `_SplitView` / `RangeSource` / `MetadataFilterSource` all work with ZERO code of their own, as does any third-party wrapper using that attribute name. **A wrapper must NOT decorate the URI it passes up** (no `#train`, no `#0:1000`): the handle identifies the DATASET, how much of it a run consumed is already recorded by the wrapper's own configuration, and decorating would mean one dataset reached two ways stops comparing equal — the single property the handle exists to have, and what makes a consumer's dedup exact. **`ConcatSource` answers `None` ON PURPOSE** — several datasets end to end are not one dataset, and picking a member would be a lie; `dataset_uris(source)` is the plural form that fans out over `.sources` (recursive, deduplicated). `None` is an ordinary answer everywhere (unconfigured source, in-memory stream, data with no page), never an error. Adding identity to a new source is TWO properties and no registration. Package-root exports + `__all__`; no entry point (the module holds no `@configurable`, same as `projection.py`). Rationale: `docs/architecture.md` §13. Usage: `docs/sources.md` → "Identifying a dataset". Pins: `tests/test_dataset_uri.py`. +- **A Source NAMES The Data It Reads — `dataset_uri` + `dataset_url`, And A View Propagates Them VERBATIM (`recordstream.uri`, 2026-08-02):** `SupportsDatasetIdentity` is a `@runtime_checkable` Protocol with TWO properties, and the pair is deliberate rather than one field doing double duty: **`dataset_uri`** is the CANONICAL handle (machine-parseable, stable across machines, the string two runs are COMPARED on — `hf://datasets/ylecun/mnist?split=train`, matching the convention hosted tracking services already use for a dataset source) while **`dataset_url`** is a link a PERSON opens and is `None` whenever the data has no web page. Collapsing them forces a choice between a browsable string that lies about local data and a canonical one nobody can click. `HuggingFaceSource` implements both: a Hub repo id -> `hf://datasets/?…`, a local directory -> its `file://` URI, decided by `Path(path).exists()` (the same question `load_dataset` answers); the query params are SORTED so one configuration has exactly ONE string, and both read STORED CONFIG ONLY — asking never loads, so a source that is never iterated still names itself (`tests/test_dataset_uri.py::test_asking_for_identity_never_loads_the_dataset`). **`revision` is a DECLARED ctor param BECAUSE identity reads it** (it was in the removed `**kwargs`), and `load_options` merges it over `load_kwargs` as a read-only property rather than in `__init__` — a declared key may be set post-construction, and a dict assembled in the constructor would keep the value the object was born with. **Following happens in the FREE FUNCTIONS, not in every wrapper:** `dataset_uri(x)` / `dataset_url(x)` flow a deferred `!class:` marker (as `project` does) and then follow a `.source` attribute when the object holds no handle — depth-capped (`MAX_WRAPPER_DEPTH`) and cycle-safe. **A deferred marker is flowed ONLY when its target CLASS could answer (2026-08-24):** `_materialize` resolves the class without building (`uri._target_class` — class object, registry name, or importable dotted path; a builder FUNCTION stays conservative and builds) and answers `None` when it declares neither the identity properties nor a `source` slot (`hasattr` + `confluid.declares_key`) — a run's provenance capture used to construct a full `pl.Trainer` (and attempt a dataset-less `DataLoader`) per walk just to learn each names no dataset. Pins: the never-built group in `tests/test_dataset_uri.py` (the guard plus both still-builds con cases) — so `Stream` / `DatasetSplit` / `_SplitView` / `RangeSource` / `MetadataFilterSource` all work with ZERO code of their own, as does any third-party wrapper using that attribute name. **A wrapper must NOT decorate the URI it passes up** (no `#train`, no `#0:1000`): the handle identifies the DATASET, how much of it a run consumed is already recorded by the wrapper's own configuration, and decorating would mean one dataset reached two ways stops comparing equal — the single property the handle exists to have, and what makes a consumer's dedup exact. **`ConcatSource` answers `None` ON PURPOSE** — several datasets end to end are not one dataset, and picking a member would be a lie; `dataset_uris(source)` is the plural form that fans out over `.sources` (recursive, deduplicated). `None` is an ordinary answer everywhere (unconfigured source, in-memory stream, data with no page), never an error. Adding identity to a new source is TWO properties and no registration. Package-root exports + `__all__`; no entry point (the module holds no `@configurable`, same as `projection.py`). Rationale: `docs/architecture.md` §13. Usage: `docs/sources.md` → "Identifying a dataset". Pins: `tests/test_dataset_uri.py`. - **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset` — and since 2026-08-06 the composition itself ships as **`prepare_record_dataset(source)`** = `ensure_materialized(ensure_record_dataset(source))` with `None` passing through (the `_prepare` helper every training runnable had re-composed; a consumer needing only one half still calls that half). **The torch loader triple ships beside it (`recordstream.loaders.loader_slots`, 2026-08-06):** the train/val/test `LazyClass(DataLoader, ...)` slots every torch training runnable declared identically, returned as the `LoaderSlots` NamedTuple (`slots.train`/`.val`/`.test`, typed `Lazy[DataLoader[Any]]`; train shuffled, eval not, `persistent_workers` derived from `num_workers` unless the caller NAMES it (`persistent_workers=` — `None` derives, an explicit value wins, and `True` with zero workers is refused at construction instead of at first iteration; the knob exists for a HOST fact no derivation can see, e.g. macOS terminating persistent workers slowly), `collate_fn=` the batch-shape choice, further DataLoader kwargs passing through to all three — `shuffle` refused as a shared kwarg because it is the one per-split decision the helper owns). The module is torch-ONLY and deliberately NOT package-root exported (the `ops.torch` pattern — a torch-free import of it raises an `ImportError` naming the extra); consumers import `from recordstream.loaders import loader_slots`. Config-side, the three runnable slots stay whole-value replaceable in YAML exactly as with the inline construction this replaces. Pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise), `tests/test_loaders.py`. - **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. - **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": diff --git a/recordstream/uri.py b/recordstream/uri.py index 12b38b4..400db9e 100644 --- a/recordstream/uri.py +++ b/recordstream/uri.py @@ -88,10 +88,64 @@ def _materialize(node: Any) -> Any: ``flow()`` on a LIVE object still runs confluid's post-construction ``solidify()`` hook, so a source that grows one would be materialized merely by being asked its name. A marker has nothing to read until it is built, and building one is cheap by the lazy-construction rule. + + A deferred marker is built only when its target CLASS could possibly answer — it declares + the identity protocol (a ``dataset_uri`` / ``dataset_url`` property) or a ``source`` slot + the walk would follow. Anything else answers ``None`` WITHOUT being constructed: a run's + provenance capture walks every configured slot, and it used to build a full ``pl.Trainer`` + (and attempt a dataset-less ``DataLoader``) per walk just to learn each names no dataset. + An unresolvable target keeps the old build path, so nothing that answered before answers + differently now. """ from confluid import Fluid, flow - return flow(node) if isinstance(node, Fluid) else node + if not isinstance(node, Fluid): + return node + target = _target_class(node) + if target is not None and not _has_identity_surface(target): + return None + return flow(node) + + +def _target_class(marker: Any) -> Optional[type]: + """The marker's target as a CLASS, resolved without building — ``None`` when it cannot be. + + A builder FUNCTION target also answers ``None`` (its return type is unknowable statically), + so the caller stays conservative and builds, exactly as before the guard existed. + """ + target = getattr(marker, "target", None) + if isinstance(target, type): + return target + if isinstance(target, str): + from confluid.registry import resolve_class + + resolved = resolve_class(target) + if isinstance(resolved, type): + return resolved + module, _, name = target.rpartition(".") + if module: + try: + import importlib + + candidate = getattr(importlib.import_module(module), name, None) + except Exception: + return None + if isinstance(candidate, type): + return candidate + return None + + +def _has_identity_surface(cls: type) -> bool: + """Whether ``cls`` declares anything the identity walk could read. + + ``hasattr`` catches the protocol properties and a class-level ``source``; + ``declares_key`` catches a ``source`` constructor parameter / body slot. + """ + if hasattr(cls, "dataset_uri") or hasattr(cls, "dataset_url") or hasattr(cls, "source"): + return True + from confluid import declares_key + + return declares_key(cls, "source") def dataset_uri(source: Any) -> Optional[str]: diff --git a/tests/test_dataset_uri.py b/tests/test_dataset_uri.py index 4c65624..2541d77 100644 --- a/tests/test_dataset_uri.py +++ b/tests/test_dataset_uri.py @@ -234,3 +234,47 @@ def test_dataset_uris_skips_members_that_have_no_identity() -> None: concat = ConcatSource(sources=[_FakeSource(), HuggingFaceSource(path="a/b")]) found: List[str] = dataset_uris(concat) assert found == ["hf://datasets/a/b?split=train"] + + +# --- asking never builds: a deferred marker's CLASS is checked before construction ------------- + +_BUILDS = {"count": 0} + + +class _NoDatasetSurface: + """No identity properties, no ``source`` slot — the shape of a deferred trainer/loader.""" + + def __init__(self, max_epochs: int = 1) -> None: + _BUILDS["count"] += 1 + self.max_epochs = max_epochs + + +def test_a_deferred_marker_whose_class_has_no_dataset_surface_is_never_built() -> None: + from confluid import PartialClass + + _BUILDS["count"] = 0 + assert dataset_uri(PartialClass(_NoDatasetSurface, max_epochs=3)) is None + assert dataset_url(PartialClass(_NoDatasetSurface, max_epochs=3)) is None + assert _BUILDS["count"] == 0, "the class answers the question; nothing is constructed" + + +def test_a_deferred_source_with_an_identity_property_still_builds_and_answers() -> None: + class _IdSource: + @property + def dataset_uri(self) -> str: + return "hf://datasets/x" + + from confluid import Target + + assert dataset_uri(Target(_IdSource)) == "hf://datasets/x" + + +def test_a_deferred_wrapper_with_a_source_slot_is_still_built_and_followed() -> None: + class _Wrap: + def __init__(self, source: Any = None) -> None: + self.source = source + + from confluid import Target + + inner = _FakeSource(uri="hf://datasets/inner") + assert dataset_uri(Target(_Wrap, source=inner)) == "hf://datasets/inner" From f2a77f2f26d8a0f93ce0af4b5a2974bb622a6c17 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 16:04:51 +0200 Subject: [PATCH 094/102] =?UTF-8?q?feat:=20ModelPredict=20=E2=80=94=20infe?= =?UTF-8?q?rence=20as=20a=20pipeline=20op?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pipeline can carry its own inference: ModelPredict runs any callable model wrapper on each record and stamps the prediction back as a record field — a Label (classification), Boxes (detection), an int class mask (segmentation, _mask), or the restored image (restoration). The wrapper's heavy work (build the network, load checkpoint_path) happens in its solidify(), called lazily on the first record; the op imports no ML framework (torch duck-typed). Registered as category="op" (new confluid.configurables entry point), so a visual editor's palette and registry pickers list it automatically. --- README.md | 21 ++++ pyproject.toml | 1 + recordstream/ops/predict.py | 195 ++++++++++++++++++++++++++++++++++++ tests/test_ops_predict.py | 71 +++++++++++++ 4 files changed, 288 insertions(+) create mode 100644 recordstream/ops/predict.py create mode 100644 tests/test_ops_predict.py diff --git a/README.md b/README.md index 492b46f..c8474a9 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,27 @@ Inner ops are not materialized until the wrapper first fires, so gating an expen `Enable(ops=[...], name="visualize", enabled=False)` — which is what lets a visual editor or a generated tool schema set the toggle too (see [docs/architecture.md](docs/architecture.md#6-every-knob-is-a-declared-parameter--the-enable-toggle-2026-07-27)). +### Inference as an op (`ModelPredict`) + +A pipeline can carry its own inference: `recordstream.ops.predict.ModelPredict` runs any +callable model wrapper on each record and stamps the prediction back as a record field — +a class `Label`, `Boxes`, an int class mask, or the restored image, by `kind`. The model's +heavy work (build the network, load `checkpoint_path`) happens in its `solidify()`, called +lazily on the first record; the op itself imports no ML framework. + +```yaml +pipeline: !class:recordstream.core.stream.Stream + source: !class:recordstream.sources.huggingface.HuggingFaceSource {path: ylecun/mnist, split: test} + ops: + - !class:recordstream.ops.image.ConvertToImage {width: 224, height: 224} + - !class:recordstream.ops.predict.ModelPredict + model: !class: {checkpoint_path: runs/checkpoints/mnist/last.ckpt} + kind: classification # or detection / segmentation / restoration +``` + +A viewer reads the stamped `predict*` fields back as layers; `recordstream run` executes +the same document offline. + ## 📚 Documentation | Page | Covers | diff --git a/pyproject.toml b/pyproject.toml index 74b25b4..39bb0a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,7 @@ recordstream-flow = "recordstream.flow" # The queryable-metadata scan protocol + MetadataFilterSource view source recordstream-storage-query = "recordstream.storage.query" recordstream-ops-sink = "recordstream.ops.sink" +recordstream-ops-predict = "recordstream.ops.predict" recordstream-ops-numpy = "recordstream.ops.numpy" recordstream-ops-torch = "recordstream.ops.torch" recordstream-ops-target = "recordstream.ops.target" diff --git a/recordstream/ops/predict.py b/recordstream/ops/predict.py new file mode 100644 index 0000000..46bf16f --- /dev/null +++ b/recordstream/ops/predict.py @@ -0,0 +1,195 @@ +"""Run a task model over records — inference as ONE op in a pipeline. + +This is what lets a graph (or any config) CARRY its inference: a source, its +preprocessing, and a ``ModelPredict`` op wired to a project's model wrapper compose +into one runnable pipeline — a viewer executes it per record and reads the stamped +fields back as layers, and the same document runs offline with ``recordstream run``. +The op lives here (not in a viewer or a project package) because it is fully generic: +the model is any callable ``model(batch)``, duck-typed — no torch import of its own. +""" + +from typing import Any, Dict, Literal, get_args + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Boxes, Label + +logger = get_logger(__name__) + +#: What a model's output means — how it is stamped back onto the record. +PredictKind = Literal["classification", "detection", "segmentation", "restoration"] + + +def _to_numpy(value: Any) -> np.ndarray: + if hasattr(value, "detach"): + value = value.detach().cpu() + return np.asarray(value) # type: ignore[no-any-return] + + +def _batch(x: Any) -> Any: + return x[None] if hasattr(x, "shape") else np.asarray(x)[None] + + +def _first(out: Any) -> Any: + """The first element of a batched output (a sequence, a batched array, or a mapping of batched arrays).""" + if isinstance(out, (list, tuple)): + return out[0] + if isinstance(out, dict): + return {k: (v[0] if hasattr(v, "__len__") and len(v) else v) for k, v in out.items()} + if hasattr(out, "shape") and len(out.shape) > 0: + return out[0] + return out + + +def _fields(out: Any, *names: str) -> Any: + """``out[name]`` / ``out.name`` for the first name present.""" + for name in names: + if isinstance(out, dict) and name in out: + return out[name] + if hasattr(out, name): + return getattr(out, name) + raise ValueError(f"model output carries none of {names} (got {type(out).__name__})") + + +def _softmax(logits: np.ndarray) -> np.ndarray: + shifted = logits - logits.max(axis=-1, keepdims=True) + exp = np.exp(shifted) + return np.asarray(exp / exp.sum(axis=-1, keepdims=True), dtype=np.float64) + + +def _image_hw(value: Any) -> "tuple[int, int]": + """The (H, W) of an image field — HWC, HW, or CHW (1/3 channels first).""" + arr = _to_numpy(value) + if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3): + return int(arr.shape[1]), int(arr.shape[2]) + return int(arr.shape[0]), int(arr.shape[1]) + + +@configurable(category="op") +class ModelPredict: + """Run a task model on ONE record and stamp its prediction as a record field. + + The model is any callable ``model(batch)`` — typically a project's checkpointed + wrapper. Its real work (build the network, load the checkpoint) happens in its + ``solidify()``, called lazily on the first record, so constructing this op is free. + What gets stamped follows the field conventions a viewer reads back as layers: + + - ``classification``: ``output`` = the predicted class as a :class:`~recordstream.items.Label` + (output shape ``[1, C]`` probs/logits, or a mapping/object with ``probs``/``logits``); + - ``detection``: ``output`` = a :class:`~recordstream.items.Boxes` (per-image + ``boxes``/``scores``/``labels``, pixel xyxy); + - ``segmentation``: ``_mask`` = an int class mask ``[H, W]`` (argmax over + ``[C, H, W]`` logits when needed); + - ``restoration``: ``output`` = the restored image array (channels-first is moved last). + + Args: + model: The callable model (a checkpointed wrapper; solidified on first use). + kind: What the model's output means — ``classification`` / ``detection`` / + ``segmentation`` / ``restoration``. + key: Record field fed to the model. + output: Record field stamped with the prediction (``segmentation`` stamps + ``_mask``). Keep the ``predict`` prefix — that is what marks a + field as a prediction downstream. + device: Where a torch model runs (``cpu`` / ``cuda`` / ``mps``); ignored otherwise. + """ + + def __init__( + self, + model: Any = None, # any callable model wrapper — naming a real type would force a torch-shaped import + kind: PredictKind = "classification", + key: str = "image", + output: str = "predict", + device: str = "cpu", + ) -> None: + if kind not in get_args(PredictKind): + raise ValueError(f"ModelPredict kind must be one of {get_args(PredictKind)}, got {kind!r}") + self.model = model + self.kind = kind + self.key = key + self.output = output + self.device = device + self._ready: Any = None + + def _model(self) -> Any: + if self._ready is None: + model = self.model + if model is None: + raise ValueError("ModelPredict needs 'model' (a callable model wrapper)") + if hasattr(model, "solidify") and callable(model.solidify): + built = model.solidify() + model = built if built is not None else model + if hasattr(model, "eval") and callable(model.eval): + model.eval() + if self.device and hasattr(model, "to") and callable(model.to): + model.to(self.device) + self._ready = model + return self._ready + + def _call(self, batch: Any) -> Any: + model = self._model() + try: + import torch # noqa: F401 + + with torch.no_grad(): + if hasattr(batch, "to") and self.device: + batch = batch.to(self.device) + return model(batch) + except ImportError: + return model(batch) + + def __call__(self, record: Dict[str, Any]) -> Dict[str, Any]: + if self.key not in record: + raise ValueError(f"ModelPredict: record has no field {self.key!r} (fields: {sorted(record)})") + out = self._call(_batch(record[self.key])) + if self.kind == "classification": + return {**record, self.output: self._classification(out)} + if self.kind == "detection": + return {**record, self.output: self._detection(out, record[self.key])} + if self.kind == "segmentation": + return {**record, f"{self.output}_mask": self._segmentation(out)} + return {**record, self.output: self._restoration(out)} + + def _classification(self, out: Any) -> Label: + values = out + if isinstance(out, dict) or (not hasattr(out, "shape") and hasattr(out, "probs")): + values = _fields(out, "probs", "logits") + scores = _to_numpy(_first(values)).astype(np.float64).reshape(-1) + if scores.min() < 0.0 or scores.sum() > 1.0001: + scores = _softmax(scores) + return Label(int(np.argmax(scores))) + + def _detection(self, out: Any, image: Any) -> Boxes: + first = _first(out) + boxes = _to_numpy(_fields(first, "boxes")).reshape(-1, 4) + scores = _to_numpy(_fields(first, "scores")).reshape(-1) if _has(first, "scores") else np.ones(len(boxes)) + labels = ( + _to_numpy(_fields(first, "labels")).reshape(-1).astype(int) + if _has(first, "labels") + else np.zeros(len(boxes), dtype=int) + ) + return Boxes(boxes=boxes.tolist(), labels=labels.tolist(), scores=scores.tolist(), canvas=_image_hw(image)) + + def _segmentation(self, out: Any) -> np.ndarray: + values = out + if isinstance(out, dict) or (not hasattr(out, "shape") and (hasattr(out, "mask") or hasattr(out, "logits"))): + values = _fields(out, "mask", "logits", "probs") + arr = _to_numpy(_first(values)) + return np.argmax(arr, axis=0).astype(np.int64) if arr.ndim == 3 else arr.astype(np.int64) + + def _restoration(self, out: Any) -> np.ndarray: + values = out + if isinstance(out, dict) or (not hasattr(out, "shape") and hasattr(out, "image")): + values = _fields(out, "image") + arr = _to_numpy(_first(values)) + if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3): + arr = np.moveaxis(arr, 0, -1) + return arr + + +def _has(out: Any, name: str) -> bool: + return (isinstance(out, dict) and name in out) or hasattr(out, name) + + +__all__ = ["ModelPredict", "PredictKind"] diff --git a/tests/test_ops_predict.py b/tests/test_ops_predict.py new file mode 100644 index 0000000..5553ace --- /dev/null +++ b/tests/test_ops_predict.py @@ -0,0 +1,71 @@ +"""Inference as an op: ``ModelPredict`` stamps a model's output back onto the record. + +A pipeline that carries this op IS the predictions flow — a viewer executes it per +record and reads the stamped fields as layers; ``recordstream run`` executes the same +document offline. The model is duck-typed (any callable), so none of this needs torch. +""" + +from typing import Any + +import numpy as np +import pytest + +from recordstream.items import Boxes, Label +from recordstream.ops.predict import ModelPredict + + +class _Classifier: + """logits [1, 2] favouring class 1; counts solidify() calls (the lazy checkpoint load).""" + + def __init__(self) -> None: + self.solidified = 0 + self.calls = 0 + + def solidify(self) -> None: + self.solidified += 1 + + def __call__(self, batch: Any) -> np.ndarray: + self.calls += 1 + return np.array([[0.1, 2.0]]) + + +class TestModelPredict: + def test_classification_stamps_a_label_and_solidifies_once(self) -> None: + model = _Classifier() + op = ModelPredict(model=model, kind="classification") + record = {"image": np.zeros((8, 12)), "class": 0} + out = op(record) + assert isinstance(out["predict"], Label) and out["predict"].value == 1 + assert "predict" not in record # a new dict — the input record is never mutated + op(record) + assert model.solidified == 1 and model.calls == 2 # the checkpoint loads once, not per record + + def test_detection_stamps_boxes_with_the_images_canvas(self) -> None: + def model(batch: Any) -> Any: + return {"boxes": [[[1.0, 2.0, 5.0, 6.0]]], "scores": [[0.9]], "labels": [[1]]} + + out = ModelPredict(model=model, kind="detection")({"image": np.zeros((8, 12))}) + stamped = out["predict"] + assert isinstance(stamped, Boxes) + assert stamped.boxes == [[1.0, 2.0, 5.0, 6.0]] and stamped.scores == [0.9] and stamped.labels == [1] + assert stamped.canvas is not None and tuple(stamped.canvas) == (8, 12) + + def test_segmentation_argmaxes_logits_into_an_int_mask(self) -> None: + logits = np.zeros((1, 2, 4, 6)) + logits[0, 1, :2] = 5.0 # top rows are class 1 + out = ModelPredict(model=lambda b: logits, kind="segmentation")({"image": np.zeros((4, 6))}) + mask = out["predict_mask"] + assert mask.shape == (4, 6) and mask.dtype == np.int64 + assert mask[0, 0] == 1 and mask[3, 0] == 0 + + def test_restoration_moves_channels_last(self) -> None: + out = ModelPredict(model=lambda b: np.ones((1, 3, 4, 6)), kind="restoration")({"image": np.zeros((4, 6))}) + assert out["predict"].shape == (4, 6, 3) + + def test_bad_kind_and_missing_pieces_are_located_value_errors(self) -> None: + with pytest.raises(ValueError, match="kind"): + ModelPredict(kind="nope") # type: ignore[arg-type] + with pytest.raises(ValueError, match="model"): + ModelPredict(kind="classification")({"image": np.zeros((2, 2))}) + with pytest.raises(ValueError, match="no field 'image'"): + ModelPredict(model=_Classifier())({"other": 1}) From ea6561a66c33d6e6cd9b04ab7908edf030077014 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 16:22:36 +0200 Subject: [PATCH 095/102] =?UTF-8?q?chore(ci):=20loggair=20resolves=20from?= =?UTF-8?q?=20PyPI=20=E2=80=94=20drop=20the=20GitHub=20pre-install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerated by 'aisland source set loggair pypi' (loggair 0.2.0 released): the internal-dependency pre-step installing loggair from git@main is gone; .[dev] now resolves the published wheel. --- .github/workflows/ci.yml | 4 ---- Jenkinsfile | 1 - 2 files changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7132c8..e63c3bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" @@ -59,7 +58,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" @@ -94,7 +92,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" @@ -133,7 +130,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/loggair.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" diff --git a/Jenkinsfile b/Jenkinsfile index 02a12eb..c6c71e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,7 +33,6 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" From ad590bb4687d316e6e0176515802acac0451e096 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 17:45:53 +0200 Subject: [PATCH 096/102] =?UTF-8?q?chore(ci):=20confluid=20resolves=20from?= =?UTF-8?q?=20PyPI=20=E2=80=94=20drop=20the=20GitHub=20pre-install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerated by 'aisland source set confluid pypi' (confluid 0.3.0 released): the internal-dependency pre-step installing confluid from git@main is gone; .[dev] now resolves the published wheel. --- .github/workflows/ci.yml | 4 ---- Jenkinsfile | 1 - Jenkinsfile.local | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e63c3bc..c442574 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" - name: Run Isort @@ -58,7 +57,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" - name: Run Tests @@ -92,7 +90,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" - name: Run Examples @@ -130,7 +127,6 @@ jobs: # Internal Gearlux dependencies — installed FIRST with --no-deps so # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" # Notebook-only dependencies live in the optional `[notebook]` extra diff --git a/Jenkinsfile b/Jenkinsfile index c6c71e3..94e3d03 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,7 +33,6 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 41995f0..1bb8348 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -42,8 +42,8 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/confluid/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/liquifai/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live From d37964fbc1e9f7d0a0ac49e89daa3ffeb9df8849 Mon Sep 17 00:00:00 2001 From: gearlux Date: Mon, 24 Aug 2026 19:02:32 +0200 Subject: [PATCH 097/102] =?UTF-8?q?chore(ci):=20liquifai=20resolves=20from?= =?UTF-8?q?=20PyPI=20=E2=80=94=20drop=20the=20GitHub=20pre-install?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerated by 'aisland source set liquifai pypi' (liquifai 0.2.0 released). --- .github/workflows/ci.yml | 12 ------------ Jenkinsfile | 4 ---- Jenkinsfile.local | 2 +- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c442574..4c4522a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" - name: Run Isort run: isort --check-only . @@ -55,9 +52,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" - name: Run Tests run: | @@ -88,9 +82,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" - name: Run Examples run: | @@ -125,9 +116,6 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/liquifai.git@main uv pip install --system -e ".[dev,torch,keras]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. diff --git a/Jenkinsfile b/Jenkinsfile index 94e3d03..613078c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,10 +30,6 @@ pipeline { sh "${VENV_BIN}/pip install --upgrade pip uv" echo 'Installing Dependencies...' - // Internal Gearlux dependencies — installed FIRST with --no-deps - // so .[dev] below finds them pre-satisfied instead of hitting PyPI - // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main" sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships diff --git a/Jenkinsfile.local b/Jenkinsfile.local index 1bb8348..52a0919 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -43,8 +43,8 @@ pipeline { // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). sh "if [ -f '${env.WORKSPACE_ROOT}/confluid/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" - sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "if [ -f '${env.WORKSPACE_ROOT}/liquifai/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships From c972a3877fb5d4c46a7089beed51d43d1ad16ad0 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 25 Aug 2026 10:20:26 +0200 Subject: [PATCH 098/102] =?UTF-8?q?chore:=20prepare=20the=200.1.0a1=20rele?= =?UTF-8?q?ase=20=E2=80=94=20landing-page=20metadata,=20real=20floors,=20a?= =?UTF-8?q?bsolute=20README=20links?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package built fine without any of this, which is why all of it was missing at once: `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are optional to the build but ARE the PyPI project page. Add them, plus a LICENSE and a CHANGELOG, and set the version to the pre-release it actually is (0.1.0a1, matching the `Development Status :: 3 - Alpha` classifier). - Floors now state what the suite runs against: confluid>=0.3.0, loggair>=0.2.0, liquifai>=0.2.0. The old `confluid>=0.1.0` described a combination nobody could install — liquifai itself requires confluid>=0.2.0. - README links to repo files by absolute GitHub URL: PyPI resolves a relative link against pypi.org, so `docs/storage.md` 404s there while working on GitHub. - tests/test_packaging.py pins the metadata, the classifier/version agreement, the declared readme+license files, and that no requirement is a direct URL (PyPI refuses those); test_docs_links.py gains the absolute-link rules. - setuptools floor raised to >=77 for PEP 639 `license`/`license-files`. - Two follow-ups recorded in TASKS.md (torch-less discovery warning, the `materialize_runnable` shim liquifai 0.2.0 makes removable). --- AGENTS.md | 4 ++ CHANGELOG.md | 65 +++++++++++++++++++++++++++ LICENSE | 21 +++++++++ README.md | 56 ++++++++++++----------- TASKS.md | 2 + docs/architecture.md | 3 +- pyproject.toml | 38 +++++++++++++--- tests/test_docs_links.py | 46 +++++++++++++++++-- tests/test_packaging.py | 97 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 293 insertions(+), 39 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 tests/test_packaging.py diff --git a/AGENTS.md b/AGENTS.md index d694d62..9d57d2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,10 @@ Core engine feature-complete on the **record model**; the full surface (items · Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). - **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Boxes`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Boxes`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) +- **The README Is The PyPI LANDING PAGE — Repo Links Are ABSOLUTE (2026-08-24):** `pyproject.toml` sets `readme = "README.md"`, so the README's rendered form IS the project page, and PyPI resolves a relative link against `pypi.org`, not against the repository — `docs/storage.md` becomes `pypi.org/docs/storage.md` and 404s. The link works perfectly on GitHub, so nothing in the repo catches it, which is why it is a test rather than a review habit: the README links to repo files as `https://github.com/Gearlux/recordstream/blob/main/` (26 links converted on 2026-08-24), while the `docs/*.md` pages are read on GitHub only and KEEP their relative links to each other. Spelling a README link absolutely also takes it out of reach of the existing rename check, so a second rule verifies each absolute self-link against the local tree — without it, converting the links would have traded a PyPI 404 for a GitHub one. Pins: `tests/test_docs_links.py::test_the_readme_links_to_repo_files_by_absolute_url` / `::test_readme_links_into_this_repo_name_a_file_that_exists`. +- **A Dependency FLOOR States What Is TESTED, Not The Oldest Release That Once Worked (2026-08-24):** the floors were `confluid>=0.1.0` / `loggair>=0.1.0` / `liquifai>=0.1.0` long after the workspace had moved on, and one of them described a combination NOBODY can install — `uv pip install recordstream confluid==0.1.0` fails with *"Because liquifai<=0.1.0 depends on confluid>=0.2.0 and you require confluid==0.1.0 … your requirements are unsatisfiable"*. A stale floor is not harmless: it is a claim about what this package supports that no test covers and no resolver honours (a lowest-direct resolve lands on 0.3.0 / 0.2.0 / 0.2.0 anyway). Raise a floor to the version the suite actually runs against when a release makes that possible, and state WHY in a comment when the reason is not the obvious one. **Below 1.0 a pre-release segment is load-bearing in the OTHER direction too:** under PEP 440 a plain `>=0.1.0` EXCLUDES every pre-release of 0.1.0, so a consumer depending on this package while it ships `0.1.0a1` must write `recordstream>=0.1.0a1` or resolve to nothing. Pin: `tests/test_packaging.py`. +- **Release Metadata Is Part Of The Package, Not Paperwork (2026-08-24):** `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are each optional to the BUILD — the wheel builds byte-identically without them — which is exactly why all four were missing at once and why the gap is invisible until the page is published (`twine check` reports only `long_description missing`). The `Development Status` classifier must agree with the version: an `aN`/`bN` version is `3 - Alpha`, a final one `4 - Beta`; they drift in opposite directions because the version moves every release and the classifier is written once. Pins: `tests/test_packaging.py::test_the_landing_page_metadata_is_present` / `::test_the_development_status_classifier_matches_the_version` / `::test_the_declared_readme_and_license_files_exist` / `::test_no_requirement_is_a_direct_url` (PyPI refuses any `Requires-Dist` carrying a URL — `warehouse/forklift/metadata.py`, "Can't have direct dependency"). + ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. - **Multiprocess Safety:** Parallel pipelines MUST use the `spawn` context. Verify pickle-safety of all operations. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..233b7e9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + +All notable changes to this project are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the versioning is +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0a1] — unreleased + +First public pre-release. The surface it ships: + +### The record model +- A record is a plain `dict` of typed values — `Image`, `Mask`, `Boxes`, `Label`, + `MultiLabel` — each owning its own metadata. Key names carry meaning; there is no + wrapper container and no role tags. +- Array-backed items subclass `NDArrayItem`, so numpy operations preserve their declared + attributes. Structured items are dataclass wrappers. `register_item` opens the set to + any package. +- `recordstream.io` is the serialization codec, so an externally registered item type + round-trips through storage with no backend change. + +### The engine +- One step-graph engine behind two facades: `Stream` / `JointStream` for the dataset + surface (`__len__` / `__getitem__` / `.batch` / `.parallel` / `.project`), and + `FlowGraph` for a `flow:` document of named steps with `from:` / `merge_from:` / + `bind:` edges. An `ops:` list is the same engine's linear spelling. +- Ops dispatch on value type: a `Transform` samples its parameters once per record and + applies a per-type kernel to every value it handles. +- Bare albumentations and torchvision `transforms.v2` transforms run as-is through the + op-family dispatch — one call is one joint draw across image, mask and boxes. The + family registry (`register_op_family`) is open to other libraries. +- 1→N expanding ops fork the remaining subgraph, in every route including spawn-parallel. +- Multiprocessing uses the `spawn` context; `ensure_materialized` and the OpenCV thread + guard cover the two fork hazards a forked loader hits. + +### Storage +- HDF5, Zarr and Directory sinks, each with a matching source, over the `typedrecord-v1` + layout. +- Metadata is queryable without loading arrays: the `SupportsMetadataScan` protocol plus + `MetadataFilterSource`. + +### Sources and projection +- `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`. +- Every source names the data it reads through `dataset_uri` / `dataset_url`, and a view + propagates the handle verbatim. +- Key projection (`project`, `iter_key`, `first_value`, `num_classes`, `class_names`), + the fittable `LabelMap`, and class-balance statistics. + +### Running +- `recordstream run ` runs any Confluid-wired runnable — a trainer, an + evaluator, a dataset processor, a workflow. The `@entrypoint` markers are the dispatch + table for a runnable that drives several tasks off one `task` knob. +- Workflow combinators (`Sequence` / `Conditional` / `Switch`) plus predicates, so a + resume-safe multi-stage pipeline is one document. + +### Batching +- `collate_records` (the `"record"` default) and `collate_list`, differing in one + decision — whether array payloads stack — plus the read-back half (`batch_values`, + `batch_boxes`, `batch_tensor`, `batch_metadata`, `multi_hot`). +- `recordstream.keras.RecordSequence` is the Keras `PyDataset` half, since Keras 3 has no + `DataLoader` to do the row scheduling. + +### Install shape +- The core engine is numpy and installs no ML framework. `[torch]` adds the pieces that + genuinely produce tensors; `[keras]` adds the `PyDataset` adapter and names no compute + engine; `[vision]` enables the bare torchvision transform family. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..70f8163 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Gert Behiels + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index c8474a9..59d7bb2 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,18 @@ Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `RecordS ## 🚀 Key Features -- **A record is a plain dict:** the [record model](docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Boxes`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. -- **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. +- **A record is a plain dict:** the [record model](https://github.com/Gearlux/recordstream/blob/main/docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Boxes`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. +- **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](https://github.com/Gearlux/recordstream/blob/main/docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. - **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. -- **Graph pipelines:** readable [`flow:` documents](docs/graph.md) of named steps — `from:` forks, `merge_from:` merges, `bind:` feeds one step's value into another's parameter. An `ops:` list is the same engine's linear spelling; both parse to one step graph. -- **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. -- **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](docs/storage.md#queryable-metadata-recordstreamstoragequery) — filter stored datasets without loading a single array. -- **Passive Introspection:** ops declare the value types they [handle / consume / produce](docs/record-model.md) and are discoverable by category for visual editors and schema generators. +- **Graph pipelines:** readable [`flow:` documents](https://github.com/Gearlux/recordstream/blob/main/docs/graph.md) of named steps — `from:` forks, `merge_from:` merges, `bind:` feeds one step's value into another's parameter. An `ops:` list is the same engine's linear spelling; both parse to one step graph. +- **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. +- **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](https://github.com/Gearlux/recordstream/blob/main/docs/storage.md#queryable-metadata-recordstreamstoragequery) — filter stored datasets without loading a single array. +- **Passive Introspection:** ops declare the value types they [handle / consume / produce](https://github.com/Gearlux/recordstream/blob/main/docs/record-model.md) and are discoverable by category for visual editors and schema generators. - **100% Reproducibility:** Entire pipelines are serializable via **Confluid** manifests. ## 🛠 Quick Start -One pipeline mixing a **bare albumentations Compose** (image + mask + boxes move together in one draw), a **bare torchvision v2 transform**, and a **native op** — no wrappers (mirrors [`examples/record_pipeline.py`](examples/record_pipeline.py)): +One pipeline mixing a **bare albumentations Compose** (image + mask + boxes move together in one draw), a **bare torchvision v2 transform**, and a **native op** — no wrappers (mirrors [`examples/record_pipeline.py`](https://github.com/Gearlux/recordstream/blob/main/examples/record_pipeline.py)): ```python import albumentations as A @@ -90,7 +90,7 @@ recordstream run pipeline.yaml --enabled false # broadcast: every Ena Inner ops are not materialized until the wrapper first fires, so gating an expensive chain with `enabled: false` costs nothing at startup. In Python the same wrapper is one call — `Enable(ops=[...], name="visualize", enabled=False)` — which is what lets a visual editor or a -generated tool schema set the toggle too (see [docs/architecture.md](docs/architecture.md#6-every-knob-is-a-declared-parameter--the-enable-toggle-2026-07-27)). +generated tool schema set the toggle too (see [docs/architecture.md](https://github.com/Gearlux/recordstream/blob/main/docs/architecture.md#6-every-knob-is-a-declared-parameter--the-enable-toggle-2026-07-27)). ### Inference as an op (`ModelPredict`) @@ -117,19 +117,19 @@ the same document offline. | Page | Covers | |---|---| -| [docs/record-model.md](docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | -| [docs/kinds.md](docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), the Keras `RecordSequence` adapter, 1→N expanding ops | -| [docs/graph.md](docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | -| [docs/sources.md](docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing, dataset identity (`dataset_uri` / `dataset_url`) | -| [docs/storage.md](docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | -| [docs/projection.md](docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), one-peek `first_value`, `num_classes`, the fittable `LabelMap`, class-balance weights | -| [docs/predictions.md](docs/predictions.md) | The model boundary: prediction-output contracts (`ClassificationOutput` & co), `ensure_record_dataset`, the `PredictionsSink` protocol + the classification sink | -| [docs/image.md](docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), mask→class-id conversion (`ConvertToMask`), array introspection helpers | -| [docs/configure.md](docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | -| [docs/runnable.md](docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers + `run_entrypoint` dispatch with a worked example, `TorchRunner` / `ProgressReporting` | -| [docs/workflow.md](docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | -| [docs/augmentation.md](docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | -| [docs/architecture.md](docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | +| [docs/record-model.md](https://github.com/Gearlux/recordstream/blob/main/docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | +| [docs/kinds.md](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), the Keras `RecordSequence` adapter, 1→N expanding ops | +| [docs/graph.md](https://github.com/Gearlux/recordstream/blob/main/docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | +| [docs/sources.md](https://github.com/Gearlux/recordstream/blob/main/docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing, dataset identity (`dataset_uri` / `dataset_url`) | +| [docs/storage.md](https://github.com/Gearlux/recordstream/blob/main/docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | +| [docs/projection.md](https://github.com/Gearlux/recordstream/blob/main/docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), one-peek `first_value`, `num_classes`, the fittable `LabelMap`, class-balance weights | +| [docs/predictions.md](https://github.com/Gearlux/recordstream/blob/main/docs/predictions.md) | The model boundary: prediction-output contracts (`ClassificationOutput` & co), `ensure_record_dataset`, the `PredictionsSink` protocol + the classification sink | +| [docs/image.md](https://github.com/Gearlux/recordstream/blob/main/docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), mask→class-id conversion (`ConvertToMask`), array introspection helpers | +| [docs/configure.md](https://github.com/Gearlux/recordstream/blob/main/docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | +| [docs/runnable.md](https://github.com/Gearlux/recordstream/blob/main/docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers + `run_entrypoint` dispatch with a worked example, `TorchRunner` / `ProgressReporting` | +| [docs/workflow.md](https://github.com/Gearlux/recordstream/blob/main/docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | +| [docs/augmentation.md](https://github.com/Gearlux/recordstream/blob/main/docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | +| [docs/architecture.md](https://github.com/Gearlux/recordstream/blob/main/docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | ## 🧭 Scope: a modality-neutral engine @@ -142,16 +142,18 @@ RecordStream deliberately contains **no domain-specific code** — every op, sou RecordStream is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: -- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability, and [names the dataset it reads](docs/sources.md#identifying-a-dataset) so a run record can point at it (see [docs/sources.md](docs/sources.md)). +- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability, and [names the dataset it reads](https://github.com/Gearlux/recordstream/blob/main/docs/sources.md#identifying-a-dataset) so a run record can point at it (see [docs/sources.md](https://github.com/Gearlux/recordstream/blob/main/docs/sources.md)). - **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node — including bare library transforms — every run reproducible. -- **PyTorch**: `Stream` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](docs/kinds.md#batching--collate_records--the-collate-registry-recordstreamcollate) (`collate_records` is the default). -- **Keras 3**: no `DataLoader` exists to do the batching, so [`RecordSequence`](docs/kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you) is the `keras.utils.PyDataset` half — row order, slicing, per-epoch reshuffle, `collate_records` — and a `transform` callable supplies the batch shape, exactly as `collate_fn` does for torch. -- **Augmentation libraries**: [albumentations](https://albumentations.ai) and torchvision `transforms.v2` transforms run **as-is** in any ops list — the engine speaks each library's native convention (kwarg vocabulary vs dict walk), so there is nothing to wrap (see [docs/augmentation.md](docs/augmentation.md)). +- **PyTorch**: `Stream` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md#batching--collate_records--the-collate-registry-recordstreamcollate) (`collate_records` is the default). +- **Keras 3**: no `DataLoader` exists to do the batching, so [`RecordSequence`](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you) is the `keras.utils.PyDataset` half — row order, slicing, per-epoch reshuffle, `collate_records` — and a `transform` callable supplies the batch shape, exactly as `collate_fn` does for torch. +- **Augmentation libraries**: [albumentations](https://albumentations.ai) and torchvision `transforms.v2` transforms run **as-is** in any ops list — the engine speaks each library's native convention (kwarg vocabulary vs dict walk), so there is nothing to wrap (see [docs/augmentation.md](https://github.com/Gearlux/recordstream/blob/main/docs/augmentation.md)). ## 🔧 Installation +RecordStream is on PyPI as a pre-release, so `pip` needs `--pre` to see it: + ```bash -pip install git+https://github.com/Gearlux/recordstream.git@main +pip install --pre recordstream ``` The core engine is **numpy**, and installs no ML framework. A framework arrives only with the extra @@ -163,7 +165,7 @@ that needs it: | `keras` | `recordstream.keras` — the `RecordSequence` `PyDataset` adapter and the `KERAS_BACKEND` ordering. Keras 3 is an API, so this names no compute engine; it runs on whichever of torch / TensorFlow / JAX you have | ```bash -pip install "recordstream[torch] @ git+https://github.com/Gearlux/recordstream.git@main" +pip install --pre "recordstream[torch]" ``` Everything else works without either. A `Stream` is map-style (`__len__`/`__getitem__`), so a diff --git a/TASKS.md b/TASKS.md index 9d944b8..7ba91f2 100644 --- a/TASKS.md +++ b/TASKS.md @@ -3,6 +3,8 @@ Open work for this project. Cross-cutting / multi-project initiatives live in the workspace root `TASKS.md`. Completed items are not archived here — git history is the record. +- [ ] **A torch-less install WARNS on every discovery scan** @low — the documented default install (`pip install --pre recordstream`, no extra) leaves `recordstream.ops.torch` unimportable, so `confluid.registry.load_configurables` logs `WARNING … entry point 'recordstream-ops-torch' … failed to import: No module named 'torch'` every time anything scans the registry. Measured 2026-08-24 in a clean venv. Discovery itself degrades correctly — it skips the module and continues — so this is noise, not breakage, but it fires on the SUPPORTED configuration and the workspace's own "Diagnostic Log Levels" mandate reserves `warning` for conditions an operator must act on. Two ways out, and they belong to different packages: demote the skip to `debug` in confluid (it cannot tell a missing optional extra from a real defect), or make the entry point importable without torch here. Decide which layer owns it before editing either. @2026-08-24 +- [ ] **`materialize_runnable` can shrink away now that liquifai 0.2.0 is on PyPI** @low @refactor — `recordstream/cli.py:33` exists because liquifai's DI dropped every top-level config key for an `Any`-annotated command parameter; liquifai 0.1.1 fixed that at its own layer (`di.deep_flow` now takes the document) and 0.2.0 is published, so the two `flow_mode="manual"` commands can go back to `flow_mode="auto"`. The floor was raised to `liquifai>=0.2.0` on 2026-08-24, which removes the blocker the docstring names. It is NOT urgent: building against the document is correct under either liquifai, so keeping the helper costs only the indirection. A consumer shipping its own CLI calls the same helper, so revert both sides together. @2026-08-24 - [ ] **`Stream.source`'s annotation is narrower than its documented contract** @low @refactor — the docstring says "any iterable **or indexable** dataset (duck-typed)" but the annotation is `Optional[Iterable[Any]]`. A torch map-style `Dataset` iterates at runtime via the legacy `__getitem__` protocol, which mypy does not model, so passing one is statically invalid though perfectly correct — `recordstream.ensure_record_dataset` (moved in from a consumer 2026-07-29) hits exactly this and carries a documented `cast`. Attempted 2026-07-29: widening to `Union[Iterable[Any], Indexable]` with a `Protocol` BREAKS `confluid.to_pydantic` for every Stream (`SchemaError: Error building "model" validator ... Field "source"` — pydantic cannot schema a bare Protocol), so it needs either a pydantic-friendly spelling or an entry in confluid's opaque-type coercion (`_is_opaque_type` -> `Any`, the same escape hatch the torchvision `Callable`/enum landmines use). Not worth a schema regression for a type nicety; revisit if a second consumer hits the cast. - [ ] **`to_tensor`'s `normalize` heuristic silently corrupts already-standardized floats** @bug — `recordstream/ops/torch.py::to_tensor` does `elif normalize and tensor.max() > 1.0: tensor = tensor / 255.0`, i.e. it infers "a float whose max exceeds 1 must be 0-255 pixels". An ImageNet-standardized array (range ~[-2.12, 2.64]) satisfies that test, so chaining a `Normalize` op before `ToTensor` divides the standardized values by 255 and squashes them to ~[-0.01, 0.01] — no error, no warning, just a model that learns nothing. Hit for real 2026-07-29 while wiring a consumer's example config; the caller's fix is `ToTensor(normalize=false)`, which is correct but only discoverable by inspecting the tensor. Options: gate the rescale on an INTEGER dtype only (what the docstring already claims — "scale integer pixel inputs"), or keep the heuristic and warn when it fires on a float input. Changing it is a behaviour change for anyone relying on the 0-255-float path, so it needs a decision rather than a quiet edit. - [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor diff --git a/docs/architecture.md b/docs/architecture.md index 82809a8..c7fcdf2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -868,7 +868,8 @@ def predict_step(self, batch, batch_idx): ### Context `recordstream` declared `torch` as a hard dependency, so `import recordstream` imported ~2GB of -PyTorch — and matrainer inherited it transitively, declaring no torch of its own. That was fine +PyTorch — and every package built on it inherited that transitively, declaring no torch of its +own. That was fine while every consumer was a Lightning trainer. It stopped being fine when a second training engine landed: a Keras-on-TensorFlow install, or a plain-numpy dataset-conversion job, paid for a framework it never called. diff --git a/pyproject.toml b/pyproject.toml index 39bb0a3..d142140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,29 @@ [project] name = "recordstream" -version = "0.1.0" +version = "0.1.0a1" description = "Clean, functional data pipelines for ML research and production." -authors = [{ name = "Taidal", email = "info@gearlux.ai" }] +readme = "README.md" +authors = [{ name = "Gert Behiels" }] +license = "MIT" +license-files = ["LICENSE"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +# Floors are what this package is actually TESTED against, not the oldest release that +# once worked: `confluid==0.1.0` is provably unsatisfiable here (liquifai requires +# confluid>=0.2.0), so the old `>=0.1.0` described a combination nobody could install. dependencies = [ - "confluid>=0.1.0", - "loggair>=0.1.0", + "confluid>=0.3.0", + "loggair>=0.2.0", "numpy", "Pillow", "h5py", @@ -19,10 +37,16 @@ dependencies = [ # liquifai powers the generic `recordstream run ` CLI (recordstream.cli) that # runs any Confluid-wired runnable; it also brings `rich` (used by DatasetProcessor's # optional console progress bar). liquifai depends only on confluid/loggair/rich — no cycle. - "liquifai>=0.1.0", + "liquifai>=0.2.0", ] requires-python = ">=3.12" +[project.urls] +Homepage = "https://github.com/Gearlux/recordstream" +Repository = "https://github.com/Gearlux/recordstream.git" +Issues = "https://github.com/Gearlux/recordstream/issues" +Changelog = "https://github.com/Gearlux/recordstream/blob/main/CHANGELOG.md" + [project.optional-dependencies] # The torch-shaped surface, NOT the engine: `recordstream.ops.torch` (ToTensor), # `batch_tensor`, and the `outputs` builders. The core is numpy — `Stream` is a plain @@ -61,7 +85,7 @@ vision = [ ] [build-system] -requires = ["setuptools>=64", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" # CONFLUID-CONFIGURABLE DISCOVERY @@ -80,7 +104,7 @@ recordstream-sources = "recordstream.sources" recordstream-ops-parallel = "recordstream.ops.parallel" recordstream-ops-enable = "recordstream.ops.enable" recordstream-ops-random-apply = "recordstream.ops.random_apply" -# ConfigureOp (per-record parameter injection — the helios Configure pattern); entry-point +# ConfigureOp (per-record parameter injection — derive an op's parameter from the record); entry-point # changes need an editable reinstall before StreamStudio/navigaitor discovery sees the module. recordstream-ops-configure = "recordstream.ops.configure" recordstream-ops-formula = "recordstream.ops.formula" diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index 4757b23..897f90b 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -12,10 +12,11 @@ * an `#anchor` names a heading that exists in that file, * every docs page is reachable from the README index. -This project's README links to docs RELATIVELY, which is correct for a project not -published to PyPI — the absolute-URL rule exists because a PyPI landing page cannot -resolve a relative link. Those relative links are still covered by the two rules -above, since the README is in the scanned set. +The README is the PyPI landing page (``readme = "README.md"``), and PyPI resolves a +relative link against ``pypi.org``, not against the repository — so two further rules +apply to it alone: it links to repo files by ABSOLUTE GitHub URL, and each such URL +names a file that exists here. The ``docs/*.md`` pages are read on GitHub only and +keep their relative links. Fenced code blocks are excluded from the scan — see :func:`_strip_code`. @@ -153,3 +154,40 @@ def test_the_slug_rule_matches_githubs() -> None: assert _slug("Registering a class you don't own") == "registering-a-class-you-dont-own" assert _slug("`flow()` finishes the object") == "flow-finishes-the-object" assert _slug("Bare, addressed, glob — the scoping model") == "bare-addressed-glob--the-scoping-model" + + +#: `https://github.com/Gearlux/recordstream/blob/main/` — a link from the README +#: back into this repository. Any `#anchor` is captured separately so it can be dropped. +_SELF_REPO_LINK = re.compile(r"https://github\.com/Gearlux/recordstream/blob/main/([^)\s#]+)") + + +def test_the_readme_links_to_repo_files_by_absolute_url() -> None: + """The README carries no relative link to a repo file. + + It is the PyPI landing page, and PyPI resolves `docs/storage.md` against + `pypi.org/docs/storage.md`, which does not exist. The reader gets a 404 from a + link that works perfectly on GitHub, so nothing in the repo can catch it — which + is why this rule is a test rather than a review habit. + + Same-page anchors (`#installation`) are fine: they resolve on the rendered page. + """ + relative = [t for t in _links(_README) if not t.startswith("#")] + + assert not relative, ( + f"README links to repo files relatively; PyPI cannot resolve these: {relative}. " + f"Use https://github.com/Gearlux/recordstream/blob/main/." + ) + + +def test_readme_links_into_this_repo_name_a_file_that_exists() -> None: + """An absolute self-link still has to point at something. + + Spelling the link absolutely takes it out of reach of + `test_relative_links_name_a_file_that_exists`, so without this rule the rename + check silently stops covering the README — trading a PyPI 404 for a GitHub one. + """ + missing = [ + target for target in _SELF_REPO_LINK.findall(_strip_code(_README.read_text())) if not (_REPO / target).exists() + ] + + assert not missing, f"README links to repo files that do not exist: {missing}" diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..13317a9 --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,97 @@ +"""The distribution metadata a PyPI upload actually requires. + +Every rule here was written against a measured failure of this project's own build, +not from a checklist. They are cheap (they read ``pyproject.toml``; nothing is built) +and they cover the half of "does it ship?" that the test suite, the linters and the +examples all pass straight through: + +* a dependency written as a direct URL makes PyPI *reject the upload* — the file + is uploaded, parsed, and refused, so the failure arrives after everything green; +* a ``readme``/``license-files`` entry naming a file that is not there fails at + build time on a clean checkout but not in an editable install; +* missing classifiers/URLs cost nothing at build time and produce a landing page + with no license, no home, and no issue tracker. + +The sibling ``test_docs_links.py`` covers the README's *contents* once it becomes +that landing page. +""" + +import tomllib +from pathlib import Path +from typing import Any, Dict, List, cast + +_REPO = Path(__file__).resolve().parent.parent +_PYPROJECT = _REPO / "pyproject.toml" + + +def _project() -> Dict[str, Any]: + with _PYPROJECT.open("rb") as fh: + return cast(Dict[str, Any], tomllib.load(fh)["project"]) + + +def _every_requirement() -> List[str]: + """Core dependencies plus every extra's — PyPI validates all of them alike.""" + project = _project() + requirements = list(project.get("dependencies", [])) + for extra in project.get("optional-dependencies", {}).values(): + requirements.extend(extra) + return requirements + + +def test_no_requirement_is_a_direct_url() -> None: + """A ``name @ git+https://...`` requirement is refused by PyPI at upload time. + + Warehouse validates every ``Requires-Dist`` and rejects any whose requirement + carries a URL — ``warehouse/forklift/metadata.py``, "Can't have direct + dependency: {req}". It applies to extras exactly as to core dependencies, which + is the trap: an extra nobody installs still blocks the whole upload. + + An unpublished dependency therefore cannot be reached by an extra at all. Name + it plainly (``foo``) and document how to obtain it, or drop the extra. + """ + direct = [req for req in _every_requirement() if "@" in req and "://" in req.split("@", 1)[1]] + + assert not direct, f"PyPI rejects direct-URL requirements: {direct}" + + +def test_the_declared_readme_and_license_files_exist() -> None: + """``readme`` / ``license-files`` name files that are actually in the tree. + + Both are read at BUILD time, so an editable install never notices a missing one; + the failure surfaces on the clean checkout that builds the release artifact. + """ + project = _project() + declared = [project["readme"], *project.get("license-files", [])] + missing = [name for name in declared if not (_REPO / name).exists()] + + assert not missing, f"pyproject names files that do not exist: {missing}" + + +def test_the_landing_page_metadata_is_present() -> None: + """Description, license, classifiers and URLs — the PyPI page's whole frame. + + Each is optional to the build and free to omit, which is why all four went + missing at once: the wheel builds identically without them and the gap is only + visible on the published page. + """ + project = _project() + missing = [key for key in ("description", "readme", "license", "classifiers", "urls") if not project.get(key)] + + assert not missing, f"release metadata missing from pyproject: {missing}" + + +def test_the_development_status_classifier_matches_the_version() -> None: + """An ``aN``/``bN`` version says pre-release; the classifier must say so too. + + These drift in opposite directions — the version moves every release, the + classifier is written once and forgotten — leaving a "Production/Stable" badge + on an alpha. + """ + project = _project() + status = [c for c in project["classifiers"] if c.startswith("Development Status ::")] + assert len(status) == 1, f"expected exactly one Development Status classifier, got {status}" + + is_prerelease = "a" in project["version"] or "b" in project["version"] or "rc" in project["version"] + expected = "3 - Alpha" if is_prerelease else "4 - Beta" + + assert expected in status[0], f"version {project['version']} does not match classifier {status[0]!r}" From b216ca704635a67a364989b819da4c100223d0f1 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 25 Aug 2026 10:42:23 +0200 Subject: [PATCH 099/102] chore: tag-triggered PyPI release for 0.1.0a1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the hand-written release pipeline this project was missing: a `v*` tag now builds the sdist + wheel, guards that the tag matches the pyproject version, runs `twine check --strict`, smoke-tests the built wheel in a clean venv from /tmp (the repo directory shares its name with the package, so a cwd-local import would silently test the source tree), and publishes through PyPI Trusted Publishing — no stored token. Also completes the distribution itself: - MANIFEST.in ships CHANGELOG.md in the sdist. setuptools builds an sdist from what is declared — `readme` pulls README.md, `license-files` pulls LICENSE, `package-data` pulls the package — and nothing references a changelog, so it was silently absent. - build + twine join the dev extra, matching the other published projects. - The changelog entry is dated. --- .github/workflows/release.yml | 101 ++++++++++++++++++++++++++++++++++ AGENTS.md | 2 +- CHANGELOG.md | 2 +- MANIFEST.in | 9 +++ pyproject.toml | 4 ++ tests/test_packaging.py | 29 ++++++++++ 6 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 MANIFEST.in diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f51c172 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,101 @@ +# ========================================================================= +# HAND-WRITTEN workflow — NOT generated by `aisland jenkins scaffold`. +# (ci.yml / Jenkinsfile are generated; this release pipeline is separate so +# regeneration never clobbers it.) +# +# Publishes the `recordstream` distribution to PyPI when a version tag is pushed: +# +# git tag v0.1.0a1 && git push origin v0.1.0a1 +# +# Uses PyPI Trusted Publishing (OIDC, no stored API token). One-time setup on +# https://pypi.org/manage/account/publishing/ — add a "pending publisher": +# PyPI project name: recordstream +# Owner: Gearlux +# Repository: recordstream +# Workflow name: release.yml +# Environment: pypi +# +# Publish ORDER: loggair -> confluid -> liquifai are all on PyPI already, and +# this package declares floors on all three, so it publishes after them. The +# visual-editor package declares `recordstream>=0.1.0a1`, so it publishes after +# THIS one. +# ========================================================================= +name: Release to PyPI + +on: + push: + tags: ["v*"] + +jobs: + build: + name: Build & verify distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install packaging toolchain + run: python -m pip install build twine + - name: Guard - tag matches pyproject version + run: | + PKG_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + TAG_VERSION="${GITHUB_REF_NAME#v}" + if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then + echo "::error::Tag v$TAG_VERSION does not match pyproject version $PKG_VERSION" + exit 1 + fi + - name: Build sdist + wheel + run: python -m build + - name: Check package metadata + run: python -m twine check --strict dist/* + - name: Smoke-test the built wheel in a clean venv + # Runs from /tmp: with the repo root as cwd, `import recordstream` would find + # the SOURCE tree (cwd shadows site-packages) and silently test the repo + # instead of the wheel. This is the package-shadowing trap the workspace + # mandates warn about — the repo directory and the package share a name. + run: | + python -m venv /tmp/smoke + /tmp/smoke/bin/pip install --quiet dist/*.whl + cd /tmp + /tmp/smoke/bin/python - <<'EOF' + from importlib.metadata import version + + import numpy as np + + from recordstream import Image, Label, Stream, as_transform + + # A record is a plain dict of typed values; an op is type-dispatched. Running + # one record through a Stream exercises the engine, the item types and the + # op-family dispatch — the surfaces a broken wheel would take down. + records = [{"image": Image(np.zeros((4, 4, 3), dtype=np.float32)), + "class": Label("a", classes=["a", "b"])}] + stream = Stream(source=records, ops=[as_transform(lambda d: d + 1.0, handles=(Image,))]) + + out = list(stream) + assert len(out) == 1, out + assert float(out[0]["image"].max()) == 1.0 + assert out[0]["class"].value == "a" + assert len(stream) == 1 and stream[0]["class"].value == "a" # map-style protocol + print("wheel smoke-test OK, version:", version("recordstream")) + EOF + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: [build] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # required for PyPI Trusted Publishing (OIDC) + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/AGENTS.md b/AGENTS.md index 9d57d2b..bbaa1a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,7 @@ Core engine feature-complete on the **record model**; the full surface (items · - **The README Is The PyPI LANDING PAGE — Repo Links Are ABSOLUTE (2026-08-24):** `pyproject.toml` sets `readme = "README.md"`, so the README's rendered form IS the project page, and PyPI resolves a relative link against `pypi.org`, not against the repository — `docs/storage.md` becomes `pypi.org/docs/storage.md` and 404s. The link works perfectly on GitHub, so nothing in the repo catches it, which is why it is a test rather than a review habit: the README links to repo files as `https://github.com/Gearlux/recordstream/blob/main/` (26 links converted on 2026-08-24), while the `docs/*.md` pages are read on GitHub only and KEEP their relative links to each other. Spelling a README link absolutely also takes it out of reach of the existing rename check, so a second rule verifies each absolute self-link against the local tree — without it, converting the links would have traded a PyPI 404 for a GitHub one. Pins: `tests/test_docs_links.py::test_the_readme_links_to_repo_files_by_absolute_url` / `::test_readme_links_into_this_repo_name_a_file_that_exists`. - **A Dependency FLOOR States What Is TESTED, Not The Oldest Release That Once Worked (2026-08-24):** the floors were `confluid>=0.1.0` / `loggair>=0.1.0` / `liquifai>=0.1.0` long after the workspace had moved on, and one of them described a combination NOBODY can install — `uv pip install recordstream confluid==0.1.0` fails with *"Because liquifai<=0.1.0 depends on confluid>=0.2.0 and you require confluid==0.1.0 … your requirements are unsatisfiable"*. A stale floor is not harmless: it is a claim about what this package supports that no test covers and no resolver honours (a lowest-direct resolve lands on 0.3.0 / 0.2.0 / 0.2.0 anyway). Raise a floor to the version the suite actually runs against when a release makes that possible, and state WHY in a comment when the reason is not the obvious one. **Below 1.0 a pre-release segment is load-bearing in the OTHER direction too:** under PEP 440 a plain `>=0.1.0` EXCLUDES every pre-release of 0.1.0, so a consumer depending on this package while it ships `0.1.0a1` must write `recordstream>=0.1.0a1` or resolve to nothing. Pin: `tests/test_packaging.py`. -- **Release Metadata Is Part Of The Package, Not Paperwork (2026-08-24):** `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are each optional to the BUILD — the wheel builds byte-identically without them — which is exactly why all four were missing at once and why the gap is invisible until the page is published (`twine check` reports only `long_description missing`). The `Development Status` classifier must agree with the version: an `aN`/`bN` version is `3 - Alpha`, a final one `4 - Beta`; they drift in opposite directions because the version moves every release and the classifier is written once. Pins: `tests/test_packaging.py::test_the_landing_page_metadata_is_present` / `::test_the_development_status_classifier_matches_the_version` / `::test_the_declared_readme_and_license_files_exist` / `::test_no_requirement_is_a_direct_url` (PyPI refuses any `Requires-Dist` carrying a URL — `warehouse/forklift/metadata.py`, "Can't have direct dependency"). +- **Release Metadata Is Part Of The Package, Not Paperwork (2026-08-24):** `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are each optional to the BUILD — the wheel builds byte-identically without them — which is exactly why all four were missing at once and why the gap is invisible until the page is published (`twine check` reports only `long_description missing`). The `Development Status` classifier must agree with the version: an `aN`/`bN` version is `3 - Alpha`, a final one `4 - Beta`; they drift in opposite directions because the version moves every release and the classifier is written once. Pins: `tests/test_packaging.py::test_the_landing_page_metadata_is_present` / `::test_the_development_status_classifier_matches_the_version` / `::test_the_declared_readme_and_license_files_exist` / `::test_no_requirement_is_a_direct_url` (PyPI refuses any `Requires-Dist` carrying a URL — `warehouse/forklift/metadata.py`, "Can't have direct dependency"). **The sdist needs `MANIFEST.in` for anything nothing else references** — setuptools builds it from what is DECLARED (`readme` -> README.md, `license-files` -> LICENSE, `packages.find` + `package-data` -> the package), so `CHANGELOG.md` was silently absent from both projects' sdists until 2026-08-25. Pin: `tests/test_packaging.py::test_the_changelog_is_shipped_in_the_sdist`. ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. diff --git a/CHANGELOG.md b/CHANGELOG.md index 233b7e9..15ef5ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the versioning is [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0a1] — unreleased +## [0.1.0a1] — 2026-08-25 First public pre-release. The surface it ships: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e68f998 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +# What the sdist ships BEYOND what setuptools already infers. +# +# `readme = "README.md"` and `license-files = ["LICENSE"]` in pyproject already pull those +# two in, and `packages.find` + `[tool.setuptools.package-data]` pull the package. A +# CHANGELOG is referenced by nothing, so without this line it is silently absent from the +# sdist — which is the only copy of it a reader has while the repository is private. +# +# Pinned by tests/test_packaging.py::test_the_changelog_is_shipped_in_the_sdist +include CHANGELOG.md diff --git a/pyproject.toml b/pyproject.toml index d142140..17663eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,10 @@ torch = ["torch"] # torch/tensorflow/jax is actually installed, and the consumer's own extra picks one. keras = ["keras>=3.0"] dev = [ + # Release tooling: `python -m build` produces the sdist + wheel, `twine check` + # validates the metadata PyPI will re-validate on upload. + "build>=1.0.0", + "twine>=5.0.0", "black>=24.0.0,<25.0.0", "isort>=5.13.0,<6.0.0", "flake8>=6.0.0,<8.0.0", diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 13317a9..561a62a 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -95,3 +95,32 @@ def test_the_development_status_classifier_matches_the_version() -> None: expected = "3 - Alpha" if is_prerelease else "4 - Beta" assert expected in status[0], f"version {project['version']} does not match classifier {status[0]!r}" + + +def test_the_changelog_is_shipped_in_the_sdist() -> None: + """`MANIFEST.in` must carry `CHANGELOG.md` — nothing else pulls it in. + + setuptools builds an sdist from what is *referenced*: `readme` brings README.md, + `license-files` brings LICENSE, `packages.find` + `package-data` bring the package. + A CHANGELOG is referenced by nothing, so it is silently absent — measured on this + project's own sdist before `MANIFEST.in` existed. + + It matters more than a tidiness point here: `[project.urls] Changelog` points at the + file on GitHub, so while that repository is private the sdist is the ONLY copy a + reader can reach. + + Asserted on the declaration rather than by building an sdist, for the same reason + the package-data rules are: building one takes seconds and needs a clean tree, and + the declaration is the thing that actually goes stale. + """ + manifest = _REPO / "MANIFEST.in" + assert manifest.exists(), "MANIFEST.in is missing; the sdist would ship no CHANGELOG" + + included = { + line.split(maxsplit=1)[1].strip() + for line in manifest.read_text().splitlines() + if line.strip().startswith("include ") + } + missing = [name for name in ("CHANGELOG.md",) if name not in included] + + assert not missing, f"MANIFEST.in does not include: {missing}" From d758a2baad8f5a79a3e5063b6586fdbb2d183960 Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 25 Aug 2026 10:46:10 +0200 Subject: [PATCH 100/102] fix(ci): recordstream's own mypy config was missing three optional imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #7's Quality Gates failed on three pre-existing errors that no local check can see: keras ships no py.typed, scipy.ndimage stubs are absent, and matplotlib is not installed at all. None is a defect. All three are optional by design — `[keras]` and `[vision]` extras, plus the lazy matplotlib import inside ops.image._apply_colormap that only a non-gray colormap needs. What was missing is that this project's own override table never learned about them. The gap is invisible locally because the project is type-checked twice with different configs: `mypy recordstream` from the workspace root reads the shared mypy.ini, which already carries scipy and matplotlib, and reports Success on the same tree that CI fails. CI runs `mypy .` from the project directory and reads only pyproject's table — the config a published package must satisfy. Both invocations are now clean. AGENTS.md records why the project's own config has to be self-sufficient rather than leaning on the workspace one. --- AGENTS.md | 1 + pyproject.toml | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index bbaa1a4..cc07391 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,7 @@ Core engine feature-complete on the **record model**; the full surface (items · Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). - **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Boxes`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Boxes`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) +- **This Project's OWN mypy Config Must Be SELF-SUFFICIENT (2026-08-25):** it is type-checked twice with different configs, and the workspace-root one HIDES gaps in the other. `mypy recordstream` from the workspace root reads the shared `mypy.ini`, which already carries `[mypy-scipy.*]` and `[mypy-matplotlib.*]`; this project's own CI runs **`mypy .` from the project directory**, reads only `[[tool.mypy.overrides]]` in `pyproject.toml`, and is the config a PUBLISHED package has to satisfy. The gap is therefore invisible to every local check and surfaces only on the release PR — measured: PR #7's Quality Gates failed on three pre-existing errors (`keras` has no `py.typed`; `scipy.ndimage` stubs missing; `matplotlib` not found) while the workspace-root run reported `Success` on the same tree. **Every optional or lazily-imported third party needs an entry in the project's own override table, even when the workspace config already has one** — the `[keras]` / `[vision]` extras and the lazy `matplotlib` import in `ops.image._apply_colormap` are all correct by design; what was wrong was only that this project's config never learned about them. This does NOT relax the workspace mandate that mypy runs from the ROOT: that stays the routine invocation. Reproducing CI's `mypy .` is a DIAGNOSTIC for exactly this class of failure, not a new habit. - **The README Is The PyPI LANDING PAGE — Repo Links Are ABSOLUTE (2026-08-24):** `pyproject.toml` sets `readme = "README.md"`, so the README's rendered form IS the project page, and PyPI resolves a relative link against `pypi.org`, not against the repository — `docs/storage.md` becomes `pypi.org/docs/storage.md` and 404s. The link works perfectly on GitHub, so nothing in the repo catches it, which is why it is a test rather than a review habit: the README links to repo files as `https://github.com/Gearlux/recordstream/blob/main/` (26 links converted on 2026-08-24), while the `docs/*.md` pages are read on GitHub only and KEEP their relative links to each other. Spelling a README link absolutely also takes it out of reach of the existing rename check, so a second rule verifies each absolute self-link against the local tree — without it, converting the links would have traded a PyPI 404 for a GitHub one. Pins: `tests/test_docs_links.py::test_the_readme_links_to_repo_files_by_absolute_url` / `::test_readme_links_into_this_repo_name_a_file_that_exists`. - **A Dependency FLOOR States What Is TESTED, Not The Oldest Release That Once Worked (2026-08-24):** the floors were `confluid>=0.1.0` / `loggair>=0.1.0` / `liquifai>=0.1.0` long after the workspace had moved on, and one of them described a combination NOBODY can install — `uv pip install recordstream confluid==0.1.0` fails with *"Because liquifai<=0.1.0 depends on confluid>=0.2.0 and you require confluid==0.1.0 … your requirements are unsatisfiable"*. A stale floor is not harmless: it is a claim about what this package supports that no test covers and no resolver honours (a lowest-direct resolve lands on 0.3.0 / 0.2.0 / 0.2.0 anyway). Raise a floor to the version the suite actually runs against when a release makes that possible, and state WHY in a comment when the reason is not the obvious one. **Below 1.0 a pre-release segment is load-bearing in the OTHER direction too:** under PEP 440 a plain `>=0.1.0` EXCLUDES every pre-release of 0.1.0, so a consumer depending on this package while it ships `0.1.0a1` must write `recordstream>=0.1.0a1` or resolve to nothing. Pin: `tests/test_packaging.py`. - **Release Metadata Is Part Of The Package, Not Paperwork (2026-08-24):** `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are each optional to the BUILD — the wheel builds byte-identically without them — which is exactly why all four were missing at once and why the gap is invisible until the page is published (`twine check` reports only `long_description missing`). The `Development Status` classifier must agree with the version: an `aN`/`bN` version is `3 - Alpha`, a final one `4 - Beta`; they drift in opposite directions because the version moves every release and the classifier is written once. Pins: `tests/test_packaging.py::test_the_landing_page_metadata_is_present` / `::test_the_development_status_classifier_matches_the_version` / `::test_the_declared_readme_and_license_files_exist` / `::test_no_requirement_is_a_direct_url` (PyPI refuses any `Requires-Dist` carrying a URL — `warehouse/forklift/metadata.py`, "Can't have direct dependency"). **The sdist needs `MANIFEST.in` for anything nothing else references** — setuptools builds it from what is DECLARED (`readme` -> README.md, `license-files` -> LICENSE, `packages.find` + `package-data` -> the package), so `CHANGELOG.md` was silently absent from both projects' sdists until 2026-08-25. Pin: `tests/test_packaging.py::test_the_changelog_is_shipped_in_the_sdist`. diff --git a/pyproject.toml b/pyproject.toml index 17663eb..dd2605c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,12 +185,25 @@ line_length = 120 testpaths = ["tests"] python_files = ["test_*.py"] +# This project is type-checked TWICE with different configs, and only one of them is +# authoritative for a release. From the workspace root a shared `mypy.ini` supplies these +# ignores, so a gap here is invisible there; this project's OWN CI runs `mypy .` from the +# project directory, reads only the table below, and is what a published package must +# satisfy. Every optional or lazily-imported third party therefore needs an entry HERE, +# even when the workspace config already carries one. [[tool.mypy.overrides]] module = [ "albumentations.*", "datasets.*", "h5py.*", + # Lazily imported inside `ops.image._apply_colormap` (only a non-gray colormap needs + # it), so it is not a declared dependency at all — nothing installs it for CI. + "matplotlib.*", + # The `[keras]` extra. Keras ships no `py.typed`, so it is untyped rather than absent. + "keras.*", "PIL.*", + # The `[vision]` extra, behind a try/except in `ops.numpy.connected_component_boxes`. + "scipy.*", "torchvision.*", "zarr.*", ] From fbe04d815677f8d611be7fa255d936496eab9f6d Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 25 Aug 2026 10:52:57 +0200 Subject: [PATCH 101/102] fix(ci): matplotlib joins the dev extra so the colormap path is tested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ops.image._apply_colormap` lazily imports matplotlib for every non-gray colormap, and CI installs `.[dev,torch,keras]` — which had no matplotlib — so tests/test_typed_generic_ops.py::TestConvertToImage::test_parity_with_render_ helper_default_sizing died with ModuleNotFoundError. It passes locally only because the workspace venv has matplotlib installed for other reasons. Added to `dev` rather than skipped, matching the torchvision precedent two lines above: skipping would leave the colormap half of ConvertToImage and value_to_image untested in CI. It stays out of `dependencies` deliberately — the gray path is matplotlib-free by design, and a consumer rendering only grayscale must not be made to install a plotting library. Verified in a venv built to match CI exactly (`-e .[dev,torch,keras]` on 3.12): 841 passed, and all five examples run. --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index dd2605c..c35cd53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,12 @@ dev = [ # CI installs `.[dev]` (unit-tests AND verify-examples), so torchvision here keeps the # bare torchvision-v2 op family + examples exercised in CI without a workflow edit. "torchvision", + # Same reason: `ops.image._apply_colormap` lazily imports matplotlib for every + # non-gray colormap, and the gray path is matplotlib-free BY DESIGN — so without this + # the colormap half of ConvertToImage / value_to_image goes untested in CI. It stays + # out of `dependencies` deliberately: a consumer rendering only grayscale must not be + # made to install a plotting library. + "matplotlib", ] notebook = [ "matplotlib", From fd344a6bb82c1c7ce0f5fac11b1f47b9cebc95bd Mon Sep 17 00:00:00 2001 From: gearlux Date: Tue, 25 Aug 2026 11:03:48 +0200 Subject: [PATCH 102/102] fix(ci): DATA_ROOT is a cache preference, not a notebook requirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `notebooks/01 - cat_exploration.ipynb` raised RuntimeError when DATA_ROOT was unset, so Verify Notebooks failed on every runner — CI sets it nowhere and no .env is committed. It passed locally only because the notebook walks up to the workspace .env and loads it. The variable only chooses WHERE the HF cache lives. The notebook then loads huggingface/cats-image — one image — from the Hub, which needs no configured volume. So an unset DATA_ROOT now falls through to huggingface_hub's default cache, while a DATA_ROOT that is set but missing still raises: that is a misconfigured machine, not an absent preference. Verified both paths. In a checkout with no .env in any parent, matching a CI runner: "DATA_ROOT = (unset — using the default HF cache)" then "Dataset loaded with 1 examples." On this workstation, unchanged: HF_HOME = /Volumes/Store/ huggingface. Note that `env -u DATA_ROOT` alone reports a FALSE PASS — the notebook re-reads the .env itself — so the reproduction has to run outside the workspace. --- AGENTS.md | 2 +- notebooks/01 - cat_exploration.ipynb | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cc07391..05ec43b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,7 +58,7 @@ Core engine feature-complete on the **record model**; the full surface (items · Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). - **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Boxes`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Boxes`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) -- **This Project's OWN mypy Config Must Be SELF-SUFFICIENT (2026-08-25):** it is type-checked twice with different configs, and the workspace-root one HIDES gaps in the other. `mypy recordstream` from the workspace root reads the shared `mypy.ini`, which already carries `[mypy-scipy.*]` and `[mypy-matplotlib.*]`; this project's own CI runs **`mypy .` from the project directory**, reads only `[[tool.mypy.overrides]]` in `pyproject.toml`, and is the config a PUBLISHED package has to satisfy. The gap is therefore invisible to every local check and surfaces only on the release PR — measured: PR #7's Quality Gates failed on three pre-existing errors (`keras` has no `py.typed`; `scipy.ndimage` stubs missing; `matplotlib` not found) while the workspace-root run reported `Success` on the same tree. **Every optional or lazily-imported third party needs an entry in the project's own override table, even when the workspace config already has one** — the `[keras]` / `[vision]` extras and the lazy `matplotlib` import in `ops.image._apply_colormap` are all correct by design; what was wrong was only that this project's config never learned about them. This does NOT relax the workspace mandate that mypy runs from the ROOT: that stays the routine invocation. Reproducing CI's `mypy .` is a DIAGNOSTIC for exactly this class of failure, not a new habit. +- **This Project's OWN mypy Config Must Be SELF-SUFFICIENT (2026-08-25):** it is type-checked twice with different configs, and the workspace-root one HIDES gaps in the other. `mypy recordstream` from the workspace root reads the shared `mypy.ini`, which already carries `[mypy-scipy.*]` and `[mypy-matplotlib.*]`; this project's own CI runs **`mypy .` from the project directory**, reads only `[[tool.mypy.overrides]]` in `pyproject.toml`, and is the config a PUBLISHED package has to satisfy. The gap is therefore invisible to every local check and surfaces only on the release PR — measured: PR #7's Quality Gates failed on three pre-existing errors (`keras` has no `py.typed`; `scipy.ndimage` stubs missing; `matplotlib` not found) while the workspace-root run reported `Success` on the same tree. **Every optional or lazily-imported third party needs an entry in the project's own override table, even when the workspace config already has one** — the `[keras]` / `[vision]` extras and the lazy `matplotlib` import in `ops.image._apply_colormap` are all correct by design; what was wrong was only that this project's config never learned about them. This does NOT relax the workspace mandate that mypy runs from the ROOT: that stays the routine invocation. Reproducing CI's `mypy .` is a DIAGNOSTIC for exactly this class of failure, not a new habit. **A NOTEBOOK must not require the workspace environment either (2026-08-25):** `notebooks/01 - cat_exploration.ipynb` RAISED on a missing `DATA_ROOT`, so Verify Notebooks failed on every runner — while passing locally, because the notebook walks up to the workspace `.env` and loads it. `DATA_ROOT` chooses WHERE the HF cache lives; it is a preference, not a requirement, so an unset one now falls through to huggingface_hub's own cache and a SET-but-missing one still raises (a misconfigured machine, not an absent preference). **Reproducing this needs a checkout with no `.env` in ANY parent** — `env -u DATA_ROOT` is not enough and will report a false pass, because the notebook re-reads the file itself; copy the notebook somewhere outside the workspace and execute it there. - **The README Is The PyPI LANDING PAGE — Repo Links Are ABSOLUTE (2026-08-24):** `pyproject.toml` sets `readme = "README.md"`, so the README's rendered form IS the project page, and PyPI resolves a relative link against `pypi.org`, not against the repository — `docs/storage.md` becomes `pypi.org/docs/storage.md` and 404s. The link works perfectly on GitHub, so nothing in the repo catches it, which is why it is a test rather than a review habit: the README links to repo files as `https://github.com/Gearlux/recordstream/blob/main/` (26 links converted on 2026-08-24), while the `docs/*.md` pages are read on GitHub only and KEEP their relative links to each other. Spelling a README link absolutely also takes it out of reach of the existing rename check, so a second rule verifies each absolute self-link against the local tree — without it, converting the links would have traded a PyPI 404 for a GitHub one. Pins: `tests/test_docs_links.py::test_the_readme_links_to_repo_files_by_absolute_url` / `::test_readme_links_into_this_repo_name_a_file_that_exists`. - **A Dependency FLOOR States What Is TESTED, Not The Oldest Release That Once Worked (2026-08-24):** the floors were `confluid>=0.1.0` / `loggair>=0.1.0` / `liquifai>=0.1.0` long after the workspace had moved on, and one of them described a combination NOBODY can install — `uv pip install recordstream confluid==0.1.0` fails with *"Because liquifai<=0.1.0 depends on confluid>=0.2.0 and you require confluid==0.1.0 … your requirements are unsatisfiable"*. A stale floor is not harmless: it is a claim about what this package supports that no test covers and no resolver honours (a lowest-direct resolve lands on 0.3.0 / 0.2.0 / 0.2.0 anyway). Raise a floor to the version the suite actually runs against when a release makes that possible, and state WHY in a comment when the reason is not the obvious one. **Below 1.0 a pre-release segment is load-bearing in the OTHER direction too:** under PEP 440 a plain `>=0.1.0` EXCLUDES every pre-release of 0.1.0, so a consumer depending on this package while it ships `0.1.0a1` must write `recordstream>=0.1.0a1` or resolve to nothing. Pin: `tests/test_packaging.py`. - **Release Metadata Is Part Of The Package, Not Paperwork (2026-08-24):** `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are each optional to the BUILD — the wheel builds byte-identically without them — which is exactly why all four were missing at once and why the gap is invisible until the page is published (`twine check` reports only `long_description missing`). The `Development Status` classifier must agree with the version: an `aN`/`bN` version is `3 - Alpha`, a final one `4 - Beta`; they drift in opposite directions because the version moves every release and the classifier is written once. Pins: `tests/test_packaging.py::test_the_landing_page_metadata_is_present` / `::test_the_development_status_classifier_matches_the_version` / `::test_the_declared_readme_and_license_files_exist` / `::test_no_requirement_is_a_direct_url` (PyPI refuses any `Requires-Dist` carrying a URL — `warehouse/forklift/metadata.py`, "Can't have direct dependency"). **The sdist needs `MANIFEST.in` for anything nothing else references** — setuptools builds it from what is DECLARED (`readme` -> README.md, `license-files` -> LICENSE, `packages.find` + `package-data` -> the package), so `CHANGELOG.md` was silently absent from both projects' sdists until 2026-08-25. Pin: `tests/test_packaging.py::test_the_changelog_is_shipped_in_the_sdist`. diff --git a/notebooks/01 - cat_exploration.ipynb b/notebooks/01 - cat_exploration.ipynb index ec27f49..8f8a616 100644 --- a/notebooks/01 - cat_exploration.ipynb +++ b/notebooks/01 - cat_exploration.ipynb @@ -21,7 +21,7 @@ "from dotenv import load_dotenv\n", "\n", "# Walk up from the notebook to find the workspace .env (alongside the top-level\n", - "# CLAUDE.md / AGENTS.md). Loading it before importing `datasets` is required \u2014\n", + "# CLAUDE.md / AGENTS.md). Loading it before importing `datasets` is required —\n", "# huggingface_hub reads HF_HOME at import time of its cache module.\n", "here = Path.cwd()\n", "for candidate in [here, *here.parents]:\n", @@ -29,16 +29,21 @@ " load_dotenv(candidate / \".env\")\n", " break\n", "\n", + "# DATA_ROOT chooses WHERE the HF cache lives; it is not required to run this\n", + "# notebook. On a workstation it points the cache at the configured volume so a\n", + "# large dataset does not land in the home directory. Where it is unset — a CI\n", + "# runner, or a fresh clone with no .env — huggingface_hub uses its own default\n", + "# cache and the notebook runs unchanged. A DATA_ROOT that is SET but missing is\n", + "# still an error: that is a misconfigured machine, not an absent preference.\n", "data_root = os.getenv(\"DATA_ROOT\")\n", - "if not data_root:\n", - " raise RuntimeError(\"DATA_ROOT is not set \u2014 add it to the workspace .env (e.g. DATA_ROOT=/Volumes/Store).\")\n", - "if not Path(data_root).exists():\n", - " raise RuntimeError(f\"DATA_ROOT={data_root!r} does not exist on this machine \u2014 mount the volume or update .env.\")\n", + "if data_root:\n", + " if not Path(data_root).exists():\n", + " raise RuntimeError(f\"DATA_ROOT={data_root!r} does not exist on this machine — mount the volume or update .env.\")\n", + " # Point HF cache at the configured volume regardless of any inherited HF_HOME.\n", + " os.environ[\"HF_HOME\"] = str(Path(data_root) / \"huggingface\")\n", "\n", - "# Point HF cache at the configured volume regardless of any inherited HF_HOME.\n", - "os.environ[\"HF_HOME\"] = str(Path(data_root) / \"huggingface\")\n", - "print(f\"DATA_ROOT = {data_root}\")\n", - "print(f\"HF_HOME = {os.environ['HF_HOME']}\")" + "print(f\"DATA_ROOT = {data_root or '(unset — using the default HF cache)'}\")\n", + "print(f\"HF_HOME = {os.environ.get('HF_HOME', '(default)')}\")" ] }, {